Repository: nlpai-lab/KURE Branch: main Commit: 992e31d8e0b1 Files: 169 Total size: 1.6 MB Directory structure: gitextract_pyyh9iti/ ├── .gitignore ├── LICENSE ├── README.md ├── README_EN.md └── eval/ ├── evaluate.py ├── leaderboard.py ├── requirements.txt └── results/ ├── Alibaba-NLP/ │ ├── gte-Qwen2-7B-instruct/ │ │ └── Alibaba-NLP__gte-Qwen2-7B-instruct/ │ │ └── e26182b2122f4435e8b3ebecbf363990f409b45b/ │ │ ├── AutoRAGRetrieval.json │ │ ├── BelebeleRetrieval.json │ │ ├── Ko-StrategyQA.json │ │ ├── MIRACLRetrieval.json │ │ ├── MrTidyRetrieval.json │ │ ├── MultiLongDocRetrieval.json │ │ ├── PublicHealthQA.json │ │ ├── XPQARetrieval.json │ │ └── model_meta.json │ └── gte-multilingual-base/ │ └── Alibaba-NLP__gte-multilingual-base/ │ └── no_revision_available/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── BAAI/ │ ├── bge-m3/ │ │ └── BAAI__bge-m3/ │ │ └── no_revision_available/ │ │ ├── AutoRAGRetrieval.json │ │ ├── BelebeleRetrieval.json │ │ ├── Ko-StrategyQA.json │ │ ├── MIRACLRetrieval.json │ │ ├── MrTidyRetrieval.json │ │ ├── MultiLongDocRetrieval.json │ │ ├── PublicHealthQA.json │ │ ├── XPQARetrieval.json │ │ └── model_meta.json │ └── bge-multilingual-gemma2/ │ └── no_model_name_available/ │ └── no_revision_available/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── Salesforce/ │ └── SFR-Embedding-2_R/ │ └── Salesforce__SFR-Embedding-2_R/ │ └── 91762139d94ed4371a9fa31db5551272e0b83818/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── Snowflake/ │ └── snowflake-arctic-embed-l-v2.0/ │ └── no_model_name_available/ │ └── no_revision_available/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── dragonkue/ │ └── BGE-m3-ko/ │ └── dragonkue__BGE-m3-ko/ │ └── no_revision_available/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── intfloat/ │ ├── e5-mistral-7b-instruct/ │ │ └── intfloat__e5-mistral-7b-instruct/ │ │ └── 07163b72af1488142a360786df853f237b1a3ca1/ │ │ ├── AutoRAGRetrieval.json │ │ ├── BelebeleRetrieval.json │ │ ├── Ko-StrategyQA.json │ │ ├── MIRACLRetrieval.json │ │ ├── MrTidyRetrieval.json │ │ ├── MultiLongDocRetrieval.json │ │ ├── PublicHealthQA.json │ │ ├── XPQARetrieval.json │ │ └── model_meta.json │ ├── multilingual-e5-base/ │ │ └── intfloat__multilingual-e5-base/ │ │ └── d13f1b27baf31030b7fd040960d60d909913633f/ │ │ ├── AutoRAGRetrieval.json │ │ ├── BelebeleRetrieval.json │ │ ├── Ko-StrategyQA.json │ │ ├── MIRACLRetrieval.json │ │ ├── MrTidyRetrieval.json │ │ ├── MultiLongDocRetrieval.json │ │ ├── PublicHealthQA.json │ │ ├── XPQARetrieval.json │ │ └── model_meta.json │ ├── multilingual-e5-large/ │ │ └── intfloat__multilingual-e5-large/ │ │ └── ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/ │ │ ├── AutoRAGRetrieval.json │ │ ├── BelebeleRetrieval.json │ │ ├── Ko-StrategyQA.json │ │ ├── MIRACLRetrieval.json │ │ ├── MrTidyRetrieval.json │ │ ├── MultiLongDocRetrieval.json │ │ ├── PublicHealthQA.json │ │ ├── XPQARetrieval.json │ │ └── model_meta.json │ └── multilingual-e5-large-instruct/ │ └── intfloat__multilingual-e5-large-instruct/ │ └── baa7be480a7de1539afce709c8f13f833a510e0a/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── jhgan/ │ └── ko-sroberta-multitask/ │ └── jhgan__ko-sroberta-multitask/ │ └── no_revision_available/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── jinaai/ │ └── jina-embeddings-v3/ │ └── jinaai__jina-embeddings-v3/ │ └── 215a6e121fa0183376388ac6b1ae230326bfeaed/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── nlpai-lab/ │ ├── KURE-v1/ │ │ └── nlpai-lab__KURE-v1/ │ │ └── no_revision_available/ │ │ ├── AutoRAGRetrieval.json │ │ ├── BelebeleRetrieval.json │ │ ├── Ko-StrategyQA.json │ │ ├── MIRACLRetrieval.json │ │ ├── MrTidyRetrieval.json │ │ ├── MultiLongDocRetrieval.json │ │ ├── PublicHealthQA.json │ │ ├── XPQARetrieval.json │ │ └── model_meta.json │ └── KoE5/ │ └── no_model_name_available/ │ └── no_revision_available/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── nomic-ai/ │ └── nomic-embed-text-v2-moe/ │ └── no_model_name_available/ │ └── no_revision_available/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json ├── openai/ │ └── text-embedding-3-large/ │ └── no_model_name_available/ │ └── no_revision_available/ │ ├── AutoRAGRetrieval.json │ ├── BelebeleRetrieval.json │ ├── Ko-StrategyQA.json │ ├── MIRACLRetrieval.json │ ├── MrTidyRetrieval.json │ ├── MultiLongDocRetrieval.json │ ├── PublicHealthQA.json │ ├── XPQARetrieval.json │ └── model_meta.json └── upskyy/ └── bge-m3-korean/ └── upskyy__bge-m3-korean/ └── no_revision_available/ ├── AutoRAGRetrieval.json ├── BelebeleRetrieval.json ├── Ko-StrategyQA.json ├── MIRACLRetrieval.json ├── MrTidyRetrieval.json ├── MultiLongDocRetrieval.json ├── PublicHealthQA.json ├── XPQARetrieval.json └── model_meta.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ __pycache__ ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) [year] [fullname] Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # 🔎 KURE: Korea University Retrieval Embedding model ## Update Logs - 2024.12.21: [🤗 KURE-v1](https://huggingface.co/nlpai-lab/KURE-v1), [MTEB-ko-retrieval Leaderboard](https://github.com/nlpai-lab/KURE?tab=readme-ov-file#mteb-ko-retrieval-leaderboard) 공개 - 2024.10.02: [🤗 KoE5](https://huggingface.co/nlpai-lab/KoE5), [🤗 ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0) 공개 ---
KURE는 고려대학교 [NLP & AI 연구실](http://nlp.korea.ac.kr/)과 [HIAI 연구소](http://hiai.korea.ac.kr)가 개발한 한국어 특화 임베딩 모델입니다. KURE를 공개합니다.
## KURE 모델 실행 코드 ### sentence-transformers로 실행 ```bash pip install sentence-transformers ``` 아래 예제 코드로 실행해볼 수 있습니다. ```python from sentence_transformers import SentenceTransformer # Download from the 🤗 Hub model = SentenceTransformer("nlpai-lab/KURE-v1") # model = SentenceTransformer("nlpai-lab/KoE5") # Run inference sentences = [ '헌법과 법원조직법은 어떤 방식을 통해 기본권 보장 등의 다양한 법적 모색을 가능하게 했어', '4. 시사점과 개선방향 앞서 살펴본 바와 같이 우리 헌법과 ｢법원조직 법｣은 대법원 구성을 다양화하여 기본권 보장과 민주주의 확립에 있어 다각적인 법적 모색을 가능하게 하는 것을 근본 규범으로 하고 있다. 더욱이 합의체로서의 대법원 원리를 채택하고 있는 것 역시 그 구성의 다양성을 요청하는 것으로 해석된다. 이와 같은 관점에서 볼 때 현직 법원장급 고위법관을 중심으로 대법원을 구성하는 관행은 개선할 필요가 있는 것으로 보인다.', '연방헌법재판소는 2001년 1월 24일 5:3의 다수견해로 「법원조직법」 제169조 제2문이 헌법에 합치된다는 판결을 내렸음 ○ 5인의 다수 재판관은 소송관계인의 인격권 보호, 공정한 절차의 보장과 방해받지 않는 법과 진실 발견 등을 근거로 하여 텔레비전 촬영에 대한 절대적인 금지를 헌법에 합치하는 것으로 보았음 ○ 그러나 나머지 3인의 재판관은 행정법원의 소송절차는 특별한 인격권 보호의 이익도 없으며, 텔레비전 공개주의로 인해 법과 진실 발견의 과정이 언제나 위태롭게 되는 것은 아니라면서 반대의견을 제시함 ○ 왜냐하면 행정법원의 소송절차에서는 소송당사자가 개인적으로 직접 심리에 참석하기보다는 변호사가 참석하는 경우가 많으며, 심리대상도 사실문제가 아닌 법률문제가 대부분이기 때문이라는 것임 □ 한편, 연방헌법재판소는 「연방헌법재판소법」(Bundesverfassungsgerichtsgesetz: BVerfGG) 제17a조에 따라 제한적이나마 재판에 대한 방송을 허용하고 있음 ○ 「연방헌법재판소법」 제17조에서 「법원조직법」 제14절 내지 제16절의 규정을 준용하도록 하고 있지만, 녹음이나 촬영을 통한 재판공개와 관련하여서는 「법원조직법」과 다른 내용을 규정하고 있음', ] embeddings = model.encode(sentences) print(embeddings.shape) # [3, 1024] # Get the similarity scores for the embeddings similarities = model.similarity(embeddings, embeddings) print(similarities) # Results for KURE-v1 # tensor([[1.0000, 0.6967, 0.5306], # [0.6967, 1.0000, 0.4427], # [0.5306, 0.4427, 1.0000]]) # Results for KoE5 # tensor([[1.0000, 0.6721, 0.3897], # [0.6721, 1.0000, 0.3740], # [0.3897, 0.3740, 1.0000]]) ```
## MTEB-ko-retrieval Leaderboard [MTEB](https://github.com/embeddings-benchmark/mteb)에 등록된 모든 Korean Retrieval Benchmark에 대한 평가를 진행하였습니다. ### Korean Retrieval Benchmark - [Ko-StrategyQA](https://huggingface.co/datasets/taeminlee/Ko-StrategyQA): 한국어 ODQA multi-hop 검색 데이터셋 (StrategyQA 번역) - [AutoRAGRetrieval](https://huggingface.co/datasets/yjoonjang/markers_bm): 금융, 공공, 의료, 법률, 커머스 5개 분야에 대해, pdf를 파싱하여 구성한 한국어 문서 검색 데이터셋 - [MIRACLRetrieval](https://huggingface.co/datasets/miracl/miracl): Wikipedia 기반의 한국어 문서 검색 데이터셋 - [PublicHealthQA](https://huggingface.co/datasets/xhluca/publichealth-qa): 의료 및 공중보건 도메인에 대한 한국어 문서 검색 데이터셋 - [BelebeleRetrieval](https://huggingface.co/datasets/facebook/belebele): FLORES-200 기반의 한국어 문서 검색 데이터셋 - [MrTidyRetrieval](https://huggingface.co/datasets/mteb/mrtidy): Wikipedia 기반의 한국어 문서 검색 데이터셋 - [MultiLongDocRetrieval](https://huggingface.co/datasets/Shitao/MLDR): 다양한 도메인의 한국어 장문 검색 데이터셋 - [XPQARetrieval](https://huggingface.co/datasets/jinaai/xpqa): 다양한 도메인의 한국어 문서 검색 데이터셋 ### Evaluation code `evaluate.py`에 모델을 추가하여 mteb를 활용한 평가를 진행할 수 있습니다. ```bash cd eval pip install -r requirements.txt python evaluate.py ``` ### Leaderboard streamlit을 통해 모든 모델의 모든 태스크에 대한 평가 결과를 시각화합니다. ```bash streamlit run leaderboard.py ``` 아래는 모든 모델의, 모든 벤치마크 데이터셋에 대한 평균 결과입니다. 자세한 결과는 `eval/results`폴더에서 확인하실 수 있습니다. ### Top-k 1 | Model | Average Recall | Average Precision | Average NDCG | Average F1 | |-----------------------------------------|----------------------|------------------------|-------------------|-----------------| | **nlpai-lab/KURE-v1** | **0.52640** | **0.60551** | **0.60551** | **0.55784** | | dragonkue/BGE-m3-ko | 0.52361 | 0.60394 | 0.60394 | 0.55535 | | BAAI/bge-m3 | 0.51778 | 0.59846 | 0.59846 | 0.54998 | | Snowflake/snowflake-arctic-embed-l-v2.0 | 0.51246 | 0.59384 | 0.59384 | 0.54489 | | nlpai-lab/KoE5 | 0.50157 | 0.57790 | 0.57790 | 0.53178 | | intfloat/multilingual-e5-large | 0.50052 | 0.57727 | 0.57727 | 0.53122 | | jinaai/jina-embeddings-v3 | 0.48287 | 0.56068 | 0.56068 | 0.51361 | | BAAI/bge-multilingual-gemma2 | 0.47904 | 0.55472 | 0.55472 | 0.50916 | | intfloat/multilingual-e5-large-instruct | 0.47842 | 0.55435 | 0.55435 | 0.50826 | | intfloat/multilingual-e5-base | 0.46950 | 0.54490 | 0.54490 | 0.49947 | | intfloat/e5-mistral-7b-instruct | 0.46772 | 0.54394 | 0.54394 | 0.49781 | | Alibaba-NLP/gte-multilingual-base | 0.46469 | 0.53744 | 0.53744 | 0.49353 | | Alibaba-NLP/gte-Qwen2-7B-instruct | 0.46633 | 0.53625 | 0.53625 | 0.49429 | | openai/text-embedding-3-large | 0.44884 | 0.51688 | 0.51688 | 0.47572 | | Salesforce/SFR-Embedding-2_R | 0.43748 | 0.50815 | 0.50815 | 0.46504 | | upskyy/bge-m3-korean | 0.43125 | 0.50245 | 0.50245 | 0.45945 | | jhgan/ko-sroberta-multitask | 0.33788 | 0.38497 | 0.38497 | 0.35678 | ### Top-k 3 | Model | Average Recall | Average Precision | Average NDCG | Average F1 | |-----------------------------------------|----------------------|------------------------|-------------------|-----------------| | **nlpai-lab/KURE-v1** | **0.68678** | **0.28711** | **0.65538** | **0.39835** | | dragonkue/BGE-m3-ko | 0.67834 | 0.28385 | 0.64950 | 0.39378 | | BAAI/bge-m3 | 0.67526 | 0.28374 | 0.64556 | 0.39291 | | Snowflake/snowflake-arctic-embed-l-v2.0 | 0.67128 | 0.28193 | 0.64042 | 0.39072 | | intfloat/multilingual-e5-large | 0.65807 | 0.27777 | 0.62822 | 0.38423 | | nlpai-lab/KoE5 | 0.65174 | 0.27329 | 0.62369 | 0.37882 | | BAAI/bge-multilingual-gemma2 | 0.64415 | 0.27416 | 0.61105 | 0.37782 | | jinaai/jina-embeddings-v3 | 0.64116 | 0.27165 | 0.60954 | 0.37511 | | intfloat/multilingual-e5-large-instruct | 0.64353 | 0.27040 | 0.60790 | 0.37453 | | Alibaba-NLP/gte-multilingual-base | 0.63744 | 0.26404 | 0.59695 | 0.36764 | | Alibaba-NLP/gte-Qwen2-7B-instruct | 0.63163 | 0.25937 | 0.59237 | 0.36263 | | intfloat/multilingual-e5-base | 0.62099 | 0.26144 | 0.59179 | 0.36203 | | intfloat/e5-mistral-7b-instruct | 0.62087 | 0.26144 | 0.58917 | 0.36188 | | openai/text-embedding-3-large | 0.61035 | 0.25356 | 0.57329 | 0.35270 | | Salesforce/SFR-Embedding-2_R | 0.60001 | 0.25253 | 0.56346 | 0.34952 | | upskyy/bge-m3-korean | 0.59215 | 0.25076 | 0.55722 | 0.34623 | | jhgan/ko-sroberta-multitask | 0.46930 | 0.18994 | 0.43293 | 0.26696 | ### Top-k 5 | Model | Average Recall | Average Precision | Average NDCG | Average F1 | |-----------------------------------------|----------------------|------------------------|-------------------|-----------------| | **nlpai-lab/KURE-v1** | **0.73851** | **0.19130** | **0.67479** | **0.29903** | | dragonkue/BGE-m3-ko | 0.72517 | 0.18799 | 0.66692 | 0.29401 | | BAAI/bge-m3 | 0.72954 | 0.18975 | 0.66615 | 0.29632 | | Snowflake/snowflake-arctic-embed-l-v2.0 | 0.72962 | 0.18875 | 0.66236 | 0.29542 | | nlpai-lab/KoE5 | 0.70820 | 0.18287 | 0.64499 | 0.28628 | | intfloat/multilingual-e5-large | 0.70124 | 0.18316 | 0.64402 | 0.28588 | | BAAI/bge-multilingual-gemma2 | 0.70258 | 0.18556 | 0.63338 | 0.28851 | | jinaai/jina-embeddings-v3 | 0.69933 | 0.18256 | 0.63133 | 0.28505 | | intfloat/multilingual-e5-large-instruct | 0.69018 | 0.17838 | 0.62486 | 0.27933 | | Alibaba-NLP/gte-multilingual-base | 0.69365 | 0.17789 | 0.61896 | 0.27879 | | intfloat/multilingual-e5-base | 0.67250 | 0.17406 | 0.61119 | 0.27247 | | Alibaba-NLP/gte-Qwen2-7B-instruct | 0.67447 | 0.17114 | 0.60952 | 0.26943 | | intfloat/e5-mistral-7b-instruct | 0.67449 | 0.17484 | 0.60935 | 0.27349 | | openai/text-embedding-3-large | 0.66365 | 0.17004 | 0.59389 | 0.26677 | | Salesforce/SFR-Embedding-2_R | 0.65622 | 0.17018 | 0.58494 | 0.26612 | | upskyy/bge-m3-korean | 0.65477 | 0.17015 | 0.58073 | 0.26589 | | jhgan/ko-sroberta-multitask | 0.53136 | 0.13264 | 0.45879 | 0.20976 | ### Top-k 10 | Model | Average Recall | Average Precision | Average NDCG | Average F1 | |-----------------------------------------|----------------------|------------------------|-------------------|-----------------| | **nlpai-lab/KURE-v1** | **0.79682** | **0.10624** | **0.69473** | **0.18524** | | dragonkue/BGE-m3-ko | 0.78450 | 0.10492 | 0.68748 | 0.18288 | | BAAI/bge-m3 | 0.79195 | 0.10592 | 0.68723 | 0.18456 | | Snowflake/snowflake-arctic-embed-l-v2.0 | 0.78669 | 0.10462 | 0.68189 | 0.18260 | | intfloat/multilingual-e5-large | 0.75902 | 0.10147 | 0.66370 | 0.17693 | | nlpai-lab/KoE5 | 0.75296 | 0.09937 | 0.66012 | 0.17369 | | BAAI/bge-multilingual-gemma2 | 0.76153 | 0.10364 | 0.65330 | 0.18003 | | jinaai/jina-embeddings-v3 | 0.76277 | 0.10240 | 0.65290 | 0.17843 | | intfloat/multilingual-e5-large-instruct | 0.74851 | 0.09888 | 0.64451 | 0.17283 | | Alibaba-NLP/gte-multilingual-base | 0.75631 | 0.09938 | 0.64025 | 0.17363 | | Alibaba-NLP/gte-Qwen2-7B-instruct | 0.74092 | 0.09607 | 0.63258 | 0.16847 | | intfloat/multilingual-e5-base | 0.73512 | 0.09717 | 0.63216 | 0.16977 | | intfloat/e5-mistral-7b-instruct | 0.73795 | 0.09777 | 0.63076 | 0.17078 | | openai/text-embedding-3-large | 0.72946 | 0.09571 | 0.61670 | 0.16739 | | Salesforce/SFR-Embedding-2_R | 0.71662 | 0.09546 | 0.60589 | 0.16651 | | upskyy/bge-m3-korean | 0.71895 | 0.09583 | 0.60258 | 0.16712 | | jhgan/ko-sroberta-multitask | 0.61225 | 0.07826 | 0.48687 | 0.13757 |
## Training Details - KURE-v1은 [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3)를 기반으로 fine-tuning된 모델입니다. - KoE5는 [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large)를 기반으로 fine-tuning된 모델입니다. ### Training Data **KURE-v1** - 한국어 query-document-hard_negative(5개) 데이터 쌍 - 약 2,000,000 examples **KoE5** - [ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0) - 한국어 query-document-hard_negative(1개) 데이터 쌍 (open data) - 약 700,000+ examples ### Training Procedure **KURE-v1** - loss: [CachedGISTEmbedLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedgistembedloss) - batch size: 4096 - learning rate: 2e-05 - epochs: 1 **KoE5** - loss: [CachedMultipleNegativesRankingLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedmultiplenegativesrankingloss) - batch size: 512 - learning rate: 1e-05 - epochs: 1
## 주의사항 - KoE5 사용 시, prefix를 붙여 주어야 합니다. (query: {query}, passage: {document}) ## License - ```MIT``` ## Citation If you find our paper or models helpful, please consider cite as follows: ```text @inproceedings{jang2025kure, title={KURE: Embedding Model for Korean-Specific Retrieval}, author={Jang, Youngjoon and Son, Junyoung and Lee, Taemin and Hong, Seongtae and Park, JeongBae and Lim, Heuiseok}, booktitle={Annual Conference on Human and Language Technology}, pages={129--134}, year={2025}, organization={Human and Language Technology} }, @inproceedings{jang2024koe5, title={KoE5: A New Dataset and Model for Improving Korean Embedding Performance}, author={Jang, Youngjoon and Son, Junyoung and Park, Chanjun and Choi, Soonwoo and Lee, Byeonggoo and Lee, Taemin and Lim, Heuiseok}, booktitle={Annual Conference on Human and Language Technology}, pages={239--244}, year={2024}, organization={Human and Language Technology} } ``` ================================================ FILE: README_EN.md ================================================ # 🔎 KURE: Korea University Retrieval Embedding model ## Update Logs - 2024.12.21: [🤗 KURE-v1](https://huggingface.co/nlpai-lab/KURE-v1) and the [MTEB-ko-retrieval Leaderboard](https://github.com/nlpai-lab/KURE/blob/main/README_EN.md#mteb-ko-retrieval-leaderboard) have been released. - 2024.10.02: [🤗 KoE5](https://huggingface.co/nlpai-lab/KoE5) and [🤗 ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0) have been released. ---
KURE is a Korean-specific embedding model developed by Korea University's [NLP & AI Lab](http://nlp.korea.ac.kr/) and [HIAI Research Institute](http://hiai.korea.ac.kr). We are excited to release KURE.
## How to Use KURE Models ### With sentence-transformers ```bash pip install sentence-transformers ``` You can run the model with the following example code. ```python from sentence_transformers import SentenceTransformer # Download from the 🤗 Hub model = SentenceTransformer("nlpai-lab/KURE-v1") # model = SentenceTransformer("nlpai-lab/KoE5") # Run inference sentences = [ '헌법과 법원조직법은 어떤 방식을 통해 기본권 보장 등의 다양한 법적 모색을 가능하게 했어', '4. 시사점과 개선방향 앞서 살펴본 바와 같이 우리 헌법과 ｢법원조직 법｣은 대법원 구성을 다양화하여 기본권 보장과 민주주의 확립에 있어 다각적인 법적 모색을 가능하게 하는 것을 근본 규범으로 하고 있다. 더욱이 합의체로서의 대법원 원리를 채택하고 있는 것 역시 그 구성의 다양성을 요청하는 것으로 해석된다. 이와 같은 관점에서 볼 때 현직 법원장급 고위법관을 중심으로 대법원을 구성하는 관행은 개선할 필요가 있는 것으로 보인다.', '연방헌법재판소는 2001년 1월 24일 5:3의 다수견해로 「법원조직법」 제169조 제2문이 헌법에 합치된다는 판결을 내렸음 ○ 5인의 다수 재판관은 소송관계인의 인격권 보호, 공정한 절차의 보장과 방해받지 않는 법과 진실 발견 등을 근거로 하여 텔레비전 촬영에 대한 절대적인 금지를 헌법에 합치하는 것으로 보았음 ○ 그러나 나머지 3인의 재판관은 행정법원의 소송절차는 특별한 인격권 보호의 이익도 없으며, 텔레비전 공개주의로 인해 법과 진실 발견의 과정이 언제나 위태롭게 되는 것은 아니라면서 반대의견을 제시함 ○ 왜냐하면 행정법원의 소송절차에서는 소송당사자가 개인적으로 직접 심리에 참석하기보다는 변호사가 참석하는 경우가 많으며, 심리대상도 사실문제가 아닌 법률문제가 대부분이기 때문이라는 것임 □ 한편, 연방헌법재판소는 「연방헌법재판소법」(Bundesverfassungsgerichtsgesetz: BVerfGG) 제17a조에 따라 제한적이나마 재판에 대한 방송을 허용하고 있음 ○ 「연방헌법재판소법」 제17조에서 「법원조직법」 제14절 내지 제16절의 규정을 준용하도록 하고 있지만, 녹음이나 촬영을 통한 재판공개와 관련하여서는 「법원조직법」과 다른 내용을 규정하고 있음', ] embeddings = model.encode(sentences) print(embeddings.shape) # [3, 1024] # Get the similarity scores for the embeddings similarities = model.similarity(embeddings, embeddings) print(similarities) # Results for KURE-v1 # tensor([[1.0000, 0.6967, 0.5306], # [0.6967, 1.0000, 0.4427], # [0.5306, 0.4427, 1.0000]]) # Results for KoE5 # tensor([[1.0000, 0.6721, 0.3897], # [0.6721, 1.0000, 0.3740], # [0.3897, 0.3740, 1.0000]]) ```
## MTEB-ko-retrieval Leaderboard We evaluated our models on all Korean Retrieval Benchmarks registered in [MTEB](https://github.com/embeddings-benchmark/mteb). ### Korean Retrieval Benchmark - [Ko-StrategyQA](https://huggingface.co/datasets/taeminlee/Ko-StrategyQA): Korean ODQA multi-hop retrieval dataset (Translated from StrategyQA). - [AutoRAGRetrieval](https://huggingface.co/datasets/yjoonjang/markers_bm): A Korean document retrieval dataset constructed by parsing PDFs from 5 domains: finance, public, medical, legal, and commerce. - [MIRACLRetrieval](https://huggingface.co/datasets/miracl/miracl): Wikipedia-based Korean document retrieval dataset. - [PublicHealthQA](https://huggingface.co/datasets/xhluca/publichealth-qa): Korean document retrieval dataset for the medical and public health domains. - [BelebeleRetrieval](https://huggingface.co/datasets/facebook/belebele): FLORES-200 based Korean document retrieval dataset. - [MrTidyRetrieval](https://huggingface.co/datasets/mteb/mrtidy): Wikipedia-based Korean document retrieval dataset. - [MultiLongDocRetrieval](https://huggingface.co/datasets/Shitao/MLDR): Korean long-document retrieval dataset from various domains. - [XPQARetrieval](https://huggingface.co/datasets/jinaai/xpqa): Korean document retrieval dataset from various domains. ### Evaluation code You can add a model to `evaluate.py` to evaluate it using MTEB. ```bash cd eval pip install -r requirements.txt python evaluate.py ``` ### Leaderboard We visualize the evaluation results for all models on all tasks via streamlit. ```bash streamlit run leaderboard.py ``` Below are the average results for all models across all benchmark datasets. Detailed results can be found in the `eval/results` folder. ### Top-k 1 | Model | Average Recall | Average Precision | Average NDCG | Average F1 | |-----------------------------------------|----------------------|------------------------|-------------------|-----------------| | **nlpai-lab/KURE-v1** | **0.52640** | **0.60551** | **0.60551** | **0.55784** | | dragonkue/BGE-m3-ko | 0.52361 | 0.60394 | 0.60394 | 0.55535 | | BAAI/bge-m3 | 0.51778 | 0.59846 | 0.59846 | 0.54998 | | Snowflake/snowflake-arctic-embed-l-v2.0 | 0.51246 | 0.59384 | 0.59384 | 0.54489 | | nlpai-lab/KoE5 | 0.50157 | 0.57790 | 0.57790 | 0.53178 | | intfloat/multilingual-e5-large | 0.50052 | 0.57727 | 0.57727 | 0.53122 | | jinaai/jina-embeddings-v3 | 0.48287 | 0.56068 | 0.56068 | 0.51361 | | BAAI/bge-multilingual-gemma2 | 0.47904 | 0.55472 | 0.55472 | 0.50916 | | intfloat/multilingual-e5-large-instruct | 0.47842 | 0.55435 | 0.55435 | 0.50826 | | intfloat/multilingual-e5-base | 0.46950 | 0.54490 | 0.54490 | 0.49947 | | intfloat/e5-mistral-7b-instruct | 0.46772 | 0.54394 | 0.54394 | 0.49781 | | Alibaba-NLP/gte-multilingual-base | 0.46469 | 0.53744 | 0.53744 | 0.49353 | | Alibaba-NLP/gte-Qwen2-7B-instruct | 0.46633 | 0.53625 | 0.53625 | 0.49429 | | openai/text-embedding-3-large | 0.44884 | 0.51688 | 0.51688 | 0.47572 | | Salesforce/SFR-Embedding-2_R | 0.43748 | 0.50815 | 0.50815 | 0.46504 | | upskyy/bge-m3-korean | 0.43125 | 0.50245 | 0.50245 | 0.45945 | | jhgan/ko-sroberta-multitask | 0.33788 | 0.38497 | 0.38497 | 0.35678 | ### Top-k 3 | Model | Average Recall | Average Precision | Average NDCG | Average F1 | |-----------------------------------------|----------------------|------------------------|-------------------|-----------------| | **nlpai-lab/KURE-v1** | **0.68678** | **0.28711** | **0.65538** | **0.39835** | | dragonkue/BGE-m3-ko | 0.67834 | 0.28385 | 0.64950 | 0.39378 | | BAAI/bge-m3 | 0.67526 | 0.28374 | 0.64556 | 0.39291 | | Snowflake/snowflake-arctic-embed-l-v2.0 | 0.67128 | 0.28193 | 0.64042 | 0.39072 | | intfloat/multilingual-e5-large | 0.65807 | 0.27777 | 0.62822 | 0.38423 | | nlpai-lab/KoE5 | 0.65174 | 0.27329 | 0.62369 | 0.37882 | | BAAI/bge-multilingual-gemma2 | 0.64415 | 0.27416 | 0.61105 | 0.37782 | | jinaai/jina-embeddings-v3 | 0.64116 | 0.27165 | 0.60954 | 0.37511 | | intfloat/multilingual-e5-large-instruct | 0.64353 | 0.27040 | 0.60790 | 0.37453 | | Alibaba-NLP/gte-multilingual-base | 0.63744 | 0.26404 | 0.59695 | 0.36764 | | Alibaba-NLP/gte-Qwen2-7B-instruct | 0.63163 | 0.25937 | 0.59237 | 0.36263 | | intfloat/multilingual-e5-base | 0.62099 | 0.26144 | 0.59179 | 0.36203 | | intfloat/e5-mistral-7b-instruct | 0.62087 | 0.26144 | 0.58917 | 0.36188 | | openai/text-embedding-3-large | 0.61035 | 0.25356 | 0.57329 | 0.35270 | | Salesforce/SFR-Embedding-2_R | 0.60001 | 0.25253 | 0.56346 | 0.34952 | | upskyy/bge-m3-korean | 0.59215 | 0.25076 | 0.55722 | 0.34623 | | jhgan/ko-sroberta-multitask | 0.46930 | 0.18994 | 0.43293 | 0.26696 | ### Top-k 5 | Model | Average Recall | Average Precision | Average NDCG | Average F1 | |-----------------------------------------|----------------------|------------------------|-------------------|-----------------| | **nlpai-lab/KURE-v1** | **0.73851** | **0.19130** | **0.67479** | **0.29903** | | dragonkue/BGE-m3-ko | 0.72517 | 0.18799 | 0.66692 | 0.29401 | | BAAI/bge-m3 | 0.72954 | 0.18975 | 0.66615 | 0.29632 | | Snowflake/snowflake-arctic-embed-l-v2.0 | 0.72962 | 0.18875 | 0.66236 | 0.29542 | | nlpai-lab/KoE5 | 0.70820 | 0.18287 | 0.64499 | 0.28628 | | intfloat/multilingual-e5-large | 0.70124 | 0.18316 | 0.64402 | 0.28588 | | BAAI/bge-multilingual-gemma2 | 0.70258 | 0.18556 | 0.63338 | 0.28851 | | jinaai/jina-embeddings-v3 | 0.69933 | 0.18256 | 0.63133 | 0.28505 | | intfloat/multilingual-e5-large-instruct | 0.69018 | 0.17838 | 0.62486 | 0.27933 | | Alibaba-NLP/gte-multilingual-base | 0.69365 | 0.17789 | 0.61896 | 0.27879 | | intfloat/multilingual-e5-base | 0.67250 | 0.17406 | 0.61119 | 0.27247 | | Alibaba-NLP/gte-Qwen2-7B-instruct | 0.67447 | 0.17114 | 0.60952 | 0.26943 | | intfloat/e5-mistral-7b-instruct | 0.67449 | 0.17484 | 0.60935 | 0.27349 | | openai/text-embedding-3-large | 0.66365 | 0.17004 | 0.59389 | 0.26677 | | Salesforce/SFR-Embedding-2_R | 0.65622 | 0.17018 | 0.58494 | 0.26612 | | upskyy/bge-m3-korean | 0.65477 | 0.17015 | 0.58073 | 0.26589 | | jhgan/ko-sroberta-multitask | 0.53136 | 0.13264 | 0.45879 | 0.20976 | ### Top-k 10 | Model | Average Recall | Average Precision | Average NDCG | Average F1 | |-----------------------------------------|----------------------|------------------------|-------------------|-----------------| | **nlpai-lab/KURE-v1** | **0.79682** | **0.10624** | **0.69473** | **0.18524** | | dragonkue/BGE-m3-ko | 0.78450 | 0.10492 | 0.68748 | 0.18288 | | BAAI/bge-m3 | 0.79195 | 0.10592 | 0.68723 | 0.18456 | | Snowflake/snowflake-arctic-embed-l-v2.0 | 0.78669 | 0.10462 | 0.68189 | 0.18260 | | intfloat/multilingual-e5-large | 0.75902 | 0.10147 | 0.66370 | 0.17693 | | nlpai-lab/KoE5 | 0.75296 | 0.09937 | 0.66012 | 0.17369 | | BAAI/bge-multilingual-gemma2 | 0.76153 | 0.10364 | 0.65330 | 0.18003 | | jinaai/jina-embeddings-v3 | 0.76277 | 0.10240 | 0.65290 | 0.17843 | | intfloat/multilingual-e5-large-instruct | 0.74851 | 0.09888 | 0.64451 | 0.17283 | | Alibaba-NLP/gte-multilingual-base | 0.75631 | 0.09938 | 0.64025 | 0.17363 | | Alibaba-NLP/gte-Qwen2-7B-instruct | 0.74092 | 0.09607 | 0.63258 | 0.16847 | | intfloat/multilingual-e5-base | 0.73512 | 0.09717 | 0.63216 | 0.16977 | | intfloat/e5-mistral-7b-instruct | 0.73795 | 0.09777 | 0.63076 | 0.17078 | | openai/text-embedding-3-large | 0.72946 | 0.09571 | 0.61670 | 0.16739 | | Salesforce/SFR-Embedding-2_R | 0.71662 | 0.09546 | 0.60589 | 0.16651 | | upskyy/bge-m3-korean | 0.71895 | 0.09583 | 0.60258 | 0.16712 | | jhgan/ko-sroberta-multitask | 0.61225 | 0.07826 | 0.48687 | 0.13757 |
## Training Details - KURE-v1 is a model fine-tuned from [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3). - KoE5 is a model fine-tuned from [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large). ### Training Data **KURE-v1** - Korean query-document-hard_negative(5) pairs - Approx. 2,000,000 examples **KoE5** - [ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0) - Korean query-document-hard_negative(1) pairs (open data) - Approx. 700,000+ examples ### Training Procedure **KURE-v1** - loss: [CachedGISTEmbedLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedgistembedloss) - batch size: 4096 - learning rate: 2e-05 - epochs: 1 **KoE5** - loss: [CachedMultipleNegativesRankingLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedmultiplenegativesrankingloss) - batch size: 512 - learning rate: 1e-05 - epochs: 1
## Important Notes - When using KoE5, you must add a prefix for each input: (e.g., `query: {query}`, `passage: {document}`) ## License - ```MIT``` ## Citation If you find our paper or models helpful, please consider citing them as follows: ```text @misc{KURE, publisher = {Youngjoon Jang, Junyoung Son, Taemin Lee}, year = {2024}, url = {https://github.com/nlpai-lab/KURE} }, @misc{KoE5, author = {NLP & AI Lab and Human-Inspired AI research}, title = {KoE5: A New Dataset and Model for Improving Korean Embedding Performance}, year = {2024}, publisher = {Youngjoon Jang, Junyoung Son, Taemin Lee}, journal = {GitHub repository}, howpublished = {\url{https://drive.google.com/file/d/1wB02XGFH5v18iJYSYB0oJkWFYxH0ftoJ/view}}, } ``` ``` ================================================ FILE: eval/evaluate.py ================================================ """Benchmarking all datasets constituting the MTEB Korean leaderboard & average scores""" from __future__ import annotations import os import logging from multiprocessing import Process, current_process import torch from sentence_transformers import SentenceTransformer from sentence_transformers.models import StaticEmbedding import mteb from mteb import MTEB, get_tasks from mteb.encoder_interface import PromptType from mteb.models.sentence_transformer_wrapper import SentenceTransformerWrapper from mteb.models.instruct_wrapper import instruct_wrapper import argparse from dotenv import load_dotenv from setproctitle import setproctitle import traceback import logging load_dotenv() # for OPENAI parser = argparse.ArgumentParser(description="Extract contexts") parser.add_argument('--quantize', default=False, type=bool, help='quantize embeddings') args = parser.parse_args() logging.basicConfig(level=logging.INFO) logger = logging.getLogger("main") TASK_LIST_CLASSIFICATION = [] TASK_LIST_CLUSTERING = [] TASK_LIST_PAIR_CLASSIFICATION = [] TASK_LIST_RERANKING = [] TASK_LIST_RETRIEVAL = [ "Ko-StrategyQA", "AutoRAGRetrieval", "MIRACLRetrieval", # 시간이 오래 걸림 주의 "PublicHealthQA", "BelebeleRetrieval", "MrTidyRetrieval", # 시간이 오래 걸림 주의 "MultiLongDocRetrieval", "XPQARetrieval", "Tatoeba" ] TASK_LIST_STS = [] TASK_LIST = ( TASK_LIST_CLASSIFICATION + TASK_LIST_CLUSTERING + TASK_LIST_PAIR_CLASSIFICATION + TASK_LIST_RERANKING + TASK_LIST_RETRIEVAL + TASK_LIST_STS ) # MIRACL, MrTidy는 평가 시 시간이 오래 걸리기 때문에, 태스크별로 나누어 multiprocessing으로 평가합니다. # 필요 시 GPU 번호를 다르게 조정해 주세요. TASK_LIST_RETRIEVAL_GPU_MAPPING = { 0: [ "Ko-StrategyQA", "AutoRAGRetrieval", "PublicHealthQA", "BelebeleRetrieval", "XPQARetrieval", "MultiLongDocRetrieval", ], 1: ["MIRACLRetrieval"], 2: ["MrTidyRetrieval"], } model_names = [ # my_model_directory ] model_names = [ # "Salesforce/SFR-Embedding-2_R", # 4096 # "Alibaba-NLP/gte-Qwen2-7B-instruct", # 8192 # "BAAI/bge-multilingual-gemma2", # 8192 # "intfloat/e5-mistral-7b-instruct", # 32768 # "intfloat/multilingual-e5-large-instruct", # 512 # "openai/text-embedding-3-large", # 8191 # "Alibaba-NLP/gte-multilingual-base", # "intfloat/multilingual-e5-base", # 512 # "intfloat/multilingual-e5-large", # 512 # "jinaai/jina-embeddings-v3", # 8192 # "jhgan/ko-sroberta-multitask", # 128 # "BAAI/bge-m3", # 8192 # "nlpai-lab/KoE5", # 512 # "dragonkue/BGE-m3-ko", # 8192 # "Snowflake/snowflake-arctic-embed-l-v2.0", # 8192, # "nlpai-lab/KURE-v1", # 8192, "nomic-ai/nomic-embed-text-v2-moe" ] + model_names def evaluate_model(model_name, gpu_id, tasks): try: # Set the environment variable for the specific GPU os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id) model = None if not os.path.exists(model_name): # hf에 등록된 모델의 경우 if "m2v" in model_name: # model2vec의 경우: 모델명에 m2v를 포함시켜주어야 model2vec 모델로 인식합니다. static_embedding = StaticEmbedding.from_model2vec(model_name) model = SentenceTransformer(modules=[static_embedding], model_kwargs={"attn_implementation": "sdpa"}) else: if model_name == "nlpai-lab/KoE5" or model_name == "KU-HIAI-ONTHEIT/ontheit-large-v1_1" : # mE5 기반의 모델이므로, 해당 프롬프트를 추가시킵니다. model_prompts = { PromptType.query.value: "query: ", PromptType.passage.value: "passage: ", } model = SentenceTransformerWrapper(model=model_name, model_prompts=model_prompts, model_kwargs={"attn_implementation": "sdpa"}) elif model_name == "nomic-ai/nomic-embed-text-v2-moe": model_prompts = { PromptType.query.value: "search_query: ", PromptType.passage.value: "search_document: ", } model = SentenceTransformerWrapper(model=model_name, model_prompts=model_prompts, model_kwargs={"attn_implementation": "sdpa"}, trust_remote_code=True) elif model_name == "BAAI/bge-multilingual-gemma2": # mbge-gemma2의 경우, mteb에서 지원하지 않습니다. 따라서, instruct_wrapper를 사용합니다. instruction_template = '{instruction}\n' model = instruct_wrapper( model_name_or_path=model_name, instruction_template=instruction_template, attn="cccc", pooling_method="lasttoken", mode="embedding", torch_dtype=torch.float16, normalized=True, ) elif model_name == "Snowflake/snowflake-arctic-embed-l-v2.0": # mteb에서 Snowflake 모델을 지원하지 않으므로, Snowflake에서 사용하는 "query: " prefix를 임의로 추가합니다. model_prompts = { PromptType.query.value: "query: ", } model = SentenceTransformerWrapper(model=model_name, model_prompts=model_prompts, model_kwargs={"attn_implementation": "sdpa"}) else: # mteb에 등록된 모델의 경우, 프롬프트/prefix 등을 포함하여 평가할 수 있습니다. 등록되지 않은 경우, sentence-transformer를 사용하여 불러옵니다. model = mteb.get_model(model_name) else: # 직접 학습한 모델의 경우 file_name = os.path.join(model_name, "model.safetensors") if os.path.exists(file_name): if "m2v" in model_name: # model2vec의 경우: 모델명에 m2v를 포함시켜주어야 model2vec 모델로 인식합니다. static_embedding = StaticEmbedding.from_model2vec(model_name) model = SentenceTransformer(modules=[static_embedding], model_kwargs={"attn_implementation": "sdpa"}) else: model = mteb.get_model(model_name, model_kwargs={"attn_implementation": "sdpa"}) if model: setproctitle(f"{model_name}-{gpu_id}") print(f"Running tasks: {tasks} / {model_name} on GPU {gpu_id} in process {current_process().name}") evaluation = MTEB( tasks=get_tasks(tasks=tasks, languages=["kor-Kore", "kor-Hang", "kor_Hang"]) ) # 48GB VRAM 기준 적합한 batch sizes if "multilingual-e5" in model_name or "KoE5" in model_name or "ontheit" in model_name or "nomic" in model_name: batch_size = 512 elif "jina" in model_name: batch_size = 8 elif "bge-m3" in model_name or "Snowflake" in model_name: batch_size = 32 elif "gemma2" in model_name: batch_size = 256 elif "Salesforce" in model_name: batch_size = 128 else: batch_size = 64 if args.quantize: # quantized model의 경우 evaluation.run( model, output_folder=f"results/{model_name}-quantized", encode_kwargs={"batch_size": batch_size, "precision": "binary"}, ) else: evaluation.run( model, output_folder=f"results/{model_name}", encode_kwargs={"batch_size": batch_size}, ) except Exception as ex: print(ex) traceback.print_exc() if __name__ == "__main__": processes = [] for gpu_id, tasks in TASK_LIST_RETRIEVAL_GPU_MAPPING.items(): for model_name in model_names: p = Process(target=evaluate_model, args=(model_name, gpu_id, tasks)) p.start() processes.append(p) for p in processes: p.join() ================================================ FILE: eval/leaderboard.py ================================================ import streamlit as st import os import json import pandas as pd st.set_page_config(layout="wide") def app(): data = {} avg_data = {} # average score를 저장하기 위한 dictionary tasks = [ "Ko-StrategyQA", "AutoRAGRetrieval", "MIRACLRetrieval", "PublicHealthQA", "BelebeleRetrieval", "MrTidyRetrieval", "MultiLongDocRetrieval", "XPQARetrieval" ] top_k_types = ["top1", "top3", "top5", "top10"] score_types = { "top1": ["recall_at_1", "precision_at_1", "ndcg_at_1"], "top3": ["recall_at_3", "precision_at_3", "ndcg_at_3"], "top5": ["recall_at_5", "precision_at_5", "ndcg_at_5"], "top10": ["recall_at_10", "precision_at_10", "ndcg_at_10"], } # 각 작업에 대한 데이터를 초기화 for task in tasks: data[task] = {top_k: [] for top_k in top_k_types} root_dir = "results" # 데이터가 저장되어 있는 디렉토리의 모든 하위 폴더를 순회하면서 json 파일을 읽습니다. for subdir, dirs, files in os.walk(root_dir): for file in files: for task in tasks: if file == task + ".json": with open(os.path.join(subdir, file)) as f: d = json.load(f) for top_k in top_k_types: results = {} for score in score_types[top_k]: if "dev" in d["scores"] and "test" not in d["scores"]: results[score] = d["scores"]["dev"][0][score] elif "test" in d["scores"] and "dev" not in d["scores"]: results[score] = d["scores"]["test"][0][score] else: # dev, test를 모두 가지고 있는 평가 데이터셋을 위함 results[score] = (d["scores"]["dev"][0][score] + d["scores"]["test"][0][score]) / 2 # f1 score 직접 계산 f1_score = ( 2 * (results[score_types[top_k][1]] * results[score_types[top_k][0]]) / (results[score_types[top_k][1]]+ results[score_types[top_k][0]]) if (results[score_types[top_k][1]]+ results[score_types[top_k][0]])> 0 else 0 ) data[task][top_k].append( ( os.path.relpath(subdir, root_dir), results[score_types[top_k][0]], results[score_types[top_k][1]], results[score_types[top_k][2]], f1_score, ) ) # 각 작업에 대해 top1, top3, top5, top10 점수 표시 for task in tasks: st.markdown(f"# {task}") for top_k in top_k_types: st.markdown(f"## {top_k.capitalize()} Scores") df = pd.DataFrame( data[task][top_k], columns=[ "Subdir", f"Recall_{top_k}", f"Precision_{top_k}", f"NDCG_{top_k}", f"F1_{top_k}", ], ) df = df.sort_values(by=f"NDCG_{top_k}", ascending=False) st.dataframe(df, use_container_width=True) # 각 모델의 평균 점수 계산 for subdir, recall, precision, ndcg, f1 in data[task][top_k]: if subdir not in avg_data: avg_data[subdir] = { k: [[], [], [], []] for k in top_k_types } avg_data[subdir][top_k][0].append(recall) avg_data[subdir][top_k][1].append(precision) avg_data[subdir][top_k][2].append(ndcg) avg_data[subdir][top_k][3].append(f1) # 각 모델 별 평균 점수 계산 후 출력 st.markdown("# Average Scores") for top_k in top_k_types: avg_results = [] for model in avg_data: recall_avg = ( sum(avg_data[model][top_k][0]) / len(avg_data[model][top_k][0]) if avg_data[model][top_k][0] else 0 ) precision_avg = ( sum(avg_data[model][top_k][1]) / len(avg_data[model][top_k][1]) if avg_data[model][top_k][1] else 0 ) ndcg_avg = ( sum(avg_data[model][top_k][2]) / len(avg_data[model][top_k][2]) if avg_data[model][top_k][2] else 0 ) f1_avg = ( sum(avg_data[model][top_k][3]) / len(avg_data[model][top_k][3]) if avg_data[model][top_k][3] else 0 ) avg_results.append([model, recall_avg, precision_avg, ndcg_avg, f1_avg]) avg_df = pd.DataFrame( avg_results, columns=[ "Model", f"Average Recall_{top_k}", f"Average Precision_{top_k}", f"Average NDCG_{top_k}", f"Average F1_{top_k}", ], ) avg_df = avg_df.sort_values(by=f"Average NDCG_{top_k}", ascending=False) st.markdown(f"## {top_k.capitalize()} Average Scores") st.dataframe(avg_df, use_container_width=True) if __name__ == "__main__": app() ================================================ FILE: eval/requirements.txt ================================================ mteb python-dotenv streamlit setproctitle ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 846.0815389156342, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.76682, "map_at_1": 0.60526, "map_at_10": 0.71514, "map_at_100": 0.71846, "map_at_1000": 0.71854, "map_at_20": 0.71719, "map_at_3": 0.69444, "map_at_5": 0.7019, "mrr_at_1": 0.6052631578947368, "mrr_at_10": 0.7151350598719021, "mrr_at_100": 0.7184641134066749, "mrr_at_1000": 0.7185366086944813, "mrr_at_20": 0.7171920671920672, "mrr_at_3": 0.6944444444444446, "mrr_at_5": 0.7019005847953218, "nauc_map_at_1000_diff1": 0.7644331655442053, "nauc_map_at_1000_max": 0.006603581947104919, "nauc_map_at_1000_std": -0.37084270701564565, "nauc_map_at_100_diff1": 0.7644201027610169, "nauc_map_at_100_max": 0.0067126570840267364, "nauc_map_at_100_std": -0.37090267941427896, "nauc_map_at_10_diff1": 0.762543492536473, "nauc_map_at_10_max": 0.016618337725207714, "nauc_map_at_10_std": -0.36230366527185476, "nauc_map_at_1_diff1": 0.8065079498138608, "nauc_map_at_1_max": -0.008481292674104184, "nauc_map_at_1_std": -0.3152574052168019, "nauc_map_at_20_diff1": 0.7647616354525286, "nauc_map_at_20_max": 0.010913309413363433, "nauc_map_at_20_std": -0.3693593056765434, "nauc_map_at_3_diff1": 0.749148318778548, "nauc_map_at_3_max": -0.013924305023451019, "nauc_map_at_3_std": -0.39203803015117983, "nauc_map_at_5_diff1": 0.7599115394000969, "nauc_map_at_5_max": -0.008945775041782456, "nauc_map_at_5_std": -0.38416486703433883, "nauc_mrr_at_1000_diff1": 0.7644331655442053, "nauc_mrr_at_1000_max": 0.006603581947104919, "nauc_mrr_at_1000_std": -0.37084270701564565, "nauc_mrr_at_100_diff1": 0.7644201027610169, "nauc_mrr_at_100_max": 0.0067126570840267364, "nauc_mrr_at_100_std": -0.37090267941427896, "nauc_mrr_at_10_diff1": 0.762543492536473, "nauc_mrr_at_10_max": 0.016618337725207714, "nauc_mrr_at_10_std": -0.36230366527185476, "nauc_mrr_at_1_diff1": 0.8065079498138608, "nauc_mrr_at_1_max": -0.008481292674104184, "nauc_mrr_at_1_std": -0.3152574052168019, "nauc_mrr_at_20_diff1": 0.7647616354525286, "nauc_mrr_at_20_max": 0.010913309413363433, "nauc_mrr_at_20_std": -0.3693593056765434, "nauc_mrr_at_3_diff1": 0.749148318778548, "nauc_mrr_at_3_max": -0.013924305023451019, "nauc_mrr_at_3_std": -0.39203803015117983, "nauc_mrr_at_5_diff1": 0.7599115394000969, "nauc_mrr_at_5_max": -0.008945775041782456, "nauc_mrr_at_5_std": -0.38416486703433883, "nauc_ndcg_at_1000_diff1": 0.7622350239005066, "nauc_ndcg_at_1000_max": 0.017371318088600977, "nauc_ndcg_at_1000_std": -0.37321055393166663, "nauc_ndcg_at_100_diff1": 0.7619593636566969, "nauc_ndcg_at_100_max": 0.01972561053123169, "nauc_ndcg_at_100_std": -0.37452959411278264, "nauc_ndcg_at_10_diff1": 0.7575322927973843, "nauc_ndcg_at_10_max": 0.07021633482084594, "nauc_ndcg_at_10_std": -0.3375267305826439, "nauc_ndcg_at_1_diff1": 0.8065079498138608, "nauc_ndcg_at_1_max": -0.008481292674104184, "nauc_ndcg_at_1_std": -0.3152574052168019, "nauc_ndcg_at_20_diff1": 0.7652290444346898, "nauc_ndcg_at_20_max": 0.048815872410838686, "nauc_ndcg_at_20_std": -0.36658508429606196, "nauc_ndcg_at_3_diff1": 0.7321065045076984, "nauc_ndcg_at_3_max": -0.01155970025952776, "nauc_ndcg_at_3_std": -0.4161891854409051, "nauc_ndcg_at_5_diff1": 0.7516239012254612, "nauc_ndcg_at_5_max": -6.352015603966895e-05, "nauc_ndcg_at_5_std": -0.39852520659387436, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 0.7224066624409552, "nauc_precision_at_100_max": 0.35752776970742256, "nauc_precision_at_100_std": -0.5637901180614636, "nauc_precision_at_10_diff1": 0.7309271500210811, "nauc_precision_at_10_max": 0.6817757315332228, "nauc_precision_at_10_std": 0.00574347193803287, "nauc_precision_at_1_diff1": 0.8065079498138608, "nauc_precision_at_1_max": -0.008481292674104184, "nauc_precision_at_1_std": -0.3152574052168019, "nauc_precision_at_20_diff1": 0.8293909916468436, "nauc_precision_at_20_max": 0.7008965455883202, "nauc_precision_at_20_std": -0.2706904285228724, "nauc_precision_at_3_diff1": 0.6681562238166904, "nauc_precision_at_3_max": -0.00047679284331604674, "nauc_precision_at_3_std": -0.5075608401380758, "nauc_precision_at_5_diff1": 0.7204265059450241, "nauc_precision_at_5_max": 0.046535938023941965, "nauc_precision_at_5_std": -0.4530486350784987, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.7224066624409369, "nauc_recall_at_100_max": 0.35752776970737554, "nauc_recall_at_100_std": -0.5637901180615443, "nauc_recall_at_10_diff1": 0.7309271500210796, "nauc_recall_at_10_max": 0.6817757315332232, "nauc_recall_at_10_std": 0.005743471938036119, "nauc_recall_at_1_diff1": 0.8065079498138608, "nauc_recall_at_1_max": -0.008481292674104184, "nauc_recall_at_1_std": -0.3152574052168019, "nauc_recall_at_20_diff1": 0.8293909916468486, "nauc_recall_at_20_max": 0.700896545588321, "nauc_recall_at_20_std": -0.27069042852286784, "nauc_recall_at_3_diff1": 0.6681562238166918, "nauc_recall_at_3_max": -0.0004767928433143103, "nauc_recall_at_3_std": -0.5075608401380753, "nauc_recall_at_5_diff1": 0.7204265059450234, "nauc_recall_at_5_max": 0.04653593802394211, "nauc_recall_at_5_std": -0.45304863507849813, "ndcg_at_1": 0.60526, "ndcg_at_10": 0.76682, "ndcg_at_100": 0.78099, "ndcg_at_1000": 0.78225, "ndcg_at_20": 0.77376, "ndcg_at_3": 0.72128, "ndcg_at_5": 0.73524, "precision_at_1": 0.60526, "precision_at_10": 0.09298, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "precision_at_20": 0.04781, "precision_at_3": 0.26608, "precision_at_5": 0.16667, "recall_at_1": 0.60526, "recall_at_10": 0.92982, "recall_at_100": 0.99123, "recall_at_1000": 1.0, "recall_at_20": 0.95614, "recall_at_3": 0.79825, "recall_at_5": 0.83333 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 537.1708543300629, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.94808, "map_at_1": 0.91444, "map_at_10": 0.93774, "map_at_100": 0.93854, "map_at_1000": 0.93856, "map_at_20": 0.93803, "map_at_3": 0.93426, "map_at_5": 0.93648, "mrr_at_1": 0.9144444444444444, "mrr_at_10": 0.9377385361552029, "mrr_at_100": 0.9385426232247298, "mrr_at_1000": 0.9385570215532866, "mrr_at_20": 0.9380331489498157, "mrr_at_3": 0.9342592592592593, "mrr_at_5": 0.9364814814814815, "nauc_map_at_1000_diff1": 0.9341587574819448, "nauc_map_at_1000_max": 0.7978508469331898, "nauc_map_at_1000_std": -0.1247788227834717, "nauc_map_at_100_diff1": 0.9341510130917275, "nauc_map_at_100_max": 0.7978982067470314, "nauc_map_at_100_std": -0.12453847765211529, "nauc_map_at_10_diff1": 0.9344160375639369, "nauc_map_at_10_max": 0.7999522285999181, "nauc_map_at_10_std": -0.12061906912864259, "nauc_map_at_1_diff1": 0.9313119187068765, "nauc_map_at_1_max": 0.7544472334388295, "nauc_map_at_1_std": -0.19743655037772562, "nauc_map_at_20_diff1": 0.934104227715159, "nauc_map_at_20_max": 0.7991964531277118, "nauc_map_at_20_std": -0.12080639055631145, "nauc_map_at_3_diff1": 0.9355558185715604, "nauc_map_at_3_max": 0.8014531634249938, "nauc_map_at_3_std": -0.14484948909140807, "nauc_map_at_5_diff1": 0.9333012116411193, "nauc_map_at_5_max": 0.7994864612511658, "nauc_map_at_5_std": -0.1304815803872555, "nauc_mrr_at_1000_diff1": 0.9341587574819448, "nauc_mrr_at_1000_max": 0.7978508469331898, "nauc_mrr_at_1000_std": -0.1247788227834717, "nauc_mrr_at_100_diff1": 0.9341510130917275, "nauc_mrr_at_100_max": 0.7978982067470314, "nauc_mrr_at_100_std": -0.12453847765211529, "nauc_mrr_at_10_diff1": 0.9344160375639369, "nauc_mrr_at_10_max": 0.7999522285999181, "nauc_mrr_at_10_std": -0.12061906912864259, "nauc_mrr_at_1_diff1": 0.9313119187068765, "nauc_mrr_at_1_max": 0.7544472334388295, "nauc_mrr_at_1_std": -0.19743655037772562, "nauc_mrr_at_20_diff1": 0.934104227715159, "nauc_mrr_at_20_max": 0.7991964531277118, "nauc_mrr_at_20_std": -0.12080639055631145, "nauc_mrr_at_3_diff1": 0.9355558185715604, "nauc_mrr_at_3_max": 0.8014531634249938, "nauc_mrr_at_3_std": -0.14484948909140807, "nauc_mrr_at_5_diff1": 0.9333012116411193, "nauc_mrr_at_5_max": 0.7994864612511658, "nauc_mrr_at_5_std": -0.1304815803872555, "nauc_ndcg_at_1000_diff1": 0.9344502542270329, "nauc_ndcg_at_1000_max": 0.8043530829653767, "nauc_ndcg_at_1000_std": -0.10775125619222786, "nauc_ndcg_at_100_diff1": 0.9344072352372144, "nauc_ndcg_at_100_max": 0.8055734617052932, "nauc_ndcg_at_100_std": -0.10129337761974706, "nauc_ndcg_at_10_diff1": 0.9356156657712474, "nauc_ndcg_at_10_max": 0.8171392419913772, "nauc_ndcg_at_10_std": -0.07482350219227087, "nauc_ndcg_at_1_diff1": 0.9313119187068765, "nauc_ndcg_at_1_max": 0.7544472334388295, "nauc_ndcg_at_1_std": -0.19743655037772562, "nauc_ndcg_at_20_diff1": 0.9344540935531565, "nauc_ndcg_at_20_max": 0.8146097937010235, "nauc_ndcg_at_20_std": -0.07418267112495723, "nauc_ndcg_at_3_diff1": 0.936819790095856, "nauc_ndcg_at_3_max": 0.8178989129857199, "nauc_ndcg_at_3_std": -0.13242670021197311, "nauc_ndcg_at_5_diff1": 0.9321587219670335, "nauc_ndcg_at_5_max": 0.8148177822979191, "nauc_ndcg_at_5_std": -0.10319206027442693, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9346405228757582, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.9346405228757582, "nauc_precision_at_10_diff1": 0.9482570806100177, "nauc_precision_at_10_max": 0.9679686689490625, "nauc_precision_at_10_std": 0.36248573503474296, "nauc_precision_at_1_diff1": 0.9313119187068765, "nauc_precision_at_1_max": 0.7544472334388295, "nauc_precision_at_1_std": -0.19743655037772562, "nauc_precision_at_20_diff1": 0.9379084967320215, "nauc_precision_at_20_max": 0.9702769996887645, "nauc_precision_at_20_std": 0.4604419545596087, "nauc_precision_at_3_diff1": 0.9424992219109852, "nauc_precision_at_3_max": 0.8935444548189662, "nauc_precision_at_3_std": -0.07892416225750032, "nauc_precision_at_5_diff1": 0.923332295881318, "nauc_precision_at_5_max": 0.9042258878860141, "nauc_precision_at_5_std": 0.06276584707956435, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.934640522875857, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.934640522875857, "nauc_recall_at_10_diff1": 0.9482570806100225, "nauc_recall_at_10_max": 0.9679686689490574, "nauc_recall_at_10_std": 0.3624857350347621, "nauc_recall_at_1_diff1": 0.9313119187068765, "nauc_recall_at_1_max": 0.7544472334388295, "nauc_recall_at_1_std": -0.19743655037772562, "nauc_recall_at_20_diff1": 0.9379084967320268, "nauc_recall_at_20_max": 0.97027699968877, "nauc_recall_at_20_std": 0.4604419545596044, "nauc_recall_at_3_diff1": 0.9424992219109841, "nauc_recall_at_3_max": 0.893544454818967, "nauc_recall_at_3_std": -0.07892416225749478, "nauc_recall_at_5_diff1": 0.9233322958813207, "nauc_recall_at_5_max": 0.9042258878860211, "nauc_recall_at_5_std": 0.06276584707957272, "ndcg_at_1": 0.91444, "ndcg_at_10": 0.94808, "ndcg_at_100": 0.95193, "ndcg_at_1000": 0.95223, "ndcg_at_20": 0.949, "ndcg_at_3": 0.94086, "ndcg_at_5": 0.94492, "precision_at_1": 0.91444, "precision_at_10": 0.098, "precision_at_100": 0.00998, "precision_at_1000": 0.001, "precision_at_20": 0.04917, "precision_at_3": 0.32, "precision_at_5": 0.194, "recall_at_1": 0.91444, "recall_at_10": 0.98, "recall_at_100": 0.99778, "recall_at_1000": 1.0, "recall_at_20": 0.98333, "recall_at_3": 0.96, "recall_at_5": 0.97 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.95397, "map_at_1": 0.91778, "map_at_10": 0.94328, "map_at_100": 0.94402, "map_at_1000": 0.94402, "map_at_20": 0.9439, "map_at_3": 0.9387, "map_at_5": 0.94204, "mrr_at_1": 0.9177777777777778, "mrr_at_10": 0.9432760141093474, "mrr_at_100": 0.9440192855380678, "mrr_at_1000": 0.944022209514676, "mrr_at_20": 0.9438995324516478, "mrr_at_3": 0.9387037037037037, "mrr_at_5": 0.9420370370370371, "nauc_map_at_1000_diff1": 0.9379736356471905, "nauc_map_at_1000_max": 0.818228835537799, "nauc_map_at_1000_std": -0.3568158538211896, "nauc_map_at_100_diff1": 0.9379768753993101, "nauc_map_at_100_max": 0.8182383297830593, "nauc_map_at_100_std": -0.35674498481109457, "nauc_map_at_10_diff1": 0.9371832681770706, "nauc_map_at_10_max": 0.8194699100005315, "nauc_map_at_10_std": -0.3534046804367552, "nauc_map_at_1_diff1": 0.9365016276781002, "nauc_map_at_1_max": 0.7871400812577283, "nauc_map_at_1_std": -0.3898352133646259, "nauc_map_at_20_diff1": 0.9380057116884054, "nauc_map_at_20_max": 0.8186263215762201, "nauc_map_at_20_std": -0.3552421679266783, "nauc_map_at_3_diff1": 0.9384134318379933, "nauc_map_at_3_max": 0.8234983822330538, "nauc_map_at_3_std": -0.36390165330986185, "nauc_map_at_5_diff1": 0.9376907312445761, "nauc_map_at_5_max": 0.8229708283739466, "nauc_map_at_5_std": -0.3448556632450642, "nauc_mrr_at_1000_diff1": 0.9379736356471905, "nauc_mrr_at_1000_max": 0.818228835537799, "nauc_mrr_at_1000_std": -0.3568158538211896, "nauc_mrr_at_100_diff1": 0.9379768753993101, "nauc_mrr_at_100_max": 0.8182383297830593, "nauc_mrr_at_100_std": -0.35674498481109457, "nauc_mrr_at_10_diff1": 0.9371832681770706, "nauc_mrr_at_10_max": 0.8194699100005315, "nauc_mrr_at_10_std": -0.3534046804367552, "nauc_mrr_at_1_diff1": 0.9365016276781002, "nauc_mrr_at_1_max": 0.7871400812577283, "nauc_mrr_at_1_std": -0.3898352133646259, "nauc_mrr_at_20_diff1": 0.9380057116884054, "nauc_mrr_at_20_max": 0.8186263215762201, "nauc_mrr_at_20_std": -0.3552421679266783, "nauc_mrr_at_3_diff1": 0.9384134318379933, "nauc_mrr_at_3_max": 0.8234983822330538, "nauc_mrr_at_3_std": -0.36390165330986185, "nauc_mrr_at_5_diff1": 0.9376907312445761, "nauc_mrr_at_5_max": 0.8229708283739466, "nauc_mrr_at_5_std": -0.3448556632450642, "nauc_ndcg_at_1000_diff1": 0.9382191754421236, "nauc_ndcg_at_1000_max": 0.8224201913320449, "nauc_ndcg_at_1000_std": -0.34630458284754967, "nauc_ndcg_at_100_diff1": 0.9384049804187261, "nauc_ndcg_at_100_max": 0.8229542601540525, "nauc_ndcg_at_100_std": -0.3422555904085183, "nauc_ndcg_at_10_diff1": 0.9357447204568637, "nauc_ndcg_at_10_max": 0.829222262177988, "nauc_ndcg_at_10_std": -0.3281301084025004, "nauc_ndcg_at_1_diff1": 0.9365016276781002, "nauc_ndcg_at_1_max": 0.7871400812577283, "nauc_ndcg_at_1_std": -0.3898352133646259, "nauc_ndcg_at_20_diff1": 0.9386644241730514, "nauc_ndcg_at_20_max": 0.8257288395033384, "nauc_ndcg_at_20_std": -0.3312173812179968, "nauc_ndcg_at_3_diff1": 0.9388035548318442, "nauc_ndcg_at_3_max": 0.8379465154015449, "nauc_ndcg_at_3_std": -0.3444570338354112, "nauc_ndcg_at_5_diff1": 0.9374737365012075, "nauc_ndcg_at_5_max": 0.8384461172346422, "nauc_ndcg_at_5_std": -0.30419406475924055, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.9138266417678141, "nauc_precision_at_10_max": 0.93191721132897, "nauc_precision_at_10_std": -0.0321350762527383, "nauc_precision_at_1_diff1": 0.9365016276781002, "nauc_precision_at_1_max": 0.7871400812577283, "nauc_precision_at_1_std": -0.3898352133646259, "nauc_precision_at_20_diff1": 0.967320261437914, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 0.5262605042016776, "nauc_precision_at_3_diff1": 0.9404408227937671, "nauc_precision_at_3_max": 0.9062756415697625, "nauc_precision_at_3_std": -0.24736157089098976, "nauc_precision_at_5_diff1": 0.9352240896358476, "nauc_precision_at_5_max": 0.9526143790849654, "nauc_precision_at_5_std": 0.01713352007468603, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 1.0, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 1.0, "nauc_recall_at_10_diff1": 0.9138266417678085, "nauc_recall_at_10_max": 0.9319172113289713, "nauc_recall_at_10_std": -0.03213507625271808, "nauc_recall_at_1_diff1": 0.9365016276781002, "nauc_recall_at_1_max": 0.7871400812577283, "nauc_recall_at_1_std": -0.3898352133646259, "nauc_recall_at_20_diff1": 0.9673202614379001, "nauc_recall_at_20_max": 1.0, "nauc_recall_at_20_std": 0.5262605042016953, "nauc_recall_at_3_diff1": 0.9404408227937675, "nauc_recall_at_3_max": 0.9062756415697596, "nauc_recall_at_3_std": -0.24736157089098623, "nauc_recall_at_5_diff1": 0.9352240896358568, "nauc_recall_at_5_max": 0.9526143790849627, "nauc_recall_at_5_std": 0.017133520074696214, "ndcg_at_1": 0.91778, "ndcg_at_10": 0.95397, "ndcg_at_100": 0.95691, "ndcg_at_1000": 0.95704, "ndcg_at_20": 0.95622, "ndcg_at_3": 0.94507, "ndcg_at_5": 0.95104, "precision_at_1": 0.91778, "precision_at_10": 0.09867, "precision_at_100": 0.00999, "precision_at_1000": 0.001, "precision_at_20": 0.04978, "precision_at_3": 0.32111, "precision_at_5": 0.19556, "recall_at_1": 0.91778, "recall_at_10": 0.98667, "recall_at_100": 0.99889, "recall_at_1000": 1.0, "recall_at_20": 0.99556, "recall_at_3": 0.96333, "recall_at_5": 0.97778 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.92939, "map_at_1": 0.87778, "map_at_10": 0.91394, "map_at_100": 0.91475, "map_at_1000": 0.91478, "map_at_20": 0.91456, "map_at_3": 0.90759, "map_at_5": 0.9127, "mrr_at_1": 0.8777777777777778, "mrr_at_10": 0.9139395943562613, "mrr_at_100": 0.9147527793478226, "mrr_at_1000": 0.9147804033940741, "mrr_at_20": 0.9145571747606064, "mrr_at_3": 0.9075925925925927, "mrr_at_5": 0.912703703703704, "nauc_map_at_1000_diff1": 0.9034621090151944, "nauc_map_at_1000_max": 0.8166056541545396, "nauc_map_at_1000_std": -0.04958063210793317, "nauc_map_at_100_diff1": 0.9034260698542368, "nauc_map_at_100_max": 0.8166661124959635, "nauc_map_at_100_std": -0.04932797829928419, "nauc_map_at_10_diff1": 0.9032733282636335, "nauc_map_at_10_max": 0.8176267163973103, "nauc_map_at_10_std": -0.046047883529355904, "nauc_map_at_1_diff1": 0.9129032258064524, "nauc_map_at_1_max": 0.7948743587514139, "nauc_map_at_1_std": -0.0959960003477958, "nauc_map_at_20_diff1": 0.9033278279584531, "nauc_map_at_20_max": 0.8168441424897516, "nauc_map_at_20_std": -0.04894110672076424, "nauc_map_at_3_diff1": 0.8994646682977531, "nauc_map_at_3_max": 0.8155491427278718, "nauc_map_at_3_std": -0.05738561942402418, "nauc_map_at_5_diff1": 0.9023811940876922, "nauc_map_at_5_max": 0.8159936946768515, "nauc_map_at_5_std": -0.04821727179963084, "nauc_mrr_at_1000_diff1": 0.9034621090151944, "nauc_mrr_at_1000_max": 0.8166056541545396, "nauc_mrr_at_1000_std": -0.04958063210793317, "nauc_mrr_at_100_diff1": 0.9034260698542368, "nauc_mrr_at_100_max": 0.8166661124959635, "nauc_mrr_at_100_std": -0.04932797829928419, "nauc_mrr_at_10_diff1": 0.9032733282636335, "nauc_mrr_at_10_max": 0.8176267163973103, "nauc_mrr_at_10_std": -0.046047883529355904, "nauc_mrr_at_1_diff1": 0.9129032258064524, "nauc_mrr_at_1_max": 0.7948743587514139, "nauc_mrr_at_1_std": -0.0959960003477958, "nauc_mrr_at_20_diff1": 0.9033278279584531, "nauc_mrr_at_20_max": 0.8168441424897516, "nauc_mrr_at_20_std": -0.04894110672076424, "nauc_mrr_at_3_diff1": 0.8994646682977531, "nauc_mrr_at_3_max": 0.8155491427278718, "nauc_mrr_at_3_std": -0.05738561942402418, "nauc_mrr_at_5_diff1": 0.9023811940876922, "nauc_mrr_at_5_max": 0.8159936946768515, "nauc_mrr_at_5_std": -0.04821727179963084, "nauc_ndcg_at_1000_diff1": 0.9025162201523518, "nauc_ndcg_at_1000_max": 0.8210647790377918, "nauc_ndcg_at_1000_std": -0.033259752555848905, "nauc_ndcg_at_100_diff1": 0.9019832601650294, "nauc_ndcg_at_100_max": 0.822705958126526, "nauc_ndcg_at_100_std": -0.025706848528490346, "nauc_ndcg_at_10_diff1": 0.901289557068697, "nauc_ndcg_at_10_max": 0.8284852301803537, "nauc_ndcg_at_10_std": -0.006171497791011456, "nauc_ndcg_at_1_diff1": 0.9129032258064524, "nauc_ndcg_at_1_max": 0.7948743587514139, "nauc_ndcg_at_1_std": -0.0959960003477958, "nauc_ndcg_at_20_diff1": 0.901386120900766, "nauc_ndcg_at_20_max": 0.8251101196933834, "nauc_ndcg_at_20_std": -0.018990543647164154, "nauc_ndcg_at_3_diff1": 0.8937606553985863, "nauc_ndcg_at_3_max": 0.8227786405631717, "nauc_ndcg_at_3_std": -0.032438022169551876, "nauc_ndcg_at_5_diff1": 0.8991524294049409, "nauc_ndcg_at_5_max": 0.8244725886241099, "nauc_ndcg_at_5_std": -0.010211310284992422, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.85585901027076, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.8068394024276223, "nauc_precision_at_10_diff1": 0.8851762927393119, "nauc_precision_at_10_max": 0.9416433239962604, "nauc_precision_at_10_std": 0.42265795206971, "nauc_precision_at_1_diff1": 0.9129032258064524, "nauc_precision_at_1_max": 0.7948743587514139, "nauc_precision_at_1_std": -0.0959960003477958, "nauc_precision_at_20_diff1": 0.8761042878689921, "nauc_precision_at_20_max": 0.947209653091999, "nauc_precision_at_20_std": 0.4398836457660034, "nauc_precision_at_3_diff1": 0.8670494864612496, "nauc_precision_at_3_max": 0.8558543417366978, "nauc_precision_at_3_std": 0.08870214752567777, "nauc_precision_at_5_diff1": 0.8773109243697506, "nauc_precision_at_5_max": 0.8860255213196399, "nauc_precision_at_5_std": 0.2848428260192923, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8558590102707876, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.8068394024276523, "nauc_recall_at_10_diff1": 0.8851762927393169, "nauc_recall_at_10_max": 0.9416433239962636, "nauc_recall_at_10_std": 0.4226579520697196, "nauc_recall_at_1_diff1": 0.9129032258064524, "nauc_recall_at_1_max": 0.7948743587514139, "nauc_recall_at_1_std": -0.0959960003477958, "nauc_recall_at_20_diff1": 0.8761042878690056, "nauc_recall_at_20_max": 0.9472096530920127, "nauc_recall_at_20_std": 0.4398836457660057, "nauc_recall_at_3_diff1": 0.8670494864612522, "nauc_recall_at_3_max": 0.8558543417366946, "nauc_recall_at_3_std": 0.08870214752567522, "nauc_recall_at_5_diff1": 0.8773109243697501, "nauc_recall_at_5_max": 0.8860255213196395, "nauc_recall_at_5_std": 0.28484282601929706, "ndcg_at_1": 0.87778, "ndcg_at_10": 0.92939, "ndcg_at_100": 0.93337, "ndcg_at_1000": 0.93397, "ndcg_at_20": 0.93163, "ndcg_at_3": 0.91708, "ndcg_at_5": 0.92626, "precision_at_1": 0.87778, "precision_at_10": 0.09767, "precision_at_100": 0.00996, "precision_at_1000": 0.001, "precision_at_20": 0.04928, "precision_at_3": 0.31481, "precision_at_5": 0.19333, "recall_at_1": 0.87778, "recall_at_10": 0.97667, "recall_at_100": 0.99556, "recall_at_1000": 1.0, "recall_at_20": 0.98556, "recall_at_3": 0.94444, "recall_at_5": 0.96667 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 9451.070049762726, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.8108, "map_at_1": 0.49102, "map_at_10": 0.76711, "map_at_100": 0.77193, "map_at_1000": 0.77217, "map_at_20": 0.7704, "map_at_3": 0.72297, "map_at_5": 0.75609, "mrr_at_1": 0.7567567567567568, "mrr_at_10": 0.8184329418704417, "mrr_at_100": 0.8199197488873609, "mrr_at_1000": 0.820035921652267, "mrr_at_20": 0.8191797589412719, "mrr_at_3": 0.8124999999999999, "mrr_at_5": 0.816216216216216, "nauc_map_at_1000_diff1": 0.5860412800077854, "nauc_map_at_1000_max": 0.5688615375646618, "nauc_map_at_1000_std": 0.14232936442918717, "nauc_map_at_100_diff1": 0.5862654281094708, "nauc_map_at_100_max": 0.5693298489926267, "nauc_map_at_100_std": 0.14289907790694886, "nauc_map_at_10_diff1": 0.58259296969473, "nauc_map_at_10_max": 0.5700053608148513, "nauc_map_at_10_std": 0.1378994523753573, "nauc_map_at_1_diff1": 0.632693918726285, "nauc_map_at_1_max": 0.19506800049379597, "nauc_map_at_1_std": -0.11312532728724334, "nauc_map_at_20_diff1": 0.5849545323148008, "nauc_map_at_20_max": 0.5699622379553662, "nauc_map_at_20_std": 0.14174612526369168, "nauc_map_at_3_diff1": 0.5695775561789569, "nauc_map_at_3_max": 0.5001255236964945, "nauc_map_at_3_std": 0.058437237332709745, "nauc_map_at_5_diff1": 0.5759990071437616, "nauc_map_at_5_max": 0.5586872929226386, "nauc_map_at_5_std": 0.10931034514013828, "nauc_mrr_at_1000_diff1": 0.6709973477564796, "nauc_mrr_at_1000_max": 0.6282079865303426, "nauc_mrr_at_1000_std": 0.2182570851932003, "nauc_mrr_at_100_diff1": 0.6711170760242097, "nauc_mrr_at_100_max": 0.628458884294849, "nauc_mrr_at_100_std": 0.21873771212546203, "nauc_mrr_at_10_diff1": 0.6708764756266502, "nauc_mrr_at_10_max": 0.6309082959700567, "nauc_mrr_at_10_std": 0.22073367752871997, "nauc_mrr_at_1_diff1": 0.6512317513672372, "nauc_mrr_at_1_max": 0.4938058683502111, "nauc_mrr_at_1_std": 0.043607255153568676, "nauc_mrr_at_20_diff1": 0.6708019682498405, "nauc_mrr_at_20_max": 0.6295377015786073, "nauc_mrr_at_20_std": 0.21860680581568911, "nauc_mrr_at_3_diff1": 0.6749605035448772, "nauc_mrr_at_3_max": 0.6308357607971359, "nauc_mrr_at_3_std": 0.2123955334390889, "nauc_mrr_at_5_diff1": 0.6707744820554383, "nauc_mrr_at_5_max": 0.6326256995891799, "nauc_mrr_at_5_std": 0.22072942902869305, "nauc_ndcg_at_1000_diff1": 0.6193636658978984, "nauc_ndcg_at_1000_max": 0.630697544541978, "nauc_ndcg_at_1000_std": 0.2386622609694652, "nauc_ndcg_at_100_diff1": 0.6249180286049728, "nauc_ndcg_at_100_max": 0.6433073743489643, "nauc_ndcg_at_100_std": 0.2567146683561388, "nauc_ndcg_at_10_diff1": 0.6136536266134186, "nauc_ndcg_at_10_max": 0.6489710228693197, "nauc_ndcg_at_10_std": 0.24420829169202138, "nauc_ndcg_at_1_diff1": 0.6512317513672372, "nauc_ndcg_at_1_max": 0.4938058683502111, "nauc_ndcg_at_1_std": 0.043607255153568676, "nauc_ndcg_at_20_diff1": 0.6187284093130772, "nauc_ndcg_at_20_max": 0.6481730608510515, "nauc_ndcg_at_20_std": 0.24951834109610457, "nauc_ndcg_at_3_diff1": 0.5955050016913125, "nauc_ndcg_at_3_max": 0.5693055062275632, "nauc_ndcg_at_3_std": 0.13404496786641223, "nauc_ndcg_at_5_diff1": 0.6018163792854944, "nauc_ndcg_at_5_max": 0.629072716288407, "nauc_ndcg_at_5_std": 0.19753897996008837, "nauc_precision_at_1000_diff1": -0.344189042020911, "nauc_precision_at_1000_max": 0.06475013447391754, "nauc_precision_at_1000_std": 0.23917858332558542, "nauc_precision_at_100_diff1": -0.23914321934560318, "nauc_precision_at_100_max": 0.1849182738696575, "nauc_precision_at_100_std": 0.32206416448630437, "nauc_precision_at_10_diff1": -0.16380898142137407, "nauc_precision_at_10_max": 0.2985983037666543, "nauc_precision_at_10_std": 0.33963783741465275, "nauc_precision_at_1_diff1": 0.6512317513672372, "nauc_precision_at_1_max": 0.4938058683502111, "nauc_precision_at_1_std": 0.043607255153568676, "nauc_precision_at_20_diff1": -0.18782381198510406, "nauc_precision_at_20_max": 0.25344887941275157, "nauc_precision_at_20_std": 0.3301393828208045, "nauc_precision_at_3_diff1": -0.03798706450288487, "nauc_precision_at_3_max": 0.41546117059530224, "nauc_precision_at_3_std": 0.2841280644973789, "nauc_precision_at_5_diff1": -0.11961371289466115, "nauc_precision_at_5_max": 0.34773826390289747, "nauc_precision_at_5_std": 0.3053693435767689, "nauc_recall_at_1000_diff1": 0.6975728443917598, "nauc_recall_at_1000_max": 0.9852414530686907, "nauc_recall_at_1000_std": 0.9563648851837822, "nauc_recall_at_100_diff1": 0.7357693746397532, "nauc_recall_at_100_max": 0.970368714583283, "nauc_recall_at_100_std": 0.8305978541126534, "nauc_recall_at_10_diff1": 0.630960245824474, "nauc_recall_at_10_max": 0.8657233835111986, "nauc_recall_at_10_std": 0.5369361718154417, "nauc_recall_at_1_diff1": 0.632693918726285, "nauc_recall_at_1_max": 0.19506800049379597, "nauc_recall_at_1_std": -0.11312532728724334, "nauc_recall_at_20_diff1": 0.6587733846215134, "nauc_recall_at_20_max": 0.9010277052483276, "nauc_recall_at_20_std": 0.608188922151966, "nauc_recall_at_3_diff1": 0.5766836091129506, "nauc_recall_at_3_max": 0.6032568623874444, "nauc_recall_at_3_std": 0.1834427343207938, "nauc_recall_at_5_diff1": 0.5898441814477909, "nauc_recall_at_5_max": 0.7572960798154303, "nauc_recall_at_5_std": 0.34333987643488223, "ndcg_at_1": 0.75676, "ndcg_at_10": 0.8108, "ndcg_at_100": 0.82549, "ndcg_at_1000": 0.83146, "ndcg_at_20": 0.81772, "ndcg_at_3": 0.7644, "ndcg_at_5": 0.79491, "precision_at_1": 0.75676, "precision_at_10": 0.16318, "precision_at_100": 0.0176, "precision_at_1000": 0.00185, "precision_at_20": 0.08412, "precision_at_3": 0.45946, "precision_at_5": 0.30811, "recall_at_1": 0.49102, "recall_at_10": 0.87588, "recall_at_100": 0.92891, "recall_at_1000": 0.9676, "recall_at_20": 0.89667, "recall_at_3": 0.77636, "recall_at_5": 0.838 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 529443.3452823162, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.53375, "map_at_1": 0.32154, "map_at_10": 0.44673, "map_at_100": 0.46751, "map_at_1000": 0.46925, "map_at_20": 0.45789, "map_at_3": 0.40325, "map_at_5": 0.42409, "mrr_at_1": 0.5023474178403756, "mrr_at_10": 0.6107105596542215, "mrr_at_100": 0.6154599409080139, "mrr_at_1000": 0.6155662638623027, "mrr_at_20": 0.6140300142709334, "mrr_at_3": 0.5829420970266039, "mrr_at_5": 0.5989045383411579, "nauc_map_at_1000_diff1": 0.36031682483777533, "nauc_map_at_1000_max": 0.45390972288889114, "nauc_map_at_1000_std": 0.13734187164243297, "nauc_map_at_100_diff1": 0.3600105915368106, "nauc_map_at_100_max": 0.4542942414546183, "nauc_map_at_100_std": 0.13738385626349392, "nauc_map_at_10_diff1": 0.37024010443796035, "nauc_map_at_10_max": 0.45045624061818007, "nauc_map_at_10_std": 0.11853986467680915, "nauc_map_at_1_diff1": 0.4706687514348616, "nauc_map_at_1_max": 0.3785598180609727, "nauc_map_at_1_std": 0.019557004686001363, "nauc_map_at_20_diff1": 0.36274647010544186, "nauc_map_at_20_max": 0.4536888030295927, "nauc_map_at_20_std": 0.12753823479536147, "nauc_map_at_3_diff1": 0.38115606736353363, "nauc_map_at_3_max": 0.4144717952993478, "nauc_map_at_3_std": 0.09752636322817537, "nauc_map_at_5_diff1": 0.37467387847101574, "nauc_map_at_5_max": 0.43567861778923944, "nauc_map_at_5_std": 0.10313974453104882, "nauc_mrr_at_1000_diff1": 0.2852977064023687, "nauc_mrr_at_1000_max": 0.398570864440513, "nauc_mrr_at_1000_std": 0.20335934910198356, "nauc_mrr_at_100_diff1": 0.2853112466469182, "nauc_mrr_at_100_max": 0.39850219473785614, "nauc_mrr_at_100_std": 0.20357699621251318, "nauc_mrr_at_10_diff1": 0.2858037102372598, "nauc_mrr_at_10_max": 0.39935268233483895, "nauc_mrr_at_10_std": 0.20273336747754533, "nauc_mrr_at_1_diff1": 0.35781125255772933, "nauc_mrr_at_1_max": 0.3817075728005826, "nauc_mrr_at_1_std": 0.15291749551166287, "nauc_mrr_at_20_diff1": 0.28526035685683987, "nauc_mrr_at_20_max": 0.399936960823763, "nauc_mrr_at_20_std": 0.20434386179383293, "nauc_mrr_at_3_diff1": 0.2894201723559571, "nauc_mrr_at_3_max": 0.3844481810664284, "nauc_mrr_at_3_std": 0.20303269450222397, "nauc_mrr_at_5_diff1": 0.28541917051266164, "nauc_mrr_at_5_max": 0.40044268989170173, "nauc_mrr_at_5_std": 0.19709143657059006, "nauc_ndcg_at_1000_diff1": 0.3077962815159766, "nauc_ndcg_at_1000_max": 0.46609651208669883, "nauc_ndcg_at_1000_std": 0.21539275920415737, "nauc_ndcg_at_100_diff1": 0.3016914461573391, "nauc_ndcg_at_100_max": 0.4713105342695115, "nauc_ndcg_at_100_std": 0.2263944047923428, "nauc_ndcg_at_10_diff1": 0.32321710435700557, "nauc_ndcg_at_10_max": 0.4674512727191553, "nauc_ndcg_at_10_std": 0.16902846173636296, "nauc_ndcg_at_1_diff1": 0.35781125255772933, "nauc_ndcg_at_1_max": 0.3817075728005826, "nauc_ndcg_at_1_std": 0.15291749551166287, "nauc_ndcg_at_20_diff1": 0.3012113569809693, "nauc_ndcg_at_20_max": 0.47150220023564965, "nauc_ndcg_at_20_std": 0.18771797034660115, "nauc_ndcg_at_3_diff1": 0.3344897853051733, "nauc_ndcg_at_3_max": 0.41103017207276366, "nauc_ndcg_at_3_std": 0.16543980238878778, "nauc_ndcg_at_5_diff1": 0.32596617209752865, "nauc_ndcg_at_5_max": 0.44257031745396613, "nauc_ndcg_at_5_std": 0.15271626934313168, "nauc_precision_at_1000_diff1": -0.19240298740616193, "nauc_precision_at_1000_max": -0.09319519616564716, "nauc_precision_at_1000_std": 0.19996903865573903, "nauc_precision_at_100_diff1": -0.20633671484264068, "nauc_precision_at_100_max": -0.005120399255030897, "nauc_precision_at_100_std": 0.29184767615959145, "nauc_precision_at_10_diff1": -0.1063489415799065, "nauc_precision_at_10_max": 0.24302859238436084, "nauc_precision_at_10_std": 0.2998399174212745, "nauc_precision_at_1_diff1": 0.35781125255772933, "nauc_precision_at_1_max": 0.3817075728005826, "nauc_precision_at_1_std": 0.15291749551166287, "nauc_precision_at_20_diff1": -0.18982635217381524, "nauc_precision_at_20_max": 0.15302492246844374, "nauc_precision_at_20_std": 0.30148495252965213, "nauc_precision_at_3_diff1": 0.051384039218975025, "nauc_precision_at_3_max": 0.3415094720366865, "nauc_precision_at_3_std": 0.3050973666769741, "nauc_precision_at_5_diff1": -0.05159355651665696, "nauc_precision_at_5_max": 0.2895404001134896, "nauc_precision_at_5_std": 0.2870935784123534, "nauc_recall_at_1000_diff1": -0.02540042414067673, "nauc_recall_at_1000_max": 0.6914512479556674, "nauc_recall_at_1000_std": 0.6952669376653589, "nauc_recall_at_100_diff1": 0.06431828985652863, "nauc_recall_at_100_max": 0.49768612041645394, "nauc_recall_at_100_std": 0.5235483628705813, "nauc_recall_at_10_diff1": 0.21299696608876725, "nauc_recall_at_10_max": 0.46908580333257077, "nauc_recall_at_10_std": 0.20166417606311324, "nauc_recall_at_1_diff1": 0.4706687514348616, "nauc_recall_at_1_max": 0.3785598180609727, "nauc_recall_at_1_std": 0.019557004686001363, "nauc_recall_at_20_diff1": 0.11674748568996735, "nauc_recall_at_20_max": 0.4664820858052285, "nauc_recall_at_20_std": 0.24984091736066763, "nauc_recall_at_3_diff1": 0.2913781914497467, "nauc_recall_at_3_max": 0.3799568988978954, "nauc_recall_at_3_std": 0.13544116333758804, "nauc_recall_at_5_diff1": 0.25704180765800555, "nauc_recall_at_5_max": 0.43482488959261517, "nauc_recall_at_5_std": 0.12481012511593734, "ndcg_at_1": 0.50235, "ndcg_at_10": 0.53375, "ndcg_at_100": 0.59893, "ndcg_at_1000": 0.61892, "ndcg_at_20": 0.55959, "ndcg_at_3": 0.48512, "ndcg_at_5": 0.5, "precision_at_1": 0.50235, "precision_at_10": 0.12629, "precision_at_100": 0.02056, "precision_at_1000": 0.00246, "precision_at_20": 0.07535, "precision_at_3": 0.26917, "precision_at_5": 0.19812, "recall_at_1": 0.32154, "recall_at_10": 0.62643, "recall_at_100": 0.84398, "recall_at_1000": 0.95114, "recall_at_20": 0.69904, "recall_at_3": 0.46787, "recall_at_5": 0.5294 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 523189.4379520416, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.46571, "map_at_1": 0.27514, "map_at_10": 0.39939, "map_at_100": 0.41129, "map_at_1000": 0.41168, "map_at_20": 0.4066, "map_at_3": 0.36448, "map_at_5": 0.38421, "mrr_at_1": 0.3016627078384798, "mrr_at_10": 0.4210279757191871, "mrr_at_100": 0.43137655437218475, "mrr_at_1000": 0.4316617049477868, "mrr_at_20": 0.42775991444354394, "mrr_at_3": 0.39152810768012675, "mrr_at_5": 0.40779889152810783, "nauc_map_at_1000_diff1": 0.33293097264950955, "nauc_map_at_1000_max": 0.3130945117290807, "nauc_map_at_1000_std": 0.15320103165009255, "nauc_map_at_100_diff1": 0.3329715076297884, "nauc_map_at_100_max": 0.3129594820935785, "nauc_map_at_100_std": 0.15338062022944135, "nauc_map_at_10_diff1": 0.32887360247558767, "nauc_map_at_10_max": 0.3108233813084867, "nauc_map_at_10_std": 0.1546675312359396, "nauc_map_at_1_diff1": 0.36727145137891976, "nauc_map_at_1_max": 0.2835205184523079, "nauc_map_at_1_std": 0.08284292069749208, "nauc_map_at_20_diff1": 0.3312437874430616, "nauc_map_at_20_max": 0.3132079222147372, "nauc_map_at_20_std": 0.15360937950563022, "nauc_map_at_3_diff1": 0.3359312089046139, "nauc_map_at_3_max": 0.30787457325441847, "nauc_map_at_3_std": 0.13849369378070164, "nauc_map_at_5_diff1": 0.3399343435870189, "nauc_map_at_5_max": 0.31547727388019076, "nauc_map_at_5_std": 0.1446015036644677, "nauc_mrr_at_1000_diff1": 0.34708453070365647, "nauc_mrr_at_1000_max": 0.3327969140637137, "nauc_mrr_at_1000_std": 0.16830876389746163, "nauc_mrr_at_100_diff1": 0.3470604278609808, "nauc_mrr_at_100_max": 0.3326620162101879, "nauc_mrr_at_100_std": 0.16837297579442742, "nauc_mrr_at_10_diff1": 0.3437214867731117, "nauc_mrr_at_10_max": 0.3316098836220819, "nauc_mrr_at_10_std": 0.1690527564620172, "nauc_mrr_at_1_diff1": 0.3767527634408593, "nauc_mrr_at_1_max": 0.3048804875104381, "nauc_mrr_at_1_std": 0.10983635052464556, "nauc_mrr_at_20_diff1": 0.3452672537657949, "nauc_mrr_at_20_max": 0.33298105123957994, "nauc_mrr_at_20_std": 0.1682974971024297, "nauc_mrr_at_3_diff1": 0.349791941798552, "nauc_mrr_at_3_max": 0.3344859077948214, "nauc_mrr_at_3_std": 0.16473453655024795, "nauc_mrr_at_5_diff1": 0.3537503263120288, "nauc_mrr_at_5_max": 0.3353023453737103, "nauc_mrr_at_5_std": 0.1662874495535466, "nauc_ndcg_at_1000_diff1": 0.32908917733637794, "nauc_ndcg_at_1000_max": 0.32613113076041883, "nauc_ndcg_at_1000_std": 0.18277112937224538, "nauc_ndcg_at_100_diff1": 0.329842647374385, "nauc_ndcg_at_100_max": 0.3204276831537327, "nauc_ndcg_at_100_std": 0.18695185786004517, "nauc_ndcg_at_10_diff1": 0.308389151053235, "nauc_ndcg_at_10_max": 0.31744735058957213, "nauc_ndcg_at_10_std": 0.18928893750159045, "nauc_ndcg_at_1_diff1": 0.3767527634408593, "nauc_ndcg_at_1_max": 0.3048804875104381, "nauc_ndcg_at_1_std": 0.10983635052464556, "nauc_ndcg_at_20_diff1": 0.3142097652257374, "nauc_ndcg_at_20_max": 0.3256140956842127, "nauc_ndcg_at_20_std": 0.18825616208232035, "nauc_ndcg_at_3_diff1": 0.32710712769551964, "nauc_ndcg_at_3_max": 0.31887549789970226, "nauc_ndcg_at_3_std": 0.1612799742774508, "nauc_ndcg_at_5_diff1": 0.33316107860910377, "nauc_ndcg_at_5_max": 0.32772903954728455, "nauc_ndcg_at_5_std": 0.16985213318205114, "nauc_precision_at_1000_diff1": 0.05823992403845274, "nauc_precision_at_1000_max": 0.1253829515929163, "nauc_precision_at_1000_std": 0.09610589777902924, "nauc_precision_at_100_diff1": 0.14105102943548245, "nauc_precision_at_100_max": 0.1440230375138777, "nauc_precision_at_100_std": 0.17203917643895358, "nauc_precision_at_10_diff1": 0.18331538719235707, "nauc_precision_at_10_max": 0.28980912626029914, "nauc_precision_at_10_std": 0.2805987246660366, "nauc_precision_at_1_diff1": 0.3767527634408593, "nauc_precision_at_1_max": 0.3048804875104381, "nauc_precision_at_1_std": 0.10983635052464556, "nauc_precision_at_20_diff1": 0.15619077660826267, "nauc_precision_at_20_max": 0.2832576972655273, "nauc_precision_at_20_std": 0.24994856873135776, "nauc_precision_at_3_diff1": 0.30310011086092725, "nauc_precision_at_3_max": 0.3488172732208653, "nauc_precision_at_3_std": 0.2316356138771506, "nauc_precision_at_5_diff1": 0.28818187989791355, "nauc_precision_at_5_max": 0.34918912698140314, "nauc_precision_at_5_std": 0.2403851697874218, "nauc_recall_at_1000_diff1": 0.252208053700925, "nauc_recall_at_1000_max": 0.539409617920156, "nauc_recall_at_1000_std": 0.7654850168562043, "nauc_recall_at_100_diff1": 0.31153747808028986, "nauc_recall_at_100_max": 0.2589106974778827, "nauc_recall_at_100_std": 0.4433876652036297, "nauc_recall_at_10_diff1": 0.20384518056559342, "nauc_recall_at_10_max": 0.29640975881808485, "nauc_recall_at_10_std": 0.2876325943932951, "nauc_recall_at_1_diff1": 0.36727145137891976, "nauc_recall_at_1_max": 0.2835205184523079, "nauc_recall_at_1_std": 0.08284292069749208, "nauc_recall_at_20_diff1": 0.20428489732555838, "nauc_recall_at_20_max": 0.3310372371548431, "nauc_recall_at_20_std": 0.3143954063567378, "nauc_recall_at_3_diff1": 0.2765682171149457, "nauc_recall_at_3_max": 0.3119664143708647, "nauc_recall_at_3_std": 0.19408329545045494, "nauc_recall_at_5_diff1": 0.28498223121657856, "nauc_recall_at_5_max": 0.3281071536534065, "nauc_recall_at_5_std": 0.2149027801698047, "ndcg_at_1": 0.30166, "ndcg_at_10": 0.46571, "ndcg_at_100": 0.52079, "ndcg_at_1000": 0.53041, "ndcg_at_20": 0.4905, "ndcg_at_3": 0.39923, "ndcg_at_5": 0.43194, "precision_at_1": 0.30166, "precision_at_10": 0.07292, "precision_at_100": 0.01043, "precision_at_1000": 0.00113, "precision_at_20": 0.04228, "precision_at_3": 0.1734, "precision_at_5": 0.12209, "recall_at_1": 0.27514, "recall_at_10": 0.64846, "recall_at_100": 0.89351, "recall_at_1000": 0.96793, "recall_at_20": 0.7403, "recall_at_3": 0.47427, "recall_at_5": 0.54988 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 6735.936784029007, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.30313, "map_at_1": 0.21, "map_at_10": 0.27097, "map_at_100": 0.27948, "map_at_1000": 0.28033, "map_at_20": 0.27614, "map_at_3": 0.255, "map_at_5": 0.2635, "mrr_at_1": 0.21, "mrr_at_10": 0.27097023809523807, "mrr_at_100": 0.27947751541679616, "mrr_at_1000": 0.2803278246284526, "mrr_at_20": 0.27614094647182885, "mrr_at_3": 0.25499999999999995, "mrr_at_5": 0.26349999999999996, "nauc_map_at_1000_diff1": 0.35873770357574136, "nauc_map_at_1000_max": 0.31134425197944254, "nauc_map_at_1000_std": -0.06704458741305393, "nauc_map_at_100_diff1": 0.3586784080926965, "nauc_map_at_100_max": 0.311717869958562, "nauc_map_at_100_std": -0.06674245676360822, "nauc_map_at_10_diff1": 0.35860247103967324, "nauc_map_at_10_max": 0.3032036458545803, "nauc_map_at_10_std": -0.0813804882991818, "nauc_map_at_1_diff1": 0.4261168384879724, "nauc_map_at_1_max": 0.32412466150410946, "nauc_map_at_1_std": -0.10102617701553508, "nauc_map_at_20_diff1": 0.35724833739937045, "nauc_map_at_20_max": 0.31083006589902196, "nauc_map_at_20_std": -0.06922956842820864, "nauc_map_at_3_diff1": 0.3684035495695084, "nauc_map_at_3_max": 0.2922831830708324, "nauc_map_at_3_std": -0.08888473191164713, "nauc_map_at_5_diff1": 0.3678964185766755, "nauc_map_at_5_max": 0.30087480560348767, "nauc_map_at_5_std": -0.08768089007739847, "nauc_mrr_at_1000_diff1": 0.35873770357574136, "nauc_mrr_at_1000_max": 0.31134425197944254, "nauc_mrr_at_1000_std": -0.06704458741305393, "nauc_mrr_at_100_diff1": 0.3586784080926965, "nauc_mrr_at_100_max": 0.311717869958562, "nauc_mrr_at_100_std": -0.06674245676360822, "nauc_mrr_at_10_diff1": 0.35860247103967324, "nauc_mrr_at_10_max": 0.3032036458545803, "nauc_mrr_at_10_std": -0.0813804882991818, "nauc_mrr_at_1_diff1": 0.4261168384879724, "nauc_mrr_at_1_max": 0.32412466150410946, "nauc_mrr_at_1_std": -0.10102617701553508, "nauc_mrr_at_20_diff1": 0.35724833739937045, "nauc_mrr_at_20_max": 0.31083006589902196, "nauc_mrr_at_20_std": -0.06922956842820864, "nauc_mrr_at_3_diff1": 0.3684035495695084, "nauc_mrr_at_3_max": 0.2922831830708324, "nauc_mrr_at_3_std": -0.08888473191164713, "nauc_mrr_at_5_diff1": 0.3678964185766755, "nauc_mrr_at_5_max": 0.30087480560348767, "nauc_mrr_at_5_std": -0.08768089007739847, "nauc_ndcg_at_1000_diff1": 0.3352554616023902, "nauc_ndcg_at_1000_max": 0.32663083442920143, "nauc_ndcg_at_1000_std": -0.01311576117979229, "nauc_ndcg_at_100_diff1": 0.3318237851782861, "nauc_ndcg_at_100_max": 0.33657772487092585, "nauc_ndcg_at_100_std": -0.005489102690649493, "nauc_ndcg_at_10_diff1": 0.3314404267618865, "nauc_ndcg_at_10_max": 0.30633427477190284, "nauc_ndcg_at_10_std": -0.06522869923869197, "nauc_ndcg_at_1_diff1": 0.4261168384879724, "nauc_ndcg_at_1_max": 0.32412466150410946, "nauc_ndcg_at_1_std": -0.10102617701553508, "nauc_ndcg_at_20_diff1": 0.3283232800788365, "nauc_ndcg_at_20_max": 0.33313347262163, "nauc_ndcg_at_20_std": -0.02484722727801561, "nauc_ndcg_at_3_diff1": 0.35287080579869623, "nauc_ndcg_at_3_max": 0.2868857758777876, "nauc_ndcg_at_3_std": -0.08231335061500795, "nauc_ndcg_at_5_diff1": 0.35322102269226135, "nauc_ndcg_at_5_max": 0.3024221792638072, "nauc_ndcg_at_5_std": -0.07959401367124273, "nauc_precision_at_1000_diff1": 0.1879961649089168, "nauc_precision_at_1000_max": 0.3998465963566634, "nauc_precision_at_1000_std": 0.49311601150527246, "nauc_precision_at_100_diff1": 0.23821275795609478, "nauc_precision_at_100_max": 0.44484401197135665, "nauc_precision_at_100_std": 0.26764756577194854, "nauc_precision_at_10_diff1": 0.2552144786810836, "nauc_precision_at_10_max": 0.317691509184575, "nauc_precision_at_10_std": -0.017563202383024983, "nauc_precision_at_1_diff1": 0.4261168384879724, "nauc_precision_at_1_max": 0.32412466150410946, "nauc_precision_at_1_std": -0.10102617701553508, "nauc_precision_at_20_diff1": 0.2454380345408315, "nauc_precision_at_20_max": 0.41575801604860807, "nauc_precision_at_20_std": 0.12646039031800008, "nauc_precision_at_3_diff1": 0.31220954795099287, "nauc_precision_at_3_max": 0.2737508466279064, "nauc_precision_at_3_std": -0.06407548232643295, "nauc_precision_at_5_diff1": 0.31606566022809685, "nauc_precision_at_5_max": 0.309862218999275, "nauc_precision_at_5_std": -0.05685279187817251, "nauc_recall_at_1000_diff1": 0.1879961649089169, "nauc_recall_at_1000_max": 0.3998465963566645, "nauc_recall_at_1000_std": 0.4931160115052724, "nauc_recall_at_100_diff1": 0.23821275795609434, "nauc_recall_at_100_max": 0.4448440119713569, "nauc_recall_at_100_std": 0.26764756577194837, "nauc_recall_at_10_diff1": 0.25521447868108316, "nauc_recall_at_10_max": 0.31769150918457467, "nauc_recall_at_10_std": -0.017563202383025025, "nauc_recall_at_1_diff1": 0.4261168384879724, "nauc_recall_at_1_max": 0.32412466150410946, "nauc_recall_at_1_std": -0.10102617701553508, "nauc_recall_at_20_diff1": 0.24543803454083146, "nauc_recall_at_20_max": 0.4157580160486079, "nauc_recall_at_20_std": 0.12646039031800027, "nauc_recall_at_3_diff1": 0.31220954795099265, "nauc_recall_at_3_max": 0.2737508466279063, "nauc_recall_at_3_std": -0.06407548232643298, "nauc_recall_at_5_diff1": 0.31606566022809673, "nauc_recall_at_5_max": 0.30986221899927474, "nauc_recall_at_5_std": -0.05685279187817247, "ndcg_at_1": 0.21, "ndcg_at_10": 0.30313, "ndcg_at_100": 0.34874, "ndcg_at_1000": 0.37803, "ndcg_at_20": 0.32123, "ndcg_at_3": 0.27036, "ndcg_at_5": 0.28521, "precision_at_1": 0.21, "precision_at_10": 0.0405, "precision_at_100": 0.0063, "precision_at_1000": 0.00088, "precision_at_20": 0.02375, "precision_at_3": 0.105, "precision_at_5": 0.07, "recall_at_1": 0.21, "recall_at_10": 0.405, "recall_at_100": 0.63, "recall_at_1000": 0.875, "recall_at_20": 0.475, "recall_at_3": 0.315, "recall_at_5": 0.35 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.29368, "map_at_1": 0.23, "map_at_10": 0.27265, "map_at_100": 0.2815, "map_at_1000": 0.28244, "map_at_20": 0.27838, "map_at_3": 0.25917, "map_at_5": 0.26867, "mrr_at_1": 0.23, "mrr_at_10": 0.27264880952380954, "mrr_at_100": 0.281497391346387, "mrr_at_1000": 0.2824432364298307, "mrr_at_20": 0.2783757250661276, "mrr_at_3": 0.25916666666666666, "mrr_at_5": 0.26866666666666666, "nauc_map_at_1000_diff1": 0.5693273819450104, "nauc_map_at_1000_max": 0.5806473230147973, "nauc_map_at_1000_std": -0.060757043584923086, "nauc_map_at_100_diff1": 0.5693786847659337, "nauc_map_at_100_max": 0.5807719113443569, "nauc_map_at_100_std": -0.06089628585980827, "nauc_map_at_10_diff1": 0.5711855003420948, "nauc_map_at_10_max": 0.5777140106516642, "nauc_map_at_10_std": -0.07319092362315573, "nauc_map_at_1_diff1": 0.6205289730174953, "nauc_map_at_1_max": 0.5944463934351216, "nauc_map_at_1_std": -0.08557679108006284, "nauc_map_at_20_diff1": 0.5724183931836933, "nauc_map_at_20_max": 0.5807674045138349, "nauc_map_at_20_std": -0.06504509217669452, "nauc_map_at_3_diff1": 0.5816747629910118, "nauc_map_at_3_max": 0.5802199578516652, "nauc_map_at_3_std": -0.08412296119192691, "nauc_map_at_5_diff1": 0.5773259359315182, "nauc_map_at_5_max": 0.5791643172847812, "nauc_map_at_5_std": -0.07712777414534365, "nauc_mrr_at_1000_diff1": 0.5693273819450104, "nauc_mrr_at_1000_max": 0.5806473230147973, "nauc_mrr_at_1000_std": -0.060757043584923086, "nauc_mrr_at_100_diff1": 0.5693786847659337, "nauc_mrr_at_100_max": 0.5807719113443569, "nauc_mrr_at_100_std": -0.06089628585980827, "nauc_mrr_at_10_diff1": 0.5711855003420948, "nauc_mrr_at_10_max": 0.5777140106516642, "nauc_mrr_at_10_std": -0.07319092362315573, "nauc_mrr_at_1_diff1": 0.6205289730174953, "nauc_mrr_at_1_max": 0.5944463934351216, "nauc_mrr_at_1_std": -0.08557679108006284, "nauc_mrr_at_20_diff1": 0.5724183931836933, "nauc_mrr_at_20_max": 0.5807674045138349, "nauc_mrr_at_20_std": -0.06504509217669452, "nauc_mrr_at_3_diff1": 0.5816747629910118, "nauc_mrr_at_3_max": 0.5802199578516652, "nauc_mrr_at_3_std": -0.08412296119192691, "nauc_mrr_at_5_diff1": 0.5773259359315182, "nauc_mrr_at_5_max": 0.5791643172847812, "nauc_mrr_at_5_std": -0.07712777414534365, "nauc_ndcg_at_1000_diff1": 0.5362013372269028, "nauc_ndcg_at_1000_max": 0.5832488209073998, "nauc_ndcg_at_1000_std": -0.001090633497059146, "nauc_ndcg_at_100_diff1": 0.5350372618108911, "nauc_ndcg_at_100_max": 0.584824061919428, "nauc_ndcg_at_100_std": 0.00032361783827936655, "nauc_ndcg_at_10_diff1": 0.5479495957959611, "nauc_ndcg_at_10_max": 0.5725374735471958, "nauc_ndcg_at_10_std": -0.061335475446542505, "nauc_ndcg_at_1_diff1": 0.6205289730174953, "nauc_ndcg_at_1_max": 0.5944463934351216, "nauc_ndcg_at_1_std": -0.08557679108006284, "nauc_ndcg_at_20_diff1": 0.5548679294963687, "nauc_ndcg_at_20_max": 0.5874811607238918, "nauc_ndcg_at_20_std": -0.02590888789912424, "nauc_ndcg_at_3_diff1": 0.570878278999106, "nauc_ndcg_at_3_max": 0.5777343931413231, "nauc_ndcg_at_3_std": -0.08385814742901972, "nauc_ndcg_at_5_diff1": 0.5630976166667157, "nauc_ndcg_at_5_max": 0.5759898239262172, "nauc_ndcg_at_5_std": -0.07118431240108142, "nauc_precision_at_1000_diff1": 0.2822715643852065, "nauc_precision_at_1000_max": 0.6049556236379151, "nauc_precision_at_1000_std": 0.5230251729256822, "nauc_precision_at_100_diff1": 0.39846672618740214, "nauc_precision_at_100_max": 0.604278114324083, "nauc_precision_at_100_std": 0.2741446353863069, "nauc_precision_at_10_diff1": 0.4793667663884957, "nauc_precision_at_10_max": 0.5578845851047082, "nauc_precision_at_10_std": -0.02404476985251651, "nauc_precision_at_1_diff1": 0.6205289730174953, "nauc_precision_at_1_max": 0.5944463934351216, "nauc_precision_at_1_std": -0.08557679108006284, "nauc_precision_at_20_diff1": 0.509670660642965, "nauc_precision_at_20_max": 0.6209520068094021, "nauc_precision_at_20_std": 0.1194722713284881, "nauc_precision_at_3_diff1": 0.5415367498592096, "nauc_precision_at_3_max": 0.5714530994601875, "nauc_precision_at_3_std": -0.08318338530005635, "nauc_precision_at_5_diff1": 0.524031100637723, "nauc_precision_at_5_max": 0.5678262837666922, "nauc_precision_at_5_std": -0.053414280198639245, "nauc_recall_at_1000_diff1": 0.2822715643852057, "nauc_recall_at_1000_max": 0.604955623637915, "nauc_recall_at_1000_std": 0.523025172925681, "nauc_recall_at_100_diff1": 0.3984667261874016, "nauc_recall_at_100_max": 0.6042781143240835, "nauc_recall_at_100_std": 0.2741446353863071, "nauc_recall_at_10_diff1": 0.47936676638849496, "nauc_recall_at_10_max": 0.557884585104708, "nauc_recall_at_10_std": -0.024044769852516724, "nauc_recall_at_1_diff1": 0.6205289730174953, "nauc_recall_at_1_max": 0.5944463934351216, "nauc_recall_at_1_std": -0.08557679108006284, "nauc_recall_at_20_diff1": 0.5096706606429648, "nauc_recall_at_20_max": 0.620952006809402, "nauc_recall_at_20_std": 0.11947227132848824, "nauc_recall_at_3_diff1": 0.5415367498592092, "nauc_recall_at_3_max": 0.5714530994601871, "nauc_recall_at_3_std": -0.08318338530005628, "nauc_recall_at_5_diff1": 0.5240311006377228, "nauc_recall_at_5_max": 0.567826283766692, "nauc_recall_at_5_std": -0.053414280198639634, "ndcg_at_1": 0.23, "ndcg_at_10": 0.29368, "ndcg_at_100": 0.34147, "ndcg_at_1000": 0.37178, "ndcg_at_20": 0.31579, "ndcg_at_3": 0.2672, "ndcg_at_5": 0.28399, "precision_at_1": 0.23, "precision_at_10": 0.036, "precision_at_100": 0.00595, "precision_at_1000": 0.00085, "precision_at_20": 0.0225, "precision_at_3": 0.09667, "precision_at_5": 0.066, "recall_at_1": 0.23, "recall_at_10": 0.36, "recall_at_100": 0.595, "recall_at_1000": 0.845, "recall_at_20": 0.45, "recall_at_3": 0.29, "recall_at_5": 0.33 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 46.12740516662598, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.85844, "map_at_1": 0.68831, "map_at_10": 0.81196, "map_at_100": 0.81196, "map_at_1000": 0.81196, "map_at_20": 0.81196, "map_at_3": 0.80303, "map_at_5": 0.80303, "mrr_at_1": 0.6883116883116883, "mrr_at_10": 0.8119562976705834, "mrr_at_100": 0.8119562976705834, "mrr_at_1000": 0.8119562976705834, "mrr_at_20": 0.8119562976705834, "mrr_at_3": 0.8030303030303031, "mrr_at_5": 0.8030303030303031, "nauc_map_at_1000_diff1": 0.7306499143837317, "nauc_map_at_1000_max": 0.3237895227119708, "nauc_map_at_1000_std": -0.16620176663433442, "nauc_map_at_100_diff1": 0.7306499143837317, "nauc_map_at_100_max": 0.3237895227119708, "nauc_map_at_100_std": -0.16620176663433442, "nauc_map_at_10_diff1": 0.7306499143837317, "nauc_map_at_10_max": 0.3237895227119708, "nauc_map_at_10_std": -0.16620176663433442, "nauc_map_at_1_diff1": 0.7422916035829402, "nauc_map_at_1_max": 0.3482164359211589, "nauc_map_at_1_std": -0.09165116325682032, "nauc_map_at_20_diff1": 0.7306499143837317, "nauc_map_at_20_max": 0.3237895227119708, "nauc_map_at_20_std": -0.16620176663433442, "nauc_map_at_3_diff1": 0.727274397793811, "nauc_map_at_3_max": 0.3229930586432481, "nauc_map_at_3_std": -0.18695188162216977, "nauc_map_at_5_diff1": 0.727274397793811, "nauc_map_at_5_max": 0.3229930586432481, "nauc_map_at_5_std": -0.18695188162216977, "nauc_mrr_at_1000_diff1": 0.7306499143837317, "nauc_mrr_at_1000_max": 0.3237895227119708, "nauc_mrr_at_1000_std": -0.16620176663433442, "nauc_mrr_at_100_diff1": 0.7306499143837317, "nauc_mrr_at_100_max": 0.3237895227119708, "nauc_mrr_at_100_std": -0.16620176663433442, "nauc_mrr_at_10_diff1": 0.7306499143837317, "nauc_mrr_at_10_max": 0.3237895227119708, "nauc_mrr_at_10_std": -0.16620176663433442, "nauc_mrr_at_1_diff1": 0.7422916035829402, "nauc_mrr_at_1_max": 0.3482164359211589, "nauc_mrr_at_1_std": -0.09165116325682032, "nauc_mrr_at_20_diff1": 0.7306499143837317, "nauc_mrr_at_20_max": 0.3237895227119708, "nauc_mrr_at_20_std": -0.16620176663433442, "nauc_mrr_at_3_diff1": 0.727274397793811, "nauc_mrr_at_3_max": 0.3229930586432481, "nauc_mrr_at_3_std": -0.18695188162216977, "nauc_mrr_at_5_diff1": 0.727274397793811, "nauc_mrr_at_5_max": 0.3229930586432481, "nauc_mrr_at_5_std": -0.18695188162216977, "nauc_ndcg_at_1000_diff1": 0.7298622798036085, "nauc_ndcg_at_1000_max": 0.32081770832924095, "nauc_ndcg_at_1000_std": -0.17269361184898482, "nauc_ndcg_at_100_diff1": 0.7298622798036085, "nauc_ndcg_at_100_max": 0.32081770832924095, "nauc_ndcg_at_100_std": -0.17269361184898482, "nauc_ndcg_at_10_diff1": 0.7298622798036085, "nauc_ndcg_at_10_max": 0.32081770832924095, "nauc_ndcg_at_10_std": -0.17269361184898482, "nauc_ndcg_at_1_diff1": 0.7422916035829402, "nauc_ndcg_at_1_max": 0.3482164359211589, "nauc_ndcg_at_1_std": -0.09165116325682032, "nauc_ndcg_at_20_diff1": 0.7298622798036085, "nauc_ndcg_at_20_max": 0.32081770832924095, "nauc_ndcg_at_20_std": -0.17269361184898482, "nauc_ndcg_at_3_diff1": 0.7219783130912044, "nauc_ndcg_at_3_max": 0.31347352334832335, "nauc_ndcg_at_3_std": -0.22725218013410522, "nauc_ndcg_at_5_diff1": 0.7219783130912044, "nauc_ndcg_at_5_max": 0.31347352334832335, "nauc_ndcg_at_5_std": -0.22725218013410522, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 1.0, "nauc_precision_at_10_max": 1.0, "nauc_precision_at_10_std": 1.0, "nauc_precision_at_1_diff1": 0.7422916035829402, "nauc_precision_at_1_max": 0.3482164359211589, "nauc_precision_at_1_std": -0.09165116325682032, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_3_diff1": 0.6835820481559275, "nauc_precision_at_3_max": 0.24373783838225974, "nauc_precision_at_3_std": -0.5271426588640465, "nauc_precision_at_5_diff1": 0.6835820481559289, "nauc_precision_at_5_max": 0.24373783838226237, "nauc_precision_at_5_std": -0.5271426588640427, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": NaN, "nauc_recall_at_10_max": NaN, "nauc_recall_at_10_std": NaN, "nauc_recall_at_1_diff1": 0.7422916035829402, "nauc_recall_at_1_max": 0.3482164359211589, "nauc_recall_at_1_std": -0.09165116325682032, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_3_diff1": 0.6835820481559286, "nauc_recall_at_3_max": 0.24373783838225746, "nauc_recall_at_3_std": -0.5271426588640481, "nauc_recall_at_5_diff1": 0.6835820481559286, "nauc_recall_at_5_max": 0.24373783838225746, "nauc_recall_at_5_std": -0.5271426588640481, "ndcg_at_1": 0.68831, "ndcg_at_10": 0.85844, "ndcg_at_100": 0.85844, "ndcg_at_1000": 0.85844, "ndcg_at_20": 0.85844, "ndcg_at_3": 0.83719, "ndcg_at_5": 0.83719, "precision_at_1": 0.68831, "precision_at_10": 0.1, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.05, "precision_at_3": 0.31169, "precision_at_5": 0.18701, "recall_at_1": 0.68831, "recall_at_10": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 1.0, "recall_at_3": 0.93506, "recall_at_5": 0.93506 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 373.6595883369446, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.37866, "map_at_1": 0.21493, "map_at_10": 0.3305, "map_at_100": 0.34654, "map_at_1000": 0.34759, "map_at_20": 0.33866, "map_at_3": 0.2948, "map_at_5": 0.31486, "mrr_at_1": 0.2996941896024465, "mrr_at_10": 0.36733168292801305, "mrr_at_100": 0.3793843923095916, "mrr_at_1000": 0.380081832466756, "mrr_at_20": 0.37381202387628654, "mrr_at_3": 0.34378185524974525, "mrr_at_5": 0.35616717635066264, "nauc_map_at_1000_diff1": 0.39605398696420363, "nauc_map_at_1000_max": 0.0412782945923716, "nauc_map_at_1000_std": -0.3673566334388843, "nauc_map_at_100_diff1": 0.395789867336939, "nauc_map_at_100_max": 0.04155953451877115, "nauc_map_at_100_std": -0.36651315519418337, "nauc_map_at_10_diff1": 0.3977984105897261, "nauc_map_at_10_max": 0.03743921340462259, "nauc_map_at_10_std": -0.37518750626502256, "nauc_map_at_1_diff1": 0.446152501423059, "nauc_map_at_1_max": 0.02663440208773937, "nauc_map_at_1_std": -0.33965214961688095, "nauc_map_at_20_diff1": 0.39625562343728227, "nauc_map_at_20_max": 0.038924158809186135, "nauc_map_at_20_std": -0.37210104220524026, "nauc_map_at_3_diff1": 0.39380176890037777, "nauc_map_at_3_max": 0.03416038244220423, "nauc_map_at_3_std": -0.3681811178017894, "nauc_map_at_5_diff1": 0.39571643464140555, "nauc_map_at_5_max": 0.03721875189175598, "nauc_map_at_5_std": -0.37458573833597125, "nauc_mrr_at_1000_diff1": 0.4204049613713672, "nauc_mrr_at_1000_max": 0.04473616768863933, "nauc_mrr_at_1000_std": -0.3683979005894126, "nauc_mrr_at_100_diff1": 0.42024967745486136, "nauc_mrr_at_100_max": 0.04499165061867992, "nauc_mrr_at_100_std": -0.3677313105342792, "nauc_mrr_at_10_diff1": 0.4213875590775421, "nauc_mrr_at_10_max": 0.03916398461276921, "nauc_mrr_at_10_std": -0.3760084893557857, "nauc_mrr_at_1_diff1": 0.43496360123353245, "nauc_mrr_at_1_max": 0.05114237339724553, "nauc_mrr_at_1_std": -0.3737822614588633, "nauc_mrr_at_20_diff1": 0.42058757596953117, "nauc_mrr_at_20_max": 0.042627432553596564, "nauc_mrr_at_20_std": -0.37140474184158695, "nauc_mrr_at_3_diff1": 0.42186574684849265, "nauc_mrr_at_3_max": 0.03772375422449969, "nauc_mrr_at_3_std": -0.3779124477466279, "nauc_mrr_at_5_diff1": 0.4199331499830262, "nauc_mrr_at_5_max": 0.03760842836763998, "nauc_mrr_at_5_std": -0.37814230032915724, "nauc_ndcg_at_1000_diff1": 0.3978473800651747, "nauc_ndcg_at_1000_max": 0.053960682039080894, "nauc_ndcg_at_1000_std": -0.3476679618966263, "nauc_ndcg_at_100_diff1": 0.3936901716079827, "nauc_ndcg_at_100_max": 0.06975428508696647, "nauc_ndcg_at_100_std": -0.3176601369260679, "nauc_ndcg_at_10_diff1": 0.4015378876968567, "nauc_ndcg_at_10_max": 0.03791666857344486, "nauc_ndcg_at_10_std": -0.37436111440335995, "nauc_ndcg_at_1_diff1": 0.4302105091963213, "nauc_ndcg_at_1_max": 0.04725753243883758, "nauc_ndcg_at_1_std": -0.3773456769429174, "nauc_ndcg_at_20_diff1": 0.3962429416823341, "nauc_ndcg_at_20_max": 0.04389393336467113, "nauc_ndcg_at_20_std": -0.3630483112178299, "nauc_ndcg_at_3_diff1": 0.3950615196047281, "nauc_ndcg_at_3_max": 0.030305615511054654, "nauc_ndcg_at_3_std": -0.37394136253274857, "nauc_ndcg_at_5_diff1": 0.3963854674719409, "nauc_ndcg_at_5_max": 0.034625800368698355, "nauc_ndcg_at_5_std": -0.375950275291491, "nauc_precision_at_1000_diff1": -0.034626884552404966, "nauc_precision_at_1000_max": 0.07690224605888045, "nauc_precision_at_1000_std": 0.021871172442492676, "nauc_precision_at_100_diff1": 0.0714801115254476, "nauc_precision_at_100_max": 0.13315561541052745, "nauc_precision_at_100_std": 0.03444855372602813, "nauc_precision_at_10_diff1": 0.23573365255125925, "nauc_precision_at_10_max": 0.035876362649886924, "nauc_precision_at_10_std": -0.29724374204707604, "nauc_precision_at_1_diff1": 0.4302105091963213, "nauc_precision_at_1_max": 0.04725753243883758, "nauc_precision_at_1_std": -0.3773456769429174, "nauc_precision_at_20_diff1": 0.1874260473657467, "nauc_precision_at_20_max": 0.04232031468123627, "nauc_precision_at_20_std": -0.23356189942493982, "nauc_precision_at_3_diff1": 0.26903273918480025, "nauc_precision_at_3_max": 0.04616211001781429, "nauc_precision_at_3_std": -0.3359670496699033, "nauc_precision_at_5_diff1": 0.2533080980168239, "nauc_precision_at_5_max": 0.0447726069996935, "nauc_precision_at_5_std": -0.32679138663754764, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.310564964535451, "nauc_recall_at_100_max": 0.30173839624071874, "nauc_recall_at_100_std": 0.11377334515723407, "nauc_recall_at_10_diff1": 0.37982695676707556, "nauc_recall_at_10_max": 0.042492451912603056, "nauc_recall_at_10_std": -0.35231447642883357, "nauc_recall_at_1_diff1": 0.446152501423059, "nauc_recall_at_1_max": 0.02663440208773937, "nauc_recall_at_1_std": -0.33965214961688095, "nauc_recall_at_20_diff1": 0.35638430589180825, "nauc_recall_at_20_max": 0.06328227941791602, "nauc_recall_at_20_std": -0.3113052824785763, "nauc_recall_at_3_diff1": 0.3712633595908005, "nauc_recall_at_3_max": 0.02494288594337995, "nauc_recall_at_3_std": -0.35548445928432737, "nauc_recall_at_5_diff1": 0.36571414475870745, "nauc_recall_at_5_max": 0.029103482587579426, "nauc_recall_at_5_std": -0.3588705476558524, "ndcg_at_1": 0.30122, "ndcg_at_10": 0.37866, "ndcg_at_100": 0.45423, "ndcg_at_1000": 0.4792, "ndcg_at_20": 0.40376, "ndcg_at_3": 0.3221, "ndcg_at_5": 0.34735, "precision_at_1": 0.30122, "precision_at_10": 0.07691, "precision_at_100": 0.01303, "precision_at_1000": 0.00156, "precision_at_20": 0.0458, "precision_at_3": 0.17431, "precision_at_5": 0.12508, "recall_at_1": 0.21493, "recall_at_10": 0.4843, "recall_at_100": 0.8169, "recall_at_1000": 1.0, "recall_at_20": 0.56944, "recall_at_3": 0.33876, "recall_at_5": 0.40008 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.39995, "map_at_1": 0.19177, "map_at_10": 0.33984, "map_at_100": 0.35699, "map_at_1000": 0.35838, "map_at_20": 0.34825, "map_at_3": 0.28717, "map_at_5": 0.3174, "mrr_at_1": 0.3318042813455658, "mrr_at_10": 0.40260545604582293, "mrr_at_100": 0.41334522944107477, "mrr_at_1000": 0.4138877971938659, "mrr_at_20": 0.4083959893443803, "mrr_at_3": 0.374362895005097, "mrr_at_5": 0.38934760448521916, "nauc_map_at_1000_diff1": 0.41058617945545617, "nauc_map_at_1000_max": -0.0013002952537210077, "nauc_map_at_1000_std": -0.402768625964047, "nauc_map_at_100_diff1": 0.4103859458849651, "nauc_map_at_100_max": -0.0024929692398992275, "nauc_map_at_100_std": -0.40403076519830755, "nauc_map_at_10_diff1": 0.4066546282104827, "nauc_map_at_10_max": -0.002486845640082756, "nauc_map_at_10_std": -0.4053493086971202, "nauc_map_at_1_diff1": 0.46831829280036646, "nauc_map_at_1_max": -0.014474441932302181, "nauc_map_at_1_std": -0.34551612668189086, "nauc_map_at_20_diff1": 0.4101210254500855, "nauc_map_at_20_max": -0.0047810042551561975, "nauc_map_at_20_std": -0.4079539137908949, "nauc_map_at_3_diff1": 0.4391849418232014, "nauc_map_at_3_max": 0.012395764653712059, "nauc_map_at_3_std": -0.38991991301597145, "nauc_map_at_5_diff1": 0.41917501735931717, "nauc_map_at_5_max": 0.008385330503281803, "nauc_map_at_5_std": -0.3995164775690684, "nauc_mrr_at_1000_diff1": 0.40920660076706167, "nauc_mrr_at_1000_max": 0.0012385941769198715, "nauc_mrr_at_1000_std": -0.3932741446560242, "nauc_mrr_at_100_diff1": 0.40904930361090497, "nauc_mrr_at_100_max": 0.0010593221401256512, "nauc_mrr_at_100_std": -0.39335860583329924, "nauc_mrr_at_10_diff1": 0.40766999280728256, "nauc_mrr_at_10_max": -8.088705493833043e-05, "nauc_mrr_at_10_std": -0.39530918705522655, "nauc_mrr_at_1_diff1": 0.4427688268960664, "nauc_mrr_at_1_max": 0.01633880940031036, "nauc_mrr_at_1_std": -0.38611719450212045, "nauc_mrr_at_20_diff1": 0.4090902574437497, "nauc_mrr_at_20_max": -0.0009201069513837534, "nauc_mrr_at_20_std": -0.39607998114446913, "nauc_mrr_at_3_diff1": 0.41695142105898064, "nauc_mrr_at_3_max": 0.004057669729456041, "nauc_mrr_at_3_std": -0.39551144383536135, "nauc_mrr_at_5_diff1": 0.4070267126250566, "nauc_mrr_at_5_max": 0.005278939293701363, "nauc_mrr_at_5_std": -0.3919125656637338, "nauc_ndcg_at_1000_diff1": 0.40215131033707746, "nauc_ndcg_at_1000_max": 0.001258704325013927, "nauc_ndcg_at_1000_std": -0.3908957262121216, "nauc_ndcg_at_100_diff1": 0.39832291711852713, "nauc_ndcg_at_100_max": -0.004799637066320755, "nauc_ndcg_at_100_std": -0.393164829192417, "nauc_ndcg_at_10_diff1": 0.38899882407459435, "nauc_ndcg_at_10_max": -0.018537873597670097, "nauc_ndcg_at_10_std": -0.4141314712129482, "nauc_ndcg_at_1_diff1": 0.4427688268960664, "nauc_ndcg_at_1_max": 0.01633880940031036, "nauc_ndcg_at_1_std": -0.38611719450212045, "nauc_ndcg_at_20_diff1": 0.39736311864081186, "nauc_ndcg_at_20_max": -0.026621311940814282, "nauc_ndcg_at_20_std": -0.4212133721923276, "nauc_ndcg_at_3_diff1": 0.4190616958577465, "nauc_ndcg_at_3_max": 0.009013169509649496, "nauc_ndcg_at_3_std": -0.3993861491322945, "nauc_ndcg_at_5_diff1": 0.4032274550560342, "nauc_ndcg_at_5_max": 0.0024637382960664166, "nauc_ndcg_at_5_std": -0.40340444594494707, "nauc_precision_at_1000_diff1": 0.004259057835218737, "nauc_precision_at_1000_max": 0.08826356081485175, "nauc_precision_at_1000_std": 0.01338165737445073, "nauc_precision_at_100_diff1": 0.06902501449184535, "nauc_precision_at_100_max": 0.05582969368179055, "nauc_precision_at_100_std": -0.08454217226602033, "nauc_precision_at_10_diff1": 0.17246682812480735, "nauc_precision_at_10_max": -0.012406289944737326, "nauc_precision_at_10_std": -0.3006788219394346, "nauc_precision_at_1_diff1": 0.4427688268960664, "nauc_precision_at_1_max": 0.01633880940031036, "nauc_precision_at_1_std": -0.38611719450212045, "nauc_precision_at_20_diff1": 0.163521463625998, "nauc_precision_at_20_max": -0.01981955957289997, "nauc_precision_at_20_std": -0.27855391812427455, "nauc_precision_at_3_diff1": 0.3015896868111652, "nauc_precision_at_3_max": 0.038691751764205154, "nauc_precision_at_3_std": -0.347123228838273, "nauc_precision_at_5_diff1": 0.23111191170926612, "nauc_precision_at_5_max": 0.02370344878705259, "nauc_precision_at_5_std": -0.32399634526515875, "nauc_recall_at_1000_diff1": 0.5647183521144965, "nauc_recall_at_1000_max": 0.41208649490966276, "nauc_recall_at_1000_std": -0.15695523713683882, "nauc_recall_at_100_diff1": 0.3156808945250338, "nauc_recall_at_100_max": 0.01423885208773468, "nauc_recall_at_100_std": -0.2687000214523801, "nauc_recall_at_10_diff1": 0.3193138140628656, "nauc_recall_at_10_max": -0.06363018392286901, "nauc_recall_at_10_std": -0.4226385831868655, "nauc_recall_at_1_diff1": 0.46831829280036646, "nauc_recall_at_1_max": -0.014474441932302181, "nauc_recall_at_1_std": -0.34551612668189086, "nauc_recall_at_20_diff1": 0.3433179839518331, "nauc_recall_at_20_max": -0.1038140150525632, "nauc_recall_at_20_std": -0.45289277852456233, "nauc_recall_at_3_diff1": 0.39994036352425977, "nauc_recall_at_3_max": -0.005326393503024048, "nauc_recall_at_3_std": -0.3869023976730874, "nauc_recall_at_5_diff1": 0.3578112098305799, "nauc_recall_at_5_max": -0.006546757260117465, "nauc_recall_at_5_std": -0.39532613786244336, "ndcg_at_1": 0.3318, "ndcg_at_10": 0.39995, "ndcg_at_100": 0.47688, "ndcg_at_1000": 0.50029, "ndcg_at_20": 0.42508, "ndcg_at_3": 0.33655, "ndcg_at_5": 0.35826, "precision_at_1": 0.3318, "precision_at_10": 0.10489, "precision_at_100": 0.01665, "precision_at_1000": 0.00195, "precision_at_20": 0.06063, "precision_at_3": 0.21764, "precision_at_5": 0.16483, "recall_at_1": 0.19177, "recall_at_10": 0.51012, "recall_at_100": 0.83907, "recall_at_1000": 0.99503, "recall_at_20": 0.59455, "recall_at_3": 0.32989, "recall_at_5": 0.40344 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.36948, "map_at_1": 0.20615, "map_at_10": 0.3216, "map_at_100": 0.33743, "map_at_1000": 0.33861, "map_at_20": 0.32978, "map_at_3": 0.2846, "map_at_5": 0.3068, "mrr_at_1": 0.2980456026058632, "mrr_at_10": 0.3613728607621115, "mrr_at_100": 0.37275227119258564, "mrr_at_1000": 0.3734207310023052, "mrr_at_20": 0.3672744150330474, "mrr_at_3": 0.3390336590662324, "mrr_at_5": 0.351900108577633, "nauc_map_at_1000_diff1": 0.4373721133535326, "nauc_map_at_1000_max": 0.09733685887316677, "nauc_map_at_1000_std": -0.309625544370664, "nauc_map_at_100_diff1": 0.4370183083119422, "nauc_map_at_100_max": 0.0972950650230623, "nauc_map_at_100_std": -0.3090029910345518, "nauc_map_at_10_diff1": 0.4391928166766258, "nauc_map_at_10_max": 0.09307475831631143, "nauc_map_at_10_std": -0.3162571191444478, "nauc_map_at_1_diff1": 0.5602577371730281, "nauc_map_at_1_max": 0.10724644675399574, "nauc_map_at_1_std": -0.277598300172147, "nauc_map_at_20_diff1": 0.43948424825376897, "nauc_map_at_20_max": 0.0930331336171994, "nauc_map_at_20_std": -0.31738766465049795, "nauc_map_at_3_diff1": 0.46090849293075453, "nauc_map_at_3_max": 0.10879690061773899, "nauc_map_at_3_std": -0.29945481511490535, "nauc_map_at_5_diff1": 0.43825072507014, "nauc_map_at_5_max": 0.09575663793164427, "nauc_map_at_5_std": -0.3117128053942495, "nauc_mrr_at_1000_diff1": 0.46421231294131865, "nauc_mrr_at_1000_max": 0.10977960061397075, "nauc_mrr_at_1000_std": -0.3030153340987944, "nauc_mrr_at_100_diff1": 0.46392796505574907, "nauc_mrr_at_100_max": 0.11010970849782215, "nauc_mrr_at_100_std": -0.30223842219228625, "nauc_mrr_at_10_diff1": 0.46498471958808024, "nauc_mrr_at_10_max": 0.106646067543003, "nauc_mrr_at_10_std": -0.30795365867667474, "nauc_mrr_at_1_diff1": 0.500778242420633, "nauc_mrr_at_1_max": 0.13070668371304484, "nauc_mrr_at_1_std": -0.30514994152834823, "nauc_mrr_at_20_diff1": 0.465026746737862, "nauc_mrr_at_20_max": 0.10683571107326713, "nauc_mrr_at_20_std": -0.30811545466865076, "nauc_mrr_at_3_diff1": 0.4718357358171834, "nauc_mrr_at_3_max": 0.11748827081331539, "nauc_mrr_at_3_std": -0.30066474511851476, "nauc_mrr_at_5_diff1": 0.46458638326622315, "nauc_mrr_at_5_max": 0.1046713557604587, "nauc_mrr_at_5_std": -0.308084580457927, "nauc_ndcg_at_1000_diff1": 0.4283433731130498, "nauc_ndcg_at_1000_max": 0.10949293288269658, "nauc_ndcg_at_1000_std": -0.28754953984153664, "nauc_ndcg_at_100_diff1": 0.41417211580021823, "nauc_ndcg_at_100_max": 0.11602320148783662, "nauc_ndcg_at_100_std": -0.2606633885264539, "nauc_ndcg_at_10_diff1": 0.43018721793998604, "nauc_ndcg_at_10_max": 0.08274497479111372, "nauc_ndcg_at_10_std": -0.32042032605870274, "nauc_ndcg_at_1_diff1": 0.49473135350997494, "nauc_ndcg_at_1_max": 0.13578656720000074, "nauc_ndcg_at_1_std": -0.29611939726083886, "nauc_ndcg_at_20_diff1": 0.43105443007993605, "nauc_ndcg_at_20_max": 0.0827979056933635, "nauc_ndcg_at_20_std": -0.3232816484442792, "nauc_ndcg_at_3_diff1": 0.45036219779939474, "nauc_ndcg_at_3_max": 0.10478070893705232, "nauc_ndcg_at_3_std": -0.30016964568421, "nauc_ndcg_at_5_diff1": 0.4281601682293345, "nauc_ndcg_at_5_max": 0.08520122895762401, "nauc_ndcg_at_5_std": -0.31392015201203394, "nauc_precision_at_1000_diff1": -0.10680404046939247, "nauc_precision_at_1000_max": 0.1781045855411332, "nauc_precision_at_1000_std": 0.10293066129639211, "nauc_precision_at_100_diff1": -0.02641496096230086, "nauc_precision_at_100_max": 0.18623465151680246, "nauc_precision_at_100_std": 0.08975371905085901, "nauc_precision_at_10_diff1": 0.16369753143889806, "nauc_precision_at_10_max": 0.047241852955354345, "nauc_precision_at_10_std": -0.25690527620071385, "nauc_precision_at_1_diff1": 0.49473135350997494, "nauc_precision_at_1_max": 0.13578656720000074, "nauc_precision_at_1_std": -0.29611939726083886, "nauc_precision_at_20_diff1": 0.14351541443712884, "nauc_precision_at_20_max": 0.06193074008380701, "nauc_precision_at_20_std": -0.21514301134983735, "nauc_precision_at_3_diff1": 0.25417523312181733, "nauc_precision_at_3_max": 0.11194211250906363, "nauc_precision_at_3_std": -0.2606506300756569, "nauc_precision_at_5_diff1": 0.1848561701587436, "nauc_precision_at_5_max": 0.07073984164943047, "nauc_precision_at_5_std": -0.2682731675801018, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.20706354222903484, "nauc_recall_at_100_max": 0.21365885661701703, "nauc_recall_at_100_std": 0.09035637562095138, "nauc_recall_at_10_diff1": 0.3677163354241613, "nauc_recall_at_10_max": 0.03575033957027068, "nauc_recall_at_10_std": -0.3280144585910673, "nauc_recall_at_1_diff1": 0.5602577371730281, "nauc_recall_at_1_max": 0.10724644675399574, "nauc_recall_at_1_std": -0.277598300172147, "nauc_recall_at_20_diff1": 0.3642988806351914, "nauc_recall_at_20_max": 0.026840971376837584, "nauc_recall_at_20_std": -0.34325729113061004, "nauc_recall_at_3_diff1": 0.4275675122137623, "nauc_recall_at_3_max": 0.09106384319877968, "nauc_recall_at_3_std": -0.28650617926616817, "nauc_recall_at_5_diff1": 0.3698909986822292, "nauc_recall_at_5_max": 0.03906080211614774, "nauc_recall_at_5_std": -0.3178511477594843, "ndcg_at_1": 0.29967, "ndcg_at_10": 0.36948, "ndcg_at_100": 0.44298, "ndcg_at_1000": 0.47161, "ndcg_at_20": 0.39425, "ndcg_at_3": 0.31568, "ndcg_at_5": 0.34151, "precision_at_1": 0.29967, "precision_at_10": 0.07948, "precision_at_100": 0.01355, "precision_at_1000": 0.00167, "precision_at_20": 0.04707, "precision_at_3": 0.18024, "precision_at_5": 0.13192, "recall_at_1": 0.20615, "recall_at_10": 0.46939, "recall_at_100": 0.78699, "recall_at_1000": 1.0, "recall_at_20": 0.55504, "recall_at_3": 0.32724, "recall_at_5": 0.39265 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/model_meta.json ================================================ {"name": "Alibaba-NLP/gte-Qwen2-7B-instruct", "revision": "e26182b2122f4435e8b3ebecbf363990f409b45b", "release_date": "2024-06-15", "languages": null, "n_parameters": 7613000000, "memory_usage": null, "max_tokens": null, "embed_dim": 3584, "license": "apache-2.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 9.638917446136475, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.77108, "map_at_1": 0.59649, "map_at_10": 0.71667, "map_at_100": 0.71923, "map_at_1000": 0.71923, "map_at_20": 0.71793, "map_at_3": 0.69298, "map_at_5": 0.7057, "mrr_at_1": 0.5964912280701754, "mrr_at_10": 0.7166666666666666, "mrr_at_100": 0.7192301247857292, "mrr_at_1000": 0.7192301247857292, "mrr_at_20": 0.717925795349618, "mrr_at_3": 0.6929824561403509, "mrr_at_5": 0.7057017543859649, "nauc_map_at_1000_diff1": 0.5936517099361166, "nauc_map_at_1000_max": 0.23025820800907956, "nauc_map_at_1000_std": -0.44830854310298024, "nauc_map_at_100_diff1": 0.5936517099361166, "nauc_map_at_100_max": 0.23025820800907956, "nauc_map_at_100_std": -0.44830854310298024, "nauc_map_at_10_diff1": 0.5893271930047639, "nauc_map_at_10_max": 0.2276498001316117, "nauc_map_at_10_std": -0.4551546095194692, "nauc_map_at_1_diff1": 0.6110833815898389, "nauc_map_at_1_max": 0.1908294474342627, "nauc_map_at_1_std": -0.35314996932177417, "nauc_map_at_20_diff1": 0.59448055498058, "nauc_map_at_20_max": 0.2329357588721152, "nauc_map_at_20_std": -0.45255932263784404, "nauc_map_at_3_diff1": 0.5938383228262359, "nauc_map_at_3_max": 0.23164372605869038, "nauc_map_at_3_std": -0.43564852034913903, "nauc_map_at_5_diff1": 0.5900023435482211, "nauc_map_at_5_max": 0.23294445889550133, "nauc_map_at_5_std": -0.45567242172858874, "nauc_mrr_at_1000_diff1": 0.5936517099361166, "nauc_mrr_at_1000_max": 0.23025820800907956, "nauc_mrr_at_1000_std": -0.44830854310298024, "nauc_mrr_at_100_diff1": 0.5936517099361166, "nauc_mrr_at_100_max": 0.23025820800907956, "nauc_mrr_at_100_std": -0.44830854310298024, "nauc_mrr_at_10_diff1": 0.5893271930047639, "nauc_mrr_at_10_max": 0.2276498001316117, "nauc_mrr_at_10_std": -0.4551546095194692, "nauc_mrr_at_1_diff1": 0.6110833815898389, "nauc_mrr_at_1_max": 0.1908294474342627, "nauc_mrr_at_1_std": -0.35314996932177417, "nauc_mrr_at_20_diff1": 0.59448055498058, "nauc_mrr_at_20_max": 0.2329357588721152, "nauc_mrr_at_20_std": -0.45255932263784404, "nauc_mrr_at_3_diff1": 0.5938383228262359, "nauc_mrr_at_3_max": 0.23164372605869038, "nauc_mrr_at_3_std": -0.43564852034913903, "nauc_mrr_at_5_diff1": 0.5900023435482211, "nauc_mrr_at_5_max": 0.23294445889550133, "nauc_mrr_at_5_std": -0.45567242172858874, "nauc_ndcg_at_1000_diff1": 0.5925722016847822, "nauc_ndcg_at_1000_max": 0.2381630983055214, "nauc_ndcg_at_1000_std": -0.467474281511242, "nauc_ndcg_at_100_diff1": 0.5925722016847822, "nauc_ndcg_at_100_max": 0.2381630983055214, "nauc_ndcg_at_100_std": -0.467474281511242, "nauc_ndcg_at_10_diff1": 0.5800090322494245, "nauc_ndcg_at_10_max": 0.24203689898525843, "nauc_ndcg_at_10_std": -0.5118177090163054, "nauc_ndcg_at_1_diff1": 0.6110833815898389, "nauc_ndcg_at_1_max": 0.1908294474342627, "nauc_ndcg_at_1_std": -0.35314996932177417, "nauc_ndcg_at_20_diff1": 0.598047860353563, "nauc_ndcg_at_20_max": 0.25996203653945654, "nauc_ndcg_at_20_std": -0.5029431420978402, "nauc_ndcg_at_3_diff1": 0.5896443334804378, "nauc_ndcg_at_3_max": 0.2439244231908955, "nauc_ndcg_at_3_std": -0.4669491517657949, "nauc_ndcg_at_5_diff1": 0.5816884009461497, "nauc_ndcg_at_5_max": 0.2509765225930184, "nauc_ndcg_at_5_std": -0.5086643597687968, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.47965320075378115, "nauc_precision_at_10_max": 0.40503318457167936, "nauc_precision_at_10_std": -1.143569297810643, "nauc_precision_at_1_diff1": 0.6110833815898389, "nauc_precision_at_1_max": 0.1908294474342627, "nauc_precision_at_1_std": -0.35314996932177417, "nauc_precision_at_20_diff1": 0.7008965455883202, "nauc_precision_at_20_max": 0.7183536566232184, "nauc_precision_at_20_std": -1.2588569287895695, "nauc_precision_at_3_diff1": 0.5731811728893021, "nauc_precision_at_3_max": 0.2938957496013745, "nauc_precision_at_3_std": -0.5982375555469349, "nauc_precision_at_5_diff1": 0.5358095787755137, "nauc_precision_at_5_max": 0.35334774502959576, "nauc_precision_at_5_std": -0.8077434531545805, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.4796532007537782, "nauc_recall_at_10_max": 0.40503318457167875, "nauc_recall_at_10_std": -1.1435692978106466, "nauc_recall_at_1_diff1": 0.6110833815898389, "nauc_recall_at_1_max": 0.1908294474342627, "nauc_recall_at_1_std": -0.35314996932177417, "nauc_recall_at_20_diff1": 0.700896545588321, "nauc_recall_at_20_max": 0.7183536566232177, "nauc_recall_at_20_std": -1.2588569287895657, "nauc_recall_at_3_diff1": 0.5731811728893031, "nauc_recall_at_3_max": 0.2938957496013749, "nauc_recall_at_3_std": -0.5982375555469336, "nauc_recall_at_5_diff1": 0.5358095787755134, "nauc_recall_at_5_max": 0.3533477450295955, "nauc_recall_at_5_std": -0.8077434531545817, "ndcg_at_1": 0.59649, "ndcg_at_10": 0.77108, "ndcg_at_100": 0.78411, "ndcg_at_1000": 0.78411, "ndcg_at_20": 0.77556, "ndcg_at_3": 0.72452, "ndcg_at_5": 0.7468, "precision_at_1": 0.59649, "precision_at_10": 0.09386, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04781, "precision_at_3": 0.27193, "precision_at_5": 0.17368, "recall_at_1": 0.59649, "recall_at_10": 0.9386, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.95614, "recall_at_3": 0.81579, "recall_at_5": 0.86842 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 9.514682531356812, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.8796, "map_at_1": 0.80333, "map_at_10": 0.85849, "map_at_100": 0.86055, "map_at_1000": 0.86059, "map_at_20": 0.85972, "map_at_3": 0.85074, "map_at_5": 0.85674, "mrr_at_1": 0.8033333333333333, "mrr_at_10": 0.8584872134038802, "mrr_at_100": 0.8605501771786044, "mrr_at_1000": 0.860594347604656, "mrr_at_20": 0.859719015958051, "mrr_at_3": 0.8507407407407409, "mrr_at_5": 0.8567407407407408, "nauc_map_at_1000_diff1": 0.8651361278493194, "nauc_map_at_1000_max": 0.7418586175399554, "nauc_map_at_1000_std": 0.013734735612603912, "nauc_map_at_100_diff1": 0.8651555699121707, "nauc_map_at_100_max": 0.7419304322492464, "nauc_map_at_100_std": 0.013970732162235726, "nauc_map_at_10_diff1": 0.8642127452490442, "nauc_map_at_10_max": 0.7423022391957348, "nauc_map_at_10_std": 0.013077755502568747, "nauc_map_at_1_diff1": 0.8783845490368515, "nauc_map_at_1_max": 0.7144604734281541, "nauc_map_at_1_std": -0.03075479037358894, "nauc_map_at_20_diff1": 0.8653809000316373, "nauc_map_at_20_max": 0.7427858268462051, "nauc_map_at_20_std": 0.015015758160884355, "nauc_map_at_3_diff1": 0.8595602194780724, "nauc_map_at_3_max": 0.7380410878857894, "nauc_map_at_3_std": 0.0037690970412170128, "nauc_map_at_5_diff1": 0.8639913582671778, "nauc_map_at_5_max": 0.7441652104318379, "nauc_map_at_5_std": 0.019813631471681307, "nauc_mrr_at_1000_diff1": 0.8651361278493194, "nauc_mrr_at_1000_max": 0.7418586175399554, "nauc_mrr_at_1000_std": 0.013734735612603912, "nauc_mrr_at_100_diff1": 0.8651555699121707, "nauc_mrr_at_100_max": 0.7419304322492464, "nauc_mrr_at_100_std": 0.013970732162235726, "nauc_mrr_at_10_diff1": 0.8642127452490442, "nauc_mrr_at_10_max": 0.7423022391957348, "nauc_mrr_at_10_std": 0.013077755502568747, "nauc_mrr_at_1_diff1": 0.8783845490368515, "nauc_mrr_at_1_max": 0.7144604734281541, "nauc_mrr_at_1_std": -0.03075479037358894, "nauc_mrr_at_20_diff1": 0.8653809000316373, "nauc_mrr_at_20_max": 0.7427858268462051, "nauc_mrr_at_20_std": 0.015015758160884355, "nauc_mrr_at_3_diff1": 0.8595602194780724, "nauc_mrr_at_3_max": 0.7380410878857894, "nauc_mrr_at_3_std": 0.0037690970412170128, "nauc_mrr_at_5_diff1": 0.8639913582671778, "nauc_mrr_at_5_max": 0.7441652104318379, "nauc_mrr_at_5_std": 0.019813631471681307, "nauc_ndcg_at_1000_diff1": 0.8640736382725115, "nauc_ndcg_at_1000_max": 0.7476194763678936, "nauc_ndcg_at_1000_std": 0.023164955045697332, "nauc_ndcg_at_100_diff1": 0.8646708926516866, "nauc_ndcg_at_100_max": 0.7497271398718169, "nauc_ndcg_at_100_std": 0.030462267429729645, "nauc_ndcg_at_10_diff1": 0.8601464938487059, "nauc_ndcg_at_10_max": 0.7537308843825129, "nauc_ndcg_at_10_std": 0.02743877031929201, "nauc_ndcg_at_1_diff1": 0.8783845490368515, "nauc_ndcg_at_1_max": 0.7144604734281541, "nauc_ndcg_at_1_std": -0.03075479037358894, "nauc_ndcg_at_20_diff1": 0.8660678949081592, "nauc_ndcg_at_20_max": 0.756589310017133, "nauc_ndcg_at_20_std": 0.03472573440205783, "nauc_ndcg_at_3_diff1": 0.8518734277875382, "nauc_ndcg_at_3_max": 0.7460751997142202, "nauc_ndcg_at_3_std": 0.01431613860324495, "nauc_ndcg_at_5_diff1": 0.8600816216068748, "nauc_ndcg_at_5_max": 0.7588431787096619, "nauc_ndcg_at_5_std": 0.04660310694125582, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9253034547152017, "nauc_precision_at_100_max": 0.962651727357627, "nauc_precision_at_100_std": 0.7736427904495127, "nauc_precision_at_10_diff1": 0.834376888010103, "nauc_precision_at_10_max": 0.8285091814503543, "nauc_precision_at_10_std": 0.1116237344611039, "nauc_precision_at_1_diff1": 0.8783845490368515, "nauc_precision_at_1_max": 0.7144604734281541, "nauc_precision_at_1_std": -0.03075479037358894, "nauc_precision_at_20_diff1": 0.8951580632252828, "nauc_precision_at_20_max": 0.8967587034813862, "nauc_precision_at_20_std": 0.22232893157262623, "nauc_precision_at_3_diff1": 0.8191105899724272, "nauc_precision_at_3_max": 0.7795691920178943, "nauc_precision_at_3_std": 0.05760211061168791, "nauc_precision_at_5_diff1": 0.8404324692840104, "nauc_precision_at_5_max": 0.8426852222370399, "nauc_precision_at_5_std": 0.20104338031508923, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9253034547152172, "nauc_recall_at_100_max": 0.962651727357617, "nauc_recall_at_100_std": 0.7736427904495109, "nauc_recall_at_10_diff1": 0.8343768880101047, "nauc_recall_at_10_max": 0.8285091814503583, "nauc_recall_at_10_std": 0.1116237344611009, "nauc_recall_at_1_diff1": 0.8783845490368515, "nauc_recall_at_1_max": 0.7144604734281541, "nauc_recall_at_1_std": -0.03075479037358894, "nauc_recall_at_20_diff1": 0.8951580632252891, "nauc_recall_at_20_max": 0.8967587034813924, "nauc_recall_at_20_std": 0.22232893157262798, "nauc_recall_at_3_diff1": 0.8191105899724227, "nauc_recall_at_3_max": 0.7795691920178919, "nauc_recall_at_3_std": 0.057602110611686456, "nauc_recall_at_5_diff1": 0.8404324692840107, "nauc_recall_at_5_max": 0.8426852222370445, "nauc_recall_at_5_std": 0.2010433803150886, "ndcg_at_1": 0.80333, "ndcg_at_10": 0.8796, "ndcg_at_100": 0.88994, "ndcg_at_1000": 0.89097, "ndcg_at_20": 0.88409, "ndcg_at_3": 0.86465, "ndcg_at_5": 0.87532, "precision_at_1": 0.80333, "precision_at_10": 0.09433, "precision_at_100": 0.00992, "precision_at_1000": 0.001, "precision_at_20": 0.04806, "precision_at_3": 0.30148, "precision_at_5": 0.186, "recall_at_1": 0.80333, "recall_at_10": 0.94333, "recall_at_100": 0.99222, "recall_at_1000": 1.0, "recall_at_20": 0.96111, "recall_at_3": 0.90444, "recall_at_5": 0.93 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.88062, "map_at_1": 0.79778, "map_at_10": 0.85643, "map_at_100": 0.85821, "map_at_1000": 0.85825, "map_at_20": 0.85771, "map_at_3": 0.84815, "map_at_5": 0.85343, "mrr_at_1": 0.7977777777777778, "mrr_at_10": 0.856428130511464, "mrr_at_100": 0.8582147049962549, "mrr_at_1000": 0.8582509822841501, "mrr_at_20": 0.8577066769468734, "mrr_at_3": 0.8481481481481484, "mrr_at_5": 0.8534259259259264, "nauc_map_at_1000_diff1": 0.829546134330904, "nauc_map_at_1000_max": 0.6325855821857873, "nauc_map_at_1000_std": -0.08592721118228824, "nauc_map_at_100_diff1": 0.8295521080594748, "nauc_map_at_100_max": 0.6326847487835539, "nauc_map_at_100_std": -0.0856605907756247, "nauc_map_at_10_diff1": 0.8288997350230023, "nauc_map_at_10_max": 0.6333210130677666, "nauc_map_at_10_std": -0.08382181759549567, "nauc_map_at_1_diff1": 0.8469104756035148, "nauc_map_at_1_max": 0.6136072240228472, "nauc_map_at_1_std": -0.10831081787516371, "nauc_map_at_20_diff1": 0.8291637416330593, "nauc_map_at_20_max": 0.6330502230134387, "nauc_map_at_20_std": -0.08618019653766029, "nauc_map_at_3_diff1": 0.827284260162407, "nauc_map_at_3_max": 0.6325155136406339, "nauc_map_at_3_std": -0.08899414161844547, "nauc_map_at_5_diff1": 0.8268611228256288, "nauc_map_at_5_max": 0.6324112783164382, "nauc_map_at_5_std": -0.08103465516566621, "nauc_mrr_at_1000_diff1": 0.829546134330904, "nauc_mrr_at_1000_max": 0.6325855821857873, "nauc_mrr_at_1000_std": -0.08592721118228824, "nauc_mrr_at_100_diff1": 0.8295521080594748, "nauc_mrr_at_100_max": 0.6326847487835539, "nauc_mrr_at_100_std": -0.0856605907756247, "nauc_mrr_at_10_diff1": 0.8288997350230023, "nauc_mrr_at_10_max": 0.6333210130677666, "nauc_mrr_at_10_std": -0.08382181759549567, "nauc_mrr_at_1_diff1": 0.8469104756035148, "nauc_mrr_at_1_max": 0.6136072240228472, "nauc_mrr_at_1_std": -0.10831081787516371, "nauc_mrr_at_20_diff1": 0.8291637416330593, "nauc_mrr_at_20_max": 0.6330502230134387, "nauc_mrr_at_20_std": -0.08618019653766029, "nauc_mrr_at_3_diff1": 0.827284260162407, "nauc_mrr_at_3_max": 0.6325155136406339, "nauc_mrr_at_3_std": -0.08899414161844547, "nauc_mrr_at_5_diff1": 0.8268611228256288, "nauc_mrr_at_5_max": 0.6324112783164382, "nauc_mrr_at_5_std": -0.08103465516566621, "nauc_ndcg_at_1000_diff1": 0.827275101042018, "nauc_ndcg_at_1000_max": 0.6368223960440613, "nauc_ndcg_at_1000_std": -0.0795689943247917, "nauc_ndcg_at_100_diff1": 0.8275142613783739, "nauc_ndcg_at_100_max": 0.6394490798501035, "nauc_ndcg_at_100_std": -0.07252899389683454, "nauc_ndcg_at_10_diff1": 0.8236698658167095, "nauc_ndcg_at_10_max": 0.643210997098357, "nauc_ndcg_at_10_std": -0.0682725779951663, "nauc_ndcg_at_1_diff1": 0.8469104756035148, "nauc_ndcg_at_1_max": 0.6136072240228472, "nauc_ndcg_at_1_std": -0.10831081787516371, "nauc_ndcg_at_20_diff1": 0.8243330368189321, "nauc_ndcg_at_20_max": 0.6425555639374455, "nauc_ndcg_at_20_std": -0.07771461296268094, "nauc_ndcg_at_3_diff1": 0.8196030620594603, "nauc_ndcg_at_3_max": 0.6396350400369007, "nauc_ndcg_at_3_std": -0.07909721619527248, "nauc_ndcg_at_5_diff1": 0.8181558557674538, "nauc_ndcg_at_5_max": 0.6395382719190128, "nauc_ndcg_at_5_std": -0.06251199626346327, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8627450980391947, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.8921568627450869, "nauc_precision_at_10_diff1": 0.7854751656760247, "nauc_precision_at_10_max": 0.7316731570677019, "nauc_precision_at_10_std": 0.0690601443829509, "nauc_precision_at_1_diff1": 0.8469104756035148, "nauc_precision_at_1_max": 0.6136072240228472, "nauc_precision_at_1_std": -0.10831081787516371, "nauc_precision_at_20_diff1": 0.7666705571117243, "nauc_precision_at_20_max": 0.784702769996881, "nauc_precision_at_20_std": 0.017098506069089116, "nauc_precision_at_3_diff1": 0.7846249610955512, "nauc_precision_at_3_max": 0.6719057993567797, "nauc_precision_at_3_std": -0.032783483763876776, "nauc_precision_at_5_diff1": 0.7670095360548549, "nauc_precision_at_5_max": 0.6802589888414331, "nauc_precision_at_5_std": 0.0515911282545789, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8627450980392323, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.892156862745096, "nauc_recall_at_10_diff1": 0.7854751656760276, "nauc_recall_at_10_max": 0.7316731570677064, "nauc_recall_at_10_std": 0.0690601443829571, "nauc_recall_at_1_diff1": 0.8469104756035148, "nauc_recall_at_1_max": 0.6136072240228472, "nauc_recall_at_1_std": -0.10831081787516371, "nauc_recall_at_20_diff1": 0.7666705571117347, "nauc_recall_at_20_max": 0.7847027699968866, "nauc_recall_at_20_std": 0.017098506069086708, "nauc_recall_at_3_diff1": 0.7846249610955486, "nauc_recall_at_3_max": 0.6719057993567795, "nauc_recall_at_3_std": -0.03278348376387673, "nauc_recall_at_5_diff1": 0.7670095360548592, "nauc_recall_at_5_max": 0.6802589888414381, "nauc_recall_at_5_std": 0.05159112825458121, "ndcg_at_1": 0.79778, "ndcg_at_10": 0.88062, "ndcg_at_100": 0.88921, "ndcg_at_1000": 0.88998, "ndcg_at_20": 0.88536, "ndcg_at_3": 0.86407, "ndcg_at_5": 0.8734, "precision_at_1": 0.79778, "precision_at_10": 0.09544, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "precision_at_20": 0.04867, "precision_at_3": 0.30333, "precision_at_5": 0.18644, "recall_at_1": 0.79778, "recall_at_10": 0.95444, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "recall_at_20": 0.97333, "recall_at_3": 0.91, "recall_at_5": 0.93222 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.85657, "map_at_1": 0.75889, "map_at_10": 0.82638, "map_at_100": 0.82792, "map_at_1000": 0.82798, "map_at_20": 0.82724, "map_at_3": 0.81167, "map_at_5": 0.82194, "mrr_at_1": 0.7588888888888888, "mrr_at_10": 0.8263791887125224, "mrr_at_100": 0.8279172664600791, "mrr_at_1000": 0.8279781231588634, "mrr_at_20": 0.8272371670392827, "mrr_at_3": 0.8116666666666666, "mrr_at_5": 0.8219444444444447, "nauc_map_at_1000_diff1": 0.8080662924334656, "nauc_map_at_1000_max": 0.7137610309322288, "nauc_map_at_1000_std": 0.03889981948933412, "nauc_map_at_100_diff1": 0.8080206640656972, "nauc_map_at_100_max": 0.713844142428308, "nauc_map_at_100_std": 0.03925475947340508, "nauc_map_at_10_diff1": 0.8085438358710668, "nauc_map_at_10_max": 0.7147965670336995, "nauc_map_at_10_std": 0.041601046039760524, "nauc_map_at_1_diff1": 0.8148941927542931, "nauc_map_at_1_max": 0.6885576833277857, "nauc_map_at_1_std": -0.005981815090553283, "nauc_map_at_20_diff1": 0.8079968971636359, "nauc_map_at_20_max": 0.7144601462940647, "nauc_map_at_20_std": 0.04099096241352047, "nauc_map_at_3_diff1": 0.8077983411024191, "nauc_map_at_3_max": 0.717892878661459, "nauc_map_at_3_std": 0.03975692189966652, "nauc_map_at_5_diff1": 0.8063046003130837, "nauc_map_at_5_max": 0.7106092932047338, "nauc_map_at_5_std": 0.03282925356427449, "nauc_mrr_at_1000_diff1": 0.8080662924334656, "nauc_mrr_at_1000_max": 0.7137610309322288, "nauc_mrr_at_1000_std": 0.03889981948933412, "nauc_mrr_at_100_diff1": 0.8080206640656972, "nauc_mrr_at_100_max": 0.713844142428308, "nauc_mrr_at_100_std": 0.03925475947340508, "nauc_mrr_at_10_diff1": 0.8085438358710668, "nauc_mrr_at_10_max": 0.7147965670336995, "nauc_mrr_at_10_std": 0.041601046039760524, "nauc_mrr_at_1_diff1": 0.8148941927542931, "nauc_mrr_at_1_max": 0.6885576833277857, "nauc_mrr_at_1_std": -0.005981815090553283, "nauc_mrr_at_20_diff1": 0.8079968971636359, "nauc_mrr_at_20_max": 0.7144601462940647, "nauc_mrr_at_20_std": 0.04099096241352047, "nauc_mrr_at_3_diff1": 0.8077983411024191, "nauc_mrr_at_3_max": 0.717892878661459, "nauc_mrr_at_3_std": 0.03975692189966652, "nauc_mrr_at_5_diff1": 0.8063046003130837, "nauc_mrr_at_5_max": 0.7106092932047338, "nauc_mrr_at_5_std": 0.03282925356427449, "nauc_ndcg_at_1000_diff1": 0.8071936485236224, "nauc_ndcg_at_1000_max": 0.7194181195131821, "nauc_ndcg_at_1000_std": 0.05342224213986593, "nauc_ndcg_at_100_diff1": 0.8061779133250535, "nauc_ndcg_at_100_max": 0.7219947254747765, "nauc_ndcg_at_100_std": 0.06398427568254675, "nauc_ndcg_at_10_diff1": 0.8084808695754142, "nauc_ndcg_at_10_max": 0.7284930166428097, "nauc_ndcg_at_10_std": 0.07720960494895679, "nauc_ndcg_at_1_diff1": 0.8148941927542931, "nauc_ndcg_at_1_max": 0.6885576833277857, "nauc_ndcg_at_1_std": -0.005981815090553283, "nauc_ndcg_at_20_diff1": 0.8062310856840769, "nauc_ndcg_at_20_max": 0.7277722658852183, "nauc_ndcg_at_20_std": 0.07684336868935904, "nauc_ndcg_at_3_diff1": 0.8059262738497057, "nauc_ndcg_at_3_max": 0.7294359748625586, "nauc_ndcg_at_3_std": 0.06181714689020046, "nauc_ndcg_at_5_diff1": 0.8028769320851481, "nauc_ndcg_at_5_max": 0.7164772902430775, "nauc_ndcg_at_5_std": 0.05087262789435781, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.7288048552754708, "nauc_precision_at_100_max": 0.9423436041083034, "nauc_precision_at_100_std": 0.9554154995331525, "nauc_precision_at_10_diff1": 0.813175640626619, "nauc_precision_at_10_max": 0.8620915032679718, "nauc_precision_at_10_std": 0.4347235190372405, "nauc_precision_at_1_diff1": 0.8148941927542931, "nauc_precision_at_1_max": 0.6885576833277857, "nauc_precision_at_1_std": -0.005981815090553283, "nauc_precision_at_20_diff1": 0.7854122040973202, "nauc_precision_at_20_max": 0.8985280386664453, "nauc_precision_at_20_std": 0.5554457076948398, "nauc_precision_at_3_diff1": 0.7987855205754054, "nauc_precision_at_3_max": 0.7759908703488474, "nauc_precision_at_3_std": 0.15259740259740234, "nauc_precision_at_5_diff1": 0.7832790773967219, "nauc_precision_at_5_max": 0.7469086733792593, "nauc_precision_at_5_std": 0.15426350720468127, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.7288048552754446, "nauc_recall_at_100_max": 0.9423436041083036, "nauc_recall_at_100_std": 0.9554154995331435, "nauc_recall_at_10_diff1": 0.8131756406266185, "nauc_recall_at_10_max": 0.8620915032679746, "nauc_recall_at_10_std": 0.4347235190372419, "nauc_recall_at_1_diff1": 0.8148941927542931, "nauc_recall_at_1_max": 0.6885576833277857, "nauc_recall_at_1_std": -0.005981815090553283, "nauc_recall_at_20_diff1": 0.7854122040973217, "nauc_recall_at_20_max": 0.8985280386664473, "nauc_recall_at_20_std": 0.5554457076948423, "nauc_recall_at_3_diff1": 0.7987855205754045, "nauc_recall_at_3_max": 0.7759908703488476, "nauc_recall_at_3_std": 0.15259740259740326, "nauc_recall_at_5_diff1": 0.7832790773967254, "nauc_recall_at_5_max": 0.7469086733792616, "nauc_recall_at_5_std": 0.15426350720468285, "ndcg_at_1": 0.75889, "ndcg_at_10": 0.85657, "ndcg_at_100": 0.86461, "ndcg_at_1000": 0.86607, "ndcg_at_20": 0.85967, "ndcg_at_3": 0.8276, "ndcg_at_5": 0.84601, "precision_at_1": 0.75889, "precision_at_10": 0.095, "precision_at_100": 0.00989, "precision_at_1000": 0.001, "precision_at_20": 0.04811, "precision_at_3": 0.29111, "precision_at_5": 0.18356, "recall_at_1": 0.75889, "recall_at_10": 0.95, "recall_at_100": 0.98889, "recall_at_1000": 1.0, "recall_at_20": 0.96222, "recall_at_3": 0.87333, "recall_at_5": 0.91778 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 47.95084285736084, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.75121, "map_at_1": 0.45135, "map_at_10": 0.69709, "map_at_100": 0.70379, "map_at_1000": 0.7042, "map_at_20": 0.70167, "map_at_3": 0.65496, "map_at_5": 0.68422, "mrr_at_1": 0.7010135135135135, "mrr_at_10": 0.7737069658944656, "mrr_at_100": 0.7755940101379746, "mrr_at_1000": 0.775762024387044, "mrr_at_20": 0.7747121401629913, "mrr_at_3": 0.764921171171171, "mrr_at_5": 0.7705799549549547, "nauc_map_at_1000_diff1": 0.5519197145137611, "nauc_map_at_1000_max": 0.6067402935995151, "nauc_map_at_1000_std": 0.1594828873545819, "nauc_map_at_100_diff1": 0.552117983876564, "nauc_map_at_100_max": 0.6071358257277069, "nauc_map_at_100_std": 0.15992886906044776, "nauc_map_at_10_diff1": 0.5482356659883542, "nauc_map_at_10_max": 0.6036640161616742, "nauc_map_at_10_std": 0.14948304159183426, "nauc_map_at_1_diff1": 0.5834434025577803, "nauc_map_at_1_max": 0.299451623991129, "nauc_map_at_1_std": -0.022953282119810403, "nauc_map_at_20_diff1": 0.551019680772447, "nauc_map_at_20_max": 0.6074107021103508, "nauc_map_at_20_std": 0.15853657489864967, "nauc_map_at_3_diff1": 0.5248726081949865, "nauc_map_at_3_max": 0.5594477813787555, "nauc_map_at_3_std": 0.08416007387108534, "nauc_map_at_5_diff1": 0.5373092416579366, "nauc_map_at_5_max": 0.5899555474257632, "nauc_map_at_5_std": 0.12732982461525347, "nauc_mrr_at_1000_diff1": 0.6630500026079771, "nauc_mrr_at_1000_max": 0.6736685608978698, "nauc_mrr_at_1000_std": 0.2320878485491545, "nauc_mrr_at_100_diff1": 0.663057226112171, "nauc_mrr_at_100_max": 0.6738925445530085, "nauc_mrr_at_100_std": 0.2325917862628659, "nauc_mrr_at_10_diff1": 0.6636967780082375, "nauc_mrr_at_10_max": 0.6762588126119293, "nauc_mrr_at_10_std": 0.23291273028624304, "nauc_mrr_at_1_diff1": 0.661500115820817, "nauc_mrr_at_1_max": 0.5770263549598156, "nauc_mrr_at_1_std": 0.1313742842575525, "nauc_mrr_at_20_diff1": 0.662962026964006, "nauc_mrr_at_20_max": 0.6747824530605779, "nauc_mrr_at_20_std": 0.23226267353927715, "nauc_mrr_at_3_diff1": 0.658707138510419, "nauc_mrr_at_3_max": 0.6720413722313505, "nauc_mrr_at_3_std": 0.214088474893056, "nauc_mrr_at_5_diff1": 0.6603441608824964, "nauc_mrr_at_5_max": 0.6736745604944314, "nauc_mrr_at_5_std": 0.22846499218561858, "nauc_ndcg_at_1000_diff1": 0.5907551369979055, "nauc_ndcg_at_1000_max": 0.6615877024334198, "nauc_ndcg_at_1000_std": 0.2393015359142359, "nauc_ndcg_at_100_diff1": 0.5937442728212519, "nauc_ndcg_at_100_max": 0.670641015014622, "nauc_ndcg_at_100_std": 0.25487773336698044, "nauc_ndcg_at_10_diff1": 0.5831817543877047, "nauc_ndcg_at_10_max": 0.6643418532861225, "nauc_ndcg_at_10_std": 0.22326362761367885, "nauc_ndcg_at_1_diff1": 0.661500115820817, "nauc_ndcg_at_1_max": 0.5770263549598156, "nauc_ndcg_at_1_std": 0.1313742842575525, "nauc_ndcg_at_20_diff1": 0.5899026096075843, "nauc_ndcg_at_20_max": 0.673272767215356, "nauc_ndcg_at_20_std": 0.24635138529614536, "nauc_ndcg_at_3_diff1": 0.5479702226955419, "nauc_ndcg_at_3_max": 0.6059582416432887, "nauc_ndcg_at_3_std": 0.12743795996465662, "nauc_ndcg_at_5_diff1": 0.5628160195267607, "nauc_ndcg_at_5_max": 0.6411398480758634, "nauc_ndcg_at_5_std": 0.18438351907666847, "nauc_precision_at_1000_diff1": -0.2004685252782137, "nauc_precision_at_1000_max": 0.07063855451998648, "nauc_precision_at_1000_std": 0.23065877466350113, "nauc_precision_at_100_diff1": -0.07861027277114246, "nauc_precision_at_100_max": 0.21703569000810188, "nauc_precision_at_100_std": 0.31518785228088636, "nauc_precision_at_10_diff1": 0.012468448328005923, "nauc_precision_at_10_max": 0.35106876700614753, "nauc_precision_at_10_std": 0.30589978246233046, "nauc_precision_at_1_diff1": 0.661500115820817, "nauc_precision_at_1_max": 0.5770263549598156, "nauc_precision_at_1_std": 0.1313742842575525, "nauc_precision_at_20_diff1": -0.015991023617958375, "nauc_precision_at_20_max": 0.31120661076351014, "nauc_precision_at_20_std": 0.32862281412296374, "nauc_precision_at_3_diff1": 0.10652022166353019, "nauc_precision_at_3_max": 0.4643801024890446, "nauc_precision_at_3_std": 0.21573365606067008, "nauc_precision_at_5_diff1": 0.04752964400629679, "nauc_precision_at_5_max": 0.3993637294469268, "nauc_precision_at_5_std": 0.26849236828347134, "nauc_recall_at_1000_diff1": 0.6310537710228703, "nauc_recall_at_1000_max": 0.9077919776775288, "nauc_recall_at_1000_std": 0.7655930243673666, "nauc_recall_at_100_diff1": 0.6368032349603382, "nauc_recall_at_100_max": 0.8803880729883224, "nauc_recall_at_100_std": 0.6979867070823739, "nauc_recall_at_10_diff1": 0.5650349723481966, "nauc_recall_at_10_max": 0.7654270251851792, "nauc_recall_at_10_std": 0.3773682882526592, "nauc_recall_at_1_diff1": 0.5834434025577803, "nauc_recall_at_1_max": 0.299451623991129, "nauc_recall_at_1_std": -0.022953282119810403, "nauc_recall_at_20_diff1": 0.5996092880814857, "nauc_recall_at_20_max": 0.829226851926464, "nauc_recall_at_20_std": 0.5101268399413494, "nauc_recall_at_3_diff1": 0.49205076291344274, "nauc_recall_at_3_max": 0.6109051461139894, "nauc_recall_at_3_std": 0.12072253500359617, "nauc_recall_at_5_diff1": 0.5088705234664951, "nauc_recall_at_5_max": 0.6817156339665192, "nauc_recall_at_5_std": 0.24410069255370517, "ndcg_at_1": 0.70101, "ndcg_at_10": 0.75121, "ndcg_at_100": 0.77256, "ndcg_at_1000": 0.78115, "ndcg_at_20": 0.76145, "ndcg_at_3": 0.70056, "ndcg_at_5": 0.73113, "precision_at_1": 0.70101, "precision_at_10": 0.15152, "precision_at_100": 0.01698, "precision_at_1000": 0.00183, "precision_at_20": 0.07956, "precision_at_3": 0.41667, "precision_at_5": 0.27939, "recall_at_1": 0.45135, "recall_at_10": 0.82298, "recall_at_100": 0.90386, "recall_at_1000": 0.9587, "recall_at_20": 0.85468, "recall_at_3": 0.71127, "recall_at_5": 0.77506 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 4725.387804031372, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.62697, "map_at_1": 0.38205, "map_at_10": 0.54289, "map_at_100": 0.56555, "map_at_1000": 0.56697, "map_at_20": 0.55681, "map_at_3": 0.48344, "map_at_5": 0.51568, "mrr_at_1": 0.6103286384976526, "mrr_at_10": 0.7039291303375809, "mrr_at_100": 0.7082462092431728, "mrr_at_1000": 0.7084151998358826, "mrr_at_20": 0.7072567444280695, "mrr_at_3": 0.6862284820031298, "mrr_at_5": 0.69679186228482, "nauc_map_at_1000_diff1": 0.3880663627782523, "nauc_map_at_1000_max": 0.26875837358990384, "nauc_map_at_1000_std": -0.02915658141808151, "nauc_map_at_100_diff1": 0.3877255300606842, "nauc_map_at_100_max": 0.2676573499357481, "nauc_map_at_100_std": -0.03003797795589652, "nauc_map_at_10_diff1": 0.3967106291403696, "nauc_map_at_10_max": 0.2507505837032776, "nauc_map_at_10_std": -0.051781293185286445, "nauc_map_at_1_diff1": 0.41809526232663874, "nauc_map_at_1_max": 0.08779534853304669, "nauc_map_at_1_std": -0.17386942942712602, "nauc_map_at_20_diff1": 0.3915764765548958, "nauc_map_at_20_max": 0.25739603266273464, "nauc_map_at_20_std": -0.041442559021572345, "nauc_map_at_3_diff1": 0.42231719725329264, "nauc_map_at_3_max": 0.18371697143417534, "nauc_map_at_3_std": -0.12726929425612019, "nauc_map_at_5_diff1": 0.398872122466833, "nauc_map_at_5_max": 0.2183229034565644, "nauc_map_at_5_std": -0.08595999951698406, "nauc_mrr_at_1000_diff1": 0.4088929543341095, "nauc_mrr_at_1000_max": 0.4960270065168594, "nauc_mrr_at_1000_std": 0.1986308055671587, "nauc_mrr_at_100_diff1": 0.40884542440189453, "nauc_mrr_at_100_max": 0.496298929979252, "nauc_mrr_at_100_std": 0.19901507136436228, "nauc_mrr_at_10_diff1": 0.4095868941932426, "nauc_mrr_at_10_max": 0.5016475476247045, "nauc_mrr_at_10_std": 0.20295922598033728, "nauc_mrr_at_1_diff1": 0.42845609791848677, "nauc_mrr_at_1_max": 0.40065137930038525, "nauc_mrr_at_1_std": 0.10762285365210995, "nauc_mrr_at_20_diff1": 0.4083082899116937, "nauc_mrr_at_20_max": 0.49640091643714074, "nauc_mrr_at_20_std": 0.20091163621165942, "nauc_mrr_at_3_diff1": 0.4157753000024607, "nauc_mrr_at_3_max": 0.49681140521620853, "nauc_mrr_at_3_std": 0.18299091495164696, "nauc_mrr_at_5_diff1": 0.4024032183255278, "nauc_mrr_at_5_max": 0.4965190104753864, "nauc_mrr_at_5_std": 0.1973793570513383, "nauc_ndcg_at_1000_diff1": 0.37373031518754046, "nauc_ndcg_at_1000_max": 0.37607986329290355, "nauc_ndcg_at_1000_std": 0.0908324185246208, "nauc_ndcg_at_100_diff1": 0.3717491373566511, "nauc_ndcg_at_100_max": 0.37240022954155677, "nauc_ndcg_at_100_std": 0.08618214219538683, "nauc_ndcg_at_10_diff1": 0.39086793566356887, "nauc_ndcg_at_10_max": 0.33466796739784904, "nauc_ndcg_at_10_std": 0.030749916588024073, "nauc_ndcg_at_1_diff1": 0.42845609791848677, "nauc_ndcg_at_1_max": 0.40065137930038525, "nauc_ndcg_at_1_std": 0.10762285365210995, "nauc_ndcg_at_20_diff1": 0.38054633502568497, "nauc_ndcg_at_20_max": 0.3438995134967677, "nauc_ndcg_at_20_std": 0.0562318258312696, "nauc_ndcg_at_3_diff1": 0.399795423235567, "nauc_ndcg_at_3_max": 0.33702985956781256, "nauc_ndcg_at_3_std": 0.011897751562643624, "nauc_ndcg_at_5_diff1": 0.37602006444813374, "nauc_ndcg_at_5_max": 0.32301355137987625, "nauc_ndcg_at_5_std": 0.014210821857346219, "nauc_precision_at_1000_diff1": -0.2325905952276443, "nauc_precision_at_1000_max": 0.26803251700559394, "nauc_precision_at_1000_std": 0.40373084311985863, "nauc_precision_at_100_diff1": -0.22361371198039076, "nauc_precision_at_100_max": 0.30222103314451515, "nauc_precision_at_100_std": 0.42174055990118403, "nauc_precision_at_10_diff1": -0.10782271786242098, "nauc_precision_at_10_max": 0.3906023360764504, "nauc_precision_at_10_std": 0.41343020041282674, "nauc_precision_at_1_diff1": 0.42845609791848677, "nauc_precision_at_1_max": 0.40065137930038525, "nauc_precision_at_1_std": 0.10762285365210995, "nauc_precision_at_20_diff1": -0.16620077076923456, "nauc_precision_at_20_max": 0.3540662038450409, "nauc_precision_at_20_std": 0.4385742655258709, "nauc_precision_at_3_diff1": 0.0643330951436578, "nauc_precision_at_3_max": 0.4708826625517317, "nauc_precision_at_3_std": 0.3150688758870942, "nauc_precision_at_5_diff1": -0.0791402282451768, "nauc_precision_at_5_max": 0.410837502746928, "nauc_precision_at_5_std": 0.3739032437456903, "nauc_recall_at_1000_diff1": 0.1461707557767423, "nauc_recall_at_1000_max": 0.7392773551034885, "nauc_recall_at_1000_std": 0.596389402600282, "nauc_recall_at_100_diff1": 0.2603605715058474, "nauc_recall_at_100_max": 0.4411100758773747, "nauc_recall_at_100_std": 0.20665759469441622, "nauc_recall_at_10_diff1": 0.3496804304290531, "nauc_recall_at_10_max": 0.2969594791669843, "nauc_recall_at_10_std": -0.0024174736235818084, "nauc_recall_at_1_diff1": 0.41809526232663874, "nauc_recall_at_1_max": 0.08779534853304669, "nauc_recall_at_1_std": -0.17386942942712602, "nauc_recall_at_20_diff1": 0.3034341078071222, "nauc_recall_at_20_max": 0.29415428741234795, "nauc_recall_at_20_std": 0.0621874859031154, "nauc_recall_at_3_diff1": 0.4114352782373673, "nauc_recall_at_3_max": 0.184362694523637, "nauc_recall_at_3_std": -0.1315379775711195, "nauc_recall_at_5_diff1": 0.3507401455620195, "nauc_recall_at_5_max": 0.2305916637080133, "nauc_recall_at_5_std": -0.0646045286797751, "ndcg_at_1": 0.61033, "ndcg_at_10": 0.62697, "ndcg_at_100": 0.6861, "ndcg_at_1000": 0.70173, "ndcg_at_20": 0.65736, "ndcg_at_3": 0.58892, "ndcg_at_5": 0.60065, "precision_at_1": 0.61033, "precision_at_10": 0.15681, "precision_at_100": 0.02244, "precision_at_1000": 0.00253, "precision_at_20": 0.09202, "precision_at_3": 0.33959, "precision_at_5": 0.2554, "recall_at_1": 0.38205, "recall_at_10": 0.70019, "recall_at_100": 0.89421, "recall_at_1000": 0.97981, "recall_at_20": 0.78979, "recall_at_3": 0.53982, "recall_at_5": 0.6097 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 4789.847511768341, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.56464, "map_at_1": 0.38836, "map_at_10": 0.50553, "map_at_100": 0.51348, "map_at_1000": 0.51388, "map_at_20": 0.51026, "map_at_3": 0.47816, "map_at_5": 0.4954, "mrr_at_1": 0.42042755344418054, "mrr_at_10": 0.5310343852505374, "mrr_at_100": 0.5367455539072286, "mrr_at_1000": 0.5370753449207969, "mrr_at_20": 0.5347435394689465, "mrr_at_3": 0.5079176563737136, "mrr_at_5": 0.5206254948535236, "nauc_map_at_1000_diff1": 0.25716543859897323, "nauc_map_at_1000_max": 0.28005192170293824, "nauc_map_at_1000_std": 0.03836511543988043, "nauc_map_at_100_diff1": 0.25715155161945596, "nauc_map_at_100_max": 0.2801515462923894, "nauc_map_at_100_std": 0.03847960284606766, "nauc_map_at_10_diff1": 0.25825901715325766, "nauc_map_at_10_max": 0.2774051051549839, "nauc_map_at_10_std": 0.030341349544946543, "nauc_map_at_1_diff1": 0.32573416780172126, "nauc_map_at_1_max": 0.31247776894025886, "nauc_map_at_1_std": 0.031249381404232498, "nauc_map_at_20_diff1": 0.257080817902839, "nauc_map_at_20_max": 0.2782100911069668, "nauc_map_at_20_std": 0.03451940941925907, "nauc_map_at_3_diff1": 0.26845525923224, "nauc_map_at_3_max": 0.2642457306385882, "nauc_map_at_3_std": 0.015335480167255127, "nauc_map_at_5_diff1": 0.25278758474435037, "nauc_map_at_5_max": 0.27504315746221053, "nauc_map_at_5_std": 0.029983842134557192, "nauc_mrr_at_1000_diff1": 0.26088870787930196, "nauc_mrr_at_1000_max": 0.2912226938065716, "nauc_mrr_at_1000_std": 0.04285053316771879, "nauc_mrr_at_100_diff1": 0.2609646228530708, "nauc_mrr_at_100_max": 0.2914631566489875, "nauc_mrr_at_100_std": 0.0430522291485771, "nauc_mrr_at_10_diff1": 0.2609882291729948, "nauc_mrr_at_10_max": 0.28934197519266663, "nauc_mrr_at_10_std": 0.03827684254515211, "nauc_mrr_at_1_diff1": 0.3214764381617425, "nauc_mrr_at_1_max": 0.3109742940946706, "nauc_mrr_at_1_std": 0.03263481415776953, "nauc_mrr_at_20_diff1": 0.2603922334686195, "nauc_mrr_at_20_max": 0.2900484255899885, "nauc_mrr_at_20_std": 0.040662887127286745, "nauc_mrr_at_3_diff1": 0.2654485903005151, "nauc_mrr_at_3_max": 0.2779252830401248, "nauc_mrr_at_3_std": 0.02424182001973642, "nauc_mrr_at_5_diff1": 0.2539868410845816, "nauc_mrr_at_5_max": 0.28505476937588964, "nauc_mrr_at_5_std": 0.03596238739186569, "nauc_ndcg_at_1000_diff1": 0.23557354884221127, "nauc_ndcg_at_1000_max": 0.29032978771135, "nauc_ndcg_at_1000_std": 0.06997467082862376, "nauc_ndcg_at_100_diff1": 0.23571006906147526, "nauc_ndcg_at_100_max": 0.29522278193987794, "nauc_ndcg_at_100_std": 0.07461791164210363, "nauc_ndcg_at_10_diff1": 0.23718143213771353, "nauc_ndcg_at_10_max": 0.27862306219326183, "nauc_ndcg_at_10_std": 0.036243938954022954, "nauc_ndcg_at_1_diff1": 0.3214764381617425, "nauc_ndcg_at_1_max": 0.3109742940946706, "nauc_ndcg_at_1_std": 0.03263481415776953, "nauc_ndcg_at_20_diff1": 0.23384962103391074, "nauc_ndcg_at_20_max": 0.2823622591128196, "nauc_ndcg_at_20_std": 0.05073943905013697, "nauc_ndcg_at_3_diff1": 0.2505995080003967, "nauc_ndcg_at_3_max": 0.2550849070629041, "nauc_ndcg_at_3_std": 0.012143397163197251, "nauc_ndcg_at_5_diff1": 0.22439685641544171, "nauc_ndcg_at_5_max": 0.2716990904719191, "nauc_ndcg_at_5_std": 0.03575325421970202, "nauc_precision_at_1000_diff1": -0.13230947244366867, "nauc_precision_at_1000_max": 0.050305125840942856, "nauc_precision_at_1000_std": 0.1764339714748889, "nauc_precision_at_100_diff1": -0.06222382942833803, "nauc_precision_at_100_max": 0.13842447314423142, "nauc_precision_at_100_std": 0.20705592120058777, "nauc_precision_at_10_diff1": 0.05514361830417003, "nauc_precision_at_10_max": 0.18701739784935376, "nauc_precision_at_10_std": 0.04650331876462832, "nauc_precision_at_1_diff1": 0.3214764381617425, "nauc_precision_at_1_max": 0.3109742940946706, "nauc_precision_at_1_std": 0.03263481415776953, "nauc_precision_at_20_diff1": -0.00617052929258315, "nauc_precision_at_20_max": 0.1565272393710742, "nauc_precision_at_20_std": 0.1100427948727622, "nauc_precision_at_3_diff1": 0.1468202987011345, "nauc_precision_at_3_max": 0.1849822470919322, "nauc_precision_at_3_std": -0.010667626125281129, "nauc_precision_at_5_diff1": 0.05497589349994129, "nauc_precision_at_5_max": 0.1879791183562475, "nauc_precision_at_5_std": 0.0444158747825215, "nauc_recall_at_1000_diff1": -0.03852211432611442, "nauc_recall_at_1000_max": 0.41798503899896616, "nauc_recall_at_1000_std": 0.6213555554385645, "nauc_recall_at_100_diff1": 0.10092135478885442, "nauc_recall_at_100_max": 0.4381086975662203, "nauc_recall_at_100_std": 0.4171683707601491, "nauc_recall_at_10_diff1": 0.15252772800619863, "nauc_recall_at_10_max": 0.26507602202914265, "nauc_recall_at_10_std": 0.045423224965121856, "nauc_recall_at_1_diff1": 0.32573416780172126, "nauc_recall_at_1_max": 0.31247776894025886, "nauc_recall_at_1_std": 0.031249381404232498, "nauc_recall_at_20_diff1": 0.13064148113752028, "nauc_recall_at_20_max": 0.2901054578717061, "nauc_recall_at_20_std": 0.12413182063867068, "nauc_recall_at_3_diff1": 0.19680821832881057, "nauc_recall_at_3_max": 0.20838567453412998, "nauc_recall_at_3_std": -0.00682971262625349, "nauc_recall_at_5_diff1": 0.12554959294600906, "nauc_recall_at_5_max": 0.24624746390881122, "nauc_recall_at_5_std": 0.04863875266602324, "ndcg_at_1": 0.42043, "ndcg_at_10": 0.56464, "ndcg_at_100": 0.59861, "ndcg_at_1000": 0.60804, "ndcg_at_20": 0.57975, "ndcg_at_3": 0.51325, "ndcg_at_5": 0.54051, "precision_at_1": 0.42043, "precision_at_10": 0.08076, "precision_at_100": 0.01007, "precision_at_1000": 0.00109, "precision_at_20": 0.04418, "precision_at_3": 0.21615, "precision_at_5": 0.14537, "recall_at_1": 0.38836, "recall_at_10": 0.71971, "recall_at_100": 0.86659, "recall_at_1000": 0.93666, "recall_at_20": 0.77395, "recall_at_3": 0.58591, "recall_at_5": 0.64766 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 4641.1542456150055, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.48402, "map_at_1": 0.36, "map_at_10": 0.44135, "map_at_100": 0.45013, "map_at_1000": 0.45073, "map_at_20": 0.44591, "map_at_3": 0.42167, "map_at_5": 0.42967, "mrr_at_1": 0.36, "mrr_at_10": 0.4413511904761906, "mrr_at_100": 0.450129543381387, "mrr_at_1000": 0.45073324355439376, "mrr_at_20": 0.44590807700211743, "mrr_at_3": 0.4216666666666667, "mrr_at_5": 0.4296666666666667, "nauc_map_at_1000_diff1": 0.653999576097392, "nauc_map_at_1000_max": 0.5915981156693542, "nauc_map_at_1000_std": 0.04204665806066081, "nauc_map_at_100_diff1": 0.6537378762654921, "nauc_map_at_100_max": 0.591719062921236, "nauc_map_at_100_std": 0.042690974962164625, "nauc_map_at_10_diff1": 0.652369406457342, "nauc_map_at_10_max": 0.5909253802982799, "nauc_map_at_10_std": 0.036022144467421524, "nauc_map_at_1_diff1": 0.7126436781609194, "nauc_map_at_1_max": 0.5840418831680049, "nauc_map_at_1_std": -0.021833831942476276, "nauc_map_at_20_diff1": 0.6520302323881799, "nauc_map_at_20_max": 0.5908525887352375, "nauc_map_at_20_std": 0.04020584504644928, "nauc_map_at_3_diff1": 0.6640690747768079, "nauc_map_at_3_max": 0.5918378557726748, "nauc_map_at_3_std": 0.027992687337536, "nauc_map_at_5_diff1": 0.6522396910722045, "nauc_map_at_5_max": 0.591985080836136, "nauc_map_at_5_std": 0.040401896772712935, "nauc_mrr_at_1000_diff1": 0.653999576097392, "nauc_mrr_at_1000_max": 0.5915981156693542, "nauc_mrr_at_1000_std": 0.04204665806066081, "nauc_mrr_at_100_diff1": 0.6537378762654921, "nauc_mrr_at_100_max": 0.591719062921236, "nauc_mrr_at_100_std": 0.042690974962164625, "nauc_mrr_at_10_diff1": 0.652369406457342, "nauc_mrr_at_10_max": 0.5909253802982799, "nauc_mrr_at_10_std": 0.036022144467421524, "nauc_mrr_at_1_diff1": 0.7126436781609194, "nauc_mrr_at_1_max": 0.5840418831680049, "nauc_mrr_at_1_std": -0.021833831942476276, "nauc_mrr_at_20_diff1": 0.6520302323881799, "nauc_mrr_at_20_max": 0.5908525887352375, "nauc_mrr_at_20_std": 0.04020584504644928, "nauc_mrr_at_3_diff1": 0.6640690747768079, "nauc_mrr_at_3_max": 0.5918378557726748, "nauc_mrr_at_3_std": 0.027992687337536, "nauc_mrr_at_5_diff1": 0.6522396910722045, "nauc_mrr_at_5_max": 0.591985080836136, "nauc_mrr_at_5_std": 0.040401896772712935, "nauc_ndcg_at_1000_diff1": 0.6432566916474228, "nauc_ndcg_at_1000_max": 0.5998470404188347, "nauc_ndcg_at_1000_std": 0.07410684021566737, "nauc_ndcg_at_100_diff1": 0.6360589988421134, "nauc_ndcg_at_100_max": 0.6037341341497577, "nauc_ndcg_at_100_std": 0.09785768793288292, "nauc_ndcg_at_10_diff1": 0.6293196340250062, "nauc_ndcg_at_10_max": 0.5982485402739869, "nauc_ndcg_at_10_std": 0.06006582165943897, "nauc_ndcg_at_1_diff1": 0.7126436781609194, "nauc_ndcg_at_1_max": 0.5840418831680049, "nauc_ndcg_at_1_std": -0.021833831942476276, "nauc_ndcg_at_20_diff1": 0.6261813451342695, "nauc_ndcg_at_20_max": 0.5970582271477827, "nauc_ndcg_at_20_std": 0.07622456371047974, "nauc_ndcg_at_3_diff1": 0.6506603473332017, "nauc_ndcg_at_3_max": 0.6004543836287595, "nauc_ndcg_at_3_std": 0.047787735554177944, "nauc_ndcg_at_5_diff1": 0.6287749223122684, "nauc_ndcg_at_5_max": 0.6005997287841836, "nauc_ndcg_at_5_std": 0.07005047365688216, "nauc_precision_at_1000_diff1": 0.8078120136943695, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 0.5412788805727321, "nauc_precision_at_100_max": 0.7116986657988953, "nauc_precision_at_100_std": 0.598714611129191, "nauc_precision_at_10_diff1": 0.5518721348798572, "nauc_precision_at_10_max": 0.6234424514660185, "nauc_precision_at_10_std": 0.13918715195520429, "nauc_precision_at_1_diff1": 0.7126436781609194, "nauc_precision_at_1_max": 0.5840418831680049, "nauc_precision_at_1_std": -0.021833831942476276, "nauc_precision_at_20_diff1": 0.5220822971233046, "nauc_precision_at_20_max": 0.6184366015838579, "nauc_precision_at_20_std": 0.22843360318888078, "nauc_precision_at_3_diff1": 0.6122251832111925, "nauc_precision_at_3_max": 0.6275216522318455, "nauc_precision_at_3_std": 0.10674217188541009, "nauc_precision_at_5_diff1": 0.5556584283947462, "nauc_precision_at_5_max": 0.6282966330640979, "nauc_precision_at_5_std": 0.16318549585092312, "nauc_recall_at_1000_diff1": 0.8078120136943671, "nauc_recall_at_1000_max": 1.0, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_100_diff1": 0.5412788805727307, "nauc_recall_at_100_max": 0.7116986657988936, "nauc_recall_at_100_std": 0.5987146111291889, "nauc_recall_at_10_diff1": 0.5518721348798571, "nauc_recall_at_10_max": 0.6234424514660182, "nauc_recall_at_10_std": 0.13918715195520406, "nauc_recall_at_1_diff1": 0.7126436781609194, "nauc_recall_at_1_max": 0.5840418831680049, "nauc_recall_at_1_std": -0.021833831942476276, "nauc_recall_at_20_diff1": 0.5220822971233048, "nauc_recall_at_20_max": 0.6184366015838583, "nauc_recall_at_20_std": 0.22843360318888178, "nauc_recall_at_3_diff1": 0.6122251832111921, "nauc_recall_at_3_max": 0.6275216522318453, "nauc_recall_at_3_std": 0.10674217188540977, "nauc_recall_at_5_diff1": 0.5556584283947461, "nauc_recall_at_5_max": 0.6282966330640981, "nauc_recall_at_5_std": 0.16318549585092318, "ndcg_at_1": 0.36, "ndcg_at_10": 0.48402, "ndcg_at_100": 0.53127, "ndcg_at_1000": 0.54818, "ndcg_at_20": 0.50135, "ndcg_at_3": 0.44178, "ndcg_at_5": 0.4562, "precision_at_1": 0.36, "precision_at_10": 0.062, "precision_at_100": 0.0085, "precision_at_1000": 0.00099, "precision_at_20": 0.0345, "precision_at_3": 0.16667, "precision_at_5": 0.107, "recall_at_1": 0.36, "recall_at_10": 0.62, "recall_at_100": 0.85, "recall_at_1000": 0.985, "recall_at_20": 0.69, "recall_at_3": 0.5, "recall_at_5": 0.535 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.46733, "map_at_1": 0.33, "map_at_10": 0.42326, "map_at_100": 0.43228, "map_at_1000": 0.43308, "map_at_20": 0.42878, "map_at_3": 0.40333, "map_at_5": 0.41608, "mrr_at_1": 0.33, "mrr_at_10": 0.4232619047619048, "mrr_at_100": 0.432279471134081, "mrr_at_1000": 0.4330762455314873, "mrr_at_20": 0.4287762510107712, "mrr_at_3": 0.40333333333333327, "mrr_at_5": 0.4160833333333334, "nauc_map_at_1000_diff1": 0.6409752471277966, "nauc_map_at_1000_max": 0.5946002721149415, "nauc_map_at_1000_std": 0.03492754217391834, "nauc_map_at_100_diff1": 0.6405785551672617, "nauc_map_at_100_max": 0.5947016987665, "nauc_map_at_100_std": 0.03519683850581178, "nauc_map_at_10_diff1": 0.6437977226295699, "nauc_map_at_10_max": 0.5938053519033123, "nauc_map_at_10_std": 0.031333520693003776, "nauc_map_at_1_diff1": 0.7091105164356752, "nauc_map_at_1_max": 0.6154957329818537, "nauc_map_at_1_std": 0.014005937878606113, "nauc_map_at_20_diff1": 0.6414751638548382, "nauc_map_at_20_max": 0.5939793225373083, "nauc_map_at_20_std": 0.03190726201177065, "nauc_map_at_3_diff1": 0.6404104137741969, "nauc_map_at_3_max": 0.5827813691071001, "nauc_map_at_3_std": 0.008878591452145033, "nauc_map_at_5_diff1": 0.6400465541544528, "nauc_map_at_5_max": 0.5881739870325576, "nauc_map_at_5_std": 0.021291218705272164, "nauc_mrr_at_1000_diff1": 0.6409752471277966, "nauc_mrr_at_1000_max": 0.5946002721149415, "nauc_mrr_at_1000_std": 0.03492754217391834, "nauc_mrr_at_100_diff1": 0.6405785551672617, "nauc_mrr_at_100_max": 0.5947016987665, "nauc_mrr_at_100_std": 0.03519683850581178, "nauc_mrr_at_10_diff1": 0.6437977226295699, "nauc_mrr_at_10_max": 0.5938053519033123, "nauc_mrr_at_10_std": 0.031333520693003776, "nauc_mrr_at_1_diff1": 0.7091105164356752, "nauc_mrr_at_1_max": 0.6154957329818537, "nauc_mrr_at_1_std": 0.014005937878606113, "nauc_mrr_at_20_diff1": 0.6414751638548382, "nauc_mrr_at_20_max": 0.5939793225373083, "nauc_mrr_at_20_std": 0.03190726201177065, "nauc_mrr_at_3_diff1": 0.6404104137741969, "nauc_mrr_at_3_max": 0.5827813691071001, "nauc_mrr_at_3_std": 0.008878591452145033, "nauc_mrr_at_5_diff1": 0.6400465541544528, "nauc_mrr_at_5_max": 0.5881739870325576, "nauc_mrr_at_5_std": 0.021291218705272164, "nauc_ndcg_at_1000_diff1": 0.6225573328293726, "nauc_ndcg_at_1000_max": 0.599281715322545, "nauc_ndcg_at_1000_std": 0.06719272824721652, "nauc_ndcg_at_100_diff1": 0.6080334683443672, "nauc_ndcg_at_100_max": 0.6039284709620057, "nauc_ndcg_at_100_std": 0.08347014870182251, "nauc_ndcg_at_10_diff1": 0.6233539007043118, "nauc_ndcg_at_10_max": 0.594255714519112, "nauc_ndcg_at_10_std": 0.05460105589744826, "nauc_ndcg_at_1_diff1": 0.7091105164356752, "nauc_ndcg_at_1_max": 0.6154957329818537, "nauc_ndcg_at_1_std": 0.014005937878606113, "nauc_ndcg_at_20_diff1": 0.6135707800197637, "nauc_ndcg_at_20_max": 0.5967141800825231, "nauc_ndcg_at_20_std": 0.05829972851360095, "nauc_ndcg_at_3_diff1": 0.6164968584122404, "nauc_ndcg_at_3_max": 0.5722148672843138, "nauc_ndcg_at_3_std": 0.007984550405491661, "nauc_ndcg_at_5_diff1": 0.6147514584683341, "nauc_ndcg_at_5_max": 0.5815707344595532, "nauc_ndcg_at_5_std": 0.032085748538331245, "nauc_precision_at_1000_diff1": 0.5799097416744464, "nauc_precision_at_1000_max": 0.7833022097727956, "nauc_precision_at_1000_std": 0.8712262682850928, "nauc_precision_at_100_diff1": 0.40935590273338596, "nauc_precision_at_100_max": 0.6985987880027615, "nauc_precision_at_100_std": 0.4522488429773202, "nauc_precision_at_10_diff1": 0.5601805754156903, "nauc_precision_at_10_max": 0.6014014808381931, "nauc_precision_at_10_std": 0.14141330957603684, "nauc_precision_at_1_diff1": 0.7091105164356752, "nauc_precision_at_1_max": 0.6154957329818537, "nauc_precision_at_1_std": 0.014005937878606113, "nauc_precision_at_20_diff1": 0.5030837537143249, "nauc_precision_at_20_max": 0.6184524220855502, "nauc_precision_at_20_std": 0.1739686269089894, "nauc_precision_at_3_diff1": 0.5466105358366083, "nauc_precision_at_3_max": 0.541569402047046, "nauc_precision_at_3_std": 0.005626384150355919, "nauc_precision_at_5_diff1": 0.5370287158604224, "nauc_precision_at_5_max": 0.5628308642842097, "nauc_precision_at_5_std": 0.06966931124060934, "nauc_recall_at_1000_diff1": 0.5799097416744453, "nauc_recall_at_1000_max": 0.7833022097727981, "nauc_recall_at_1000_std": 0.8712262682850952, "nauc_recall_at_100_diff1": 0.40935590273338546, "nauc_recall_at_100_max": 0.6985987880027614, "nauc_recall_at_100_std": 0.45224884297731915, "nauc_recall_at_10_diff1": 0.5601805754156904, "nauc_recall_at_10_max": 0.6014014808381932, "nauc_recall_at_10_std": 0.141413309576037, "nauc_recall_at_1_diff1": 0.7091105164356752, "nauc_recall_at_1_max": 0.6154957329818537, "nauc_recall_at_1_std": 0.014005937878606113, "nauc_recall_at_20_diff1": 0.5030837537143243, "nauc_recall_at_20_max": 0.6184524220855504, "nauc_recall_at_20_std": 0.17396862690898954, "nauc_recall_at_3_diff1": 0.5466105358366087, "nauc_recall_at_3_max": 0.5415694020470461, "nauc_recall_at_3_std": 0.005626384150356102, "nauc_recall_at_5_diff1": 0.5370287158604221, "nauc_recall_at_5_max": 0.5628308642842097, "nauc_recall_at_5_std": 0.06966931124060896, "ndcg_at_1": 0.33, "ndcg_at_10": 0.46733, "ndcg_at_100": 0.51055, "ndcg_at_1000": 0.53145, "ndcg_at_20": 0.48667, "ndcg_at_3": 0.4269, "ndcg_at_5": 0.44971, "precision_at_1": 0.33, "precision_at_10": 0.0605, "precision_at_100": 0.00805, "precision_at_1000": 0.00097, "precision_at_20": 0.034, "precision_at_3": 0.165, "precision_at_5": 0.11, "recall_at_1": 0.33, "recall_at_10": 0.605, "recall_at_100": 0.805, "recall_at_1000": 0.97, "recall_at_20": 0.68, "recall_at_3": 0.495, "recall_at_5": 0.55 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 0.8287994861602783, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.74584, "map_at_1": 0.58442, "map_at_10": 0.69325, "map_at_100": 0.6971, "map_at_1000": 0.6971, "map_at_20": 0.69433, "map_at_3": 0.6645, "map_at_5": 0.68593, "mrr_at_1": 0.5844155844155844, "mrr_at_10": 0.6932488146773862, "mrr_at_100": 0.6970974064556953, "mrr_at_1000": 0.6970974064556953, "mrr_at_20": 0.6943310657596372, "mrr_at_3": 0.6645021645021645, "mrr_at_5": 0.685930735930736, "nauc_map_at_1000_diff1": 0.7508489385385345, "nauc_map_at_1000_max": 0.33771961050685273, "nauc_map_at_1000_std": 0.11725264519083228, "nauc_map_at_100_diff1": 0.7508489385385345, "nauc_map_at_100_max": 0.33771961050685273, "nauc_map_at_100_std": 0.11725264519083228, "nauc_map_at_10_diff1": 0.7483846046747796, "nauc_map_at_10_max": 0.3278511780894081, "nauc_map_at_10_std": 0.10838335653878874, "nauc_map_at_1_diff1": 0.8173815695220201, "nauc_map_at_1_max": 0.3888661331865086, "nauc_map_at_1_std": 0.182953924038586, "nauc_map_at_20_diff1": 0.748496829413396, "nauc_map_at_20_max": 0.3343465578187079, "nauc_map_at_20_std": 0.11392342799281159, "nauc_map_at_3_diff1": 0.7388199939774669, "nauc_map_at_3_max": 0.3464508902718699, "nauc_map_at_3_std": 0.12606109572212115, "nauc_map_at_5_diff1": 0.745049440719342, "nauc_map_at_5_max": 0.3326396709852455, "nauc_map_at_5_std": 0.0958479589280496, "nauc_mrr_at_1000_diff1": 0.7508489385385345, "nauc_mrr_at_1000_max": 0.33771961050685273, "nauc_mrr_at_1000_std": 0.11725264519083228, "nauc_mrr_at_100_diff1": 0.7508489385385345, "nauc_mrr_at_100_max": 0.33771961050685273, "nauc_mrr_at_100_std": 0.11725264519083228, "nauc_mrr_at_10_diff1": 0.7483846046747796, "nauc_mrr_at_10_max": 0.3278511780894081, "nauc_mrr_at_10_std": 0.10838335653878874, "nauc_mrr_at_1_diff1": 0.8173815695220201, "nauc_mrr_at_1_max": 0.3888661331865086, "nauc_mrr_at_1_std": 0.182953924038586, "nauc_mrr_at_20_diff1": 0.748496829413396, "nauc_mrr_at_20_max": 0.3343465578187079, "nauc_mrr_at_20_std": 0.11392342799281159, "nauc_mrr_at_3_diff1": 0.7388199939774669, "nauc_mrr_at_3_max": 0.3464508902718699, "nauc_mrr_at_3_std": 0.12606109572212115, "nauc_mrr_at_5_diff1": 0.745049440719342, "nauc_mrr_at_5_max": 0.3326396709852455, "nauc_mrr_at_5_std": 0.0958479589280496, "nauc_ndcg_at_1000_diff1": 0.7412942244900864, "nauc_ndcg_at_1000_max": 0.3294039014370792, "nauc_ndcg_at_1000_std": 0.10640993082293992, "nauc_ndcg_at_100_diff1": 0.7412942244900864, "nauc_ndcg_at_100_max": 0.3294039014370792, "nauc_ndcg_at_100_std": 0.10640993082293992, "nauc_ndcg_at_10_diff1": 0.7245304898357027, "nauc_ndcg_at_10_max": 0.28679437262219604, "nauc_ndcg_at_10_std": 0.07027087615828691, "nauc_ndcg_at_1_diff1": 0.8173815695220201, "nauc_ndcg_at_1_max": 0.3888661331865086, "nauc_ndcg_at_1_std": 0.182953924038586, "nauc_ndcg_at_20_diff1": 0.7245630270157414, "nauc_ndcg_at_20_max": 0.3112863125754883, "nauc_ndcg_at_20_std": 0.09110844214276108, "nauc_ndcg_at_3_diff1": 0.7075235631793855, "nauc_ndcg_at_3_max": 0.33335386066642836, "nauc_ndcg_at_3_std": 0.09914814670853166, "nauc_ndcg_at_5_diff1": 0.7186345849221563, "nauc_ndcg_at_5_max": 0.3034500672099746, "nauc_ndcg_at_5_std": 0.03592370656674037, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.5468315567317322, "nauc_precision_at_10_max": -0.06634874229956951, "nauc_precision_at_10_std": -0.21572501610658196, "nauc_precision_at_1_diff1": 0.8173815695220201, "nauc_precision_at_1_max": 0.3888661331865086, "nauc_precision_at_1_std": 0.182953924038586, "nauc_precision_at_20_diff1": 0.5175363752626384, "nauc_precision_at_20_max": 0.11665403289856896, "nauc_precision_at_20_std": -0.057618286542947705, "nauc_precision_at_3_diff1": 0.5903452819024319, "nauc_precision_at_3_max": 0.28575847531395104, "nauc_precision_at_3_std": -0.0032297965710546225, "nauc_precision_at_5_diff1": 0.5826897053901425, "nauc_precision_at_5_max": 0.14081401961953136, "nauc_precision_at_5_std": -0.3032079255275951, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.5468315567317299, "nauc_recall_at_10_max": -0.0663487422995702, "nauc_recall_at_10_std": -0.21572501610658368, "nauc_recall_at_1_diff1": 0.8173815695220201, "nauc_recall_at_1_max": 0.3888661331865086, "nauc_recall_at_1_std": 0.182953924038586, "nauc_recall_at_20_diff1": 0.5175363752626353, "nauc_recall_at_20_max": 0.11665403289856766, "nauc_recall_at_20_std": -0.05761828654294978, "nauc_recall_at_3_diff1": 0.5903452819024316, "nauc_recall_at_3_max": 0.2857584753139499, "nauc_recall_at_3_std": -0.0032297965710566382, "nauc_recall_at_5_diff1": 0.582689705390142, "nauc_recall_at_5_max": 0.14081401961952994, "nauc_recall_at_5_std": -0.30320792552759585, "ndcg_at_1": 0.58442, "ndcg_at_10": 0.74584, "ndcg_at_100": 0.76531, "ndcg_at_1000": 0.76531, "ndcg_at_20": 0.74935, "ndcg_at_3": 0.69063, "ndcg_at_5": 0.72864, "precision_at_1": 0.58442, "precision_at_10": 0.09091, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.0461, "precision_at_3": 0.25541, "precision_at_5": 0.17143, "recall_at_1": 0.58442, "recall_at_10": 0.90909, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.92208, "recall_at_3": 0.76623, "recall_at_5": 0.85714 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 8.458330631256104, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.30702, "map_at_1": 0.16654, "map_at_10": 0.26176, "map_at_100": 0.27447, "map_at_1000": 0.27609, "map_at_20": 0.26741, "map_at_3": 0.23243, "map_at_5": 0.24625, "mrr_at_1": 0.23853211009174313, "mrr_at_10": 0.2994272122712488, "mrr_at_100": 0.3092636327608635, "mrr_at_1000": 0.3103838648517695, "mrr_at_20": 0.3037993683781453, "mrr_at_3": 0.2787971457696229, "mrr_at_5": 0.2883537206931702, "nauc_map_at_1000_diff1": 0.46037215418732746, "nauc_map_at_1000_max": 0.17509641042870955, "nauc_map_at_1000_std": -0.17188557832411727, "nauc_map_at_100_diff1": 0.46012638860618277, "nauc_map_at_100_max": 0.17596415631476192, "nauc_map_at_100_std": -0.1706743537347323, "nauc_map_at_10_diff1": 0.4638654360953357, "nauc_map_at_10_max": 0.16920793650998478, "nauc_map_at_10_std": -0.1794379089439455, "nauc_map_at_1_diff1": 0.5347896707142972, "nauc_map_at_1_max": 0.1415086449494959, "nauc_map_at_1_std": -0.15905266073600383, "nauc_map_at_20_diff1": 0.46412466330756236, "nauc_map_at_20_max": 0.17219377532778699, "nauc_map_at_20_std": -0.1770192034055921, "nauc_map_at_3_diff1": 0.4636462501096955, "nauc_map_at_3_max": 0.16869218091148191, "nauc_map_at_3_std": -0.17287896666367372, "nauc_map_at_5_diff1": 0.4574507609721948, "nauc_map_at_5_max": 0.16880300762739983, "nauc_map_at_5_std": -0.17282665641306172, "nauc_mrr_at_1000_diff1": 0.47463336275490126, "nauc_mrr_at_1000_max": 0.1754406462372458, "nauc_mrr_at_1000_std": -0.17836658248225642, "nauc_mrr_at_100_diff1": 0.47438486192134394, "nauc_mrr_at_100_max": 0.17631789973620762, "nauc_mrr_at_100_std": -0.17729650956089177, "nauc_mrr_at_10_diff1": 0.4773350221044842, "nauc_mrr_at_10_max": 0.169646370077787, "nauc_mrr_at_10_std": -0.18538268596385504, "nauc_mrr_at_1_diff1": 0.5011382624720796, "nauc_mrr_at_1_max": 0.1710328904197035, "nauc_mrr_at_1_std": -0.20048698319894603, "nauc_mrr_at_20_diff1": 0.476647662919602, "nauc_mrr_at_20_max": 0.17387141807320416, "nauc_mrr_at_20_std": -0.18091518544942548, "nauc_mrr_at_3_diff1": 0.4807309717858008, "nauc_mrr_at_3_max": 0.16802519862298554, "nauc_mrr_at_3_std": -0.188463319155195, "nauc_mrr_at_5_diff1": 0.4757825397621256, "nauc_mrr_at_5_max": 0.16830225143196942, "nauc_mrr_at_5_std": -0.18397246262737063, "nauc_ndcg_at_1000_diff1": 0.4506344245877477, "nauc_ndcg_at_1000_max": 0.18690325211360131, "nauc_ndcg_at_1000_std": -0.15412837595255235, "nauc_ndcg_at_100_diff1": 0.43891772837206106, "nauc_ndcg_at_100_max": 0.20632483522841427, "nauc_ndcg_at_100_std": -0.12462220190485555, "nauc_ndcg_at_10_diff1": 0.4608819675765099, "nauc_ndcg_at_10_max": 0.16898512929369572, "nauc_ndcg_at_10_std": -0.17887277077473462, "nauc_ndcg_at_1_diff1": 0.5011382624720796, "nauc_ndcg_at_1_max": 0.1710328904197035, "nauc_ndcg_at_1_std": -0.20048698319894603, "nauc_ndcg_at_20_diff1": 0.4610263899849392, "nauc_ndcg_at_20_max": 0.18148306889631222, "nauc_ndcg_at_20_std": -0.1669782987396665, "nauc_ndcg_at_3_diff1": 0.4619158776702101, "nauc_ndcg_at_3_max": 0.16634419046964202, "nauc_ndcg_at_3_std": -0.17793218801362695, "nauc_ndcg_at_5_diff1": 0.45333962167423275, "nauc_ndcg_at_5_max": 0.16888164823356713, "nauc_ndcg_at_5_std": -0.16992609978088177, "nauc_precision_at_1000_diff1": -0.03034936062924852, "nauc_precision_at_1000_max": 0.0970387619515828, "nauc_precision_at_1000_std": 0.007738454229137765, "nauc_precision_at_100_diff1": 0.09814930477773491, "nauc_precision_at_100_max": 0.21804049604114545, "nauc_precision_at_100_std": 0.05638855924167594, "nauc_precision_at_10_diff1": 0.3097693795188088, "nauc_precision_at_10_max": 0.15471913586482394, "nauc_precision_at_10_std": -0.18007184914104188, "nauc_precision_at_1_diff1": 0.5011382624720796, "nauc_precision_at_1_max": 0.1710328904197035, "nauc_precision_at_1_std": -0.20048698319894603, "nauc_precision_at_20_diff1": 0.29084470955788, "nauc_precision_at_20_max": 0.17499612606897091, "nauc_precision_at_20_std": -0.14158176879585316, "nauc_precision_at_3_diff1": 0.3367612336349229, "nauc_precision_at_3_max": 0.17714261527312772, "nauc_precision_at_3_std": -0.19270047922441152, "nauc_precision_at_5_diff1": 0.30820104113069113, "nauc_precision_at_5_max": 0.164027258650643, "nauc_precision_at_5_std": -0.17750788382137142, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.3042749854292903, "nauc_recall_at_100_max": 0.34308274479573453, "nauc_recall_at_100_std": 0.12212368148463992, "nauc_recall_at_10_diff1": 0.42578302543364643, "nauc_recall_at_10_max": 0.15927311372949235, "nauc_recall_at_10_std": -0.16790350265095189, "nauc_recall_at_1_diff1": 0.5347896707142972, "nauc_recall_at_1_max": 0.1415086449494959, "nauc_recall_at_1_std": -0.15905266073600383, "nauc_recall_at_20_diff1": 0.4267970206817293, "nauc_recall_at_20_max": 0.20118521431192207, "nauc_recall_at_20_std": -0.12773593944421877, "nauc_recall_at_3_diff1": 0.42538434941403624, "nauc_recall_at_3_max": 0.16120603935905187, "nauc_recall_at_3_std": -0.15232329094239325, "nauc_recall_at_5_diff1": 0.41206673640909786, "nauc_recall_at_5_max": 0.16241875942413933, "nauc_recall_at_5_std": -0.1459331777008506, "ndcg_at_1": 0.23853, "ndcg_at_10": 0.30702, "ndcg_at_100": 0.37301, "ndcg_at_1000": 0.41317, "ndcg_at_20": 0.32484, "ndcg_at_3": 0.25869, "ndcg_at_5": 0.27568, "precision_at_1": 0.23853, "precision_at_10": 0.0656, "precision_at_100": 0.01139, "precision_at_1000": 0.00156, "precision_at_20": 0.03754, "precision_at_3": 0.14526, "precision_at_5": 0.10336, "recall_at_1": 0.16654, "recall_at_10": 0.40405, "recall_at_100": 0.70308, "recall_at_1000": 1.0, "recall_at_20": 0.46774, "recall_at_3": 0.27859, "recall_at_5": 0.31873 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.31767, "map_at_1": 0.15966, "map_at_10": 0.26847, "map_at_100": 0.28679, "map_at_1000": 0.28872, "map_at_20": 0.2779, "map_at_3": 0.23239, "map_at_5": 0.25223, "mrr_at_1": 0.28287461773700306, "mrr_at_10": 0.3392656909858745, "mrr_at_100": 0.35278022303264084, "mrr_at_1000": 0.353609282078622, "mrr_at_20": 0.34644945831570667, "mrr_at_3": 0.3216106014271154, "mrr_at_5": 0.33086136595310917, "nauc_map_at_1000_diff1": 0.43014345069572735, "nauc_map_at_1000_max": 0.2047702252588999, "nauc_map_at_1000_std": -0.18619729753828168, "nauc_map_at_100_diff1": 0.43044481164149107, "nauc_map_at_100_max": 0.20366438026852898, "nauc_map_at_100_std": -0.1872260820118899, "nauc_map_at_10_diff1": 0.4358332518843372, "nauc_map_at_10_max": 0.20302947817138878, "nauc_map_at_10_std": -0.19235050148342497, "nauc_map_at_1_diff1": 0.5094094888816958, "nauc_map_at_1_max": 0.2022136942837982, "nauc_map_at_1_std": -0.16629879431908526, "nauc_map_at_20_diff1": 0.43203029390503284, "nauc_map_at_20_max": 0.20281589336854433, "nauc_map_at_20_std": -0.18975715654086692, "nauc_map_at_3_diff1": 0.45683769382324835, "nauc_map_at_3_max": 0.2114974049562774, "nauc_map_at_3_std": -0.18973643431599205, "nauc_map_at_5_diff1": 0.44408303291369394, "nauc_map_at_5_max": 0.20755508120960656, "nauc_map_at_5_std": -0.1907358768225807, "nauc_mrr_at_1000_diff1": 0.39456178133655373, "nauc_mrr_at_1000_max": 0.16875177053249982, "nauc_mrr_at_1000_std": -0.18293062246227396, "nauc_mrr_at_100_diff1": 0.39441163476792995, "nauc_mrr_at_100_max": 0.16897730217108786, "nauc_mrr_at_100_std": -0.18246884202006136, "nauc_mrr_at_10_diff1": 0.3972495519440698, "nauc_mrr_at_10_max": 0.16447577013697356, "nauc_mrr_at_10_std": -0.18918425257815177, "nauc_mrr_at_1_diff1": 0.43078365226721643, "nauc_mrr_at_1_max": 0.1702882626404106, "nauc_mrr_at_1_std": -0.19022047922326274, "nauc_mrr_at_20_diff1": 0.39416443882324576, "nauc_mrr_at_20_max": 0.16767463626082799, "nauc_mrr_at_20_std": -0.18362884131701146, "nauc_mrr_at_3_diff1": 0.4052476466508942, "nauc_mrr_at_3_max": 0.1702917598959025, "nauc_mrr_at_3_std": -0.18516509262458347, "nauc_mrr_at_5_diff1": 0.39963181278379417, "nauc_mrr_at_5_max": 0.16467520855433232, "nauc_mrr_at_5_std": -0.1915264977988557, "nauc_ndcg_at_1000_diff1": 0.3972885201945313, "nauc_ndcg_at_1000_max": 0.20065085824709233, "nauc_ndcg_at_1000_std": -0.16600079150324779, "nauc_ndcg_at_100_diff1": 0.3963014283716522, "nauc_ndcg_at_100_max": 0.2045151315369148, "nauc_ndcg_at_100_std": -0.1558031913999578, "nauc_ndcg_at_10_diff1": 0.4089925178791608, "nauc_ndcg_at_10_max": 0.18660983146430193, "nauc_ndcg_at_10_std": -0.19358731965441847, "nauc_ndcg_at_1_diff1": 0.43078365226721643, "nauc_ndcg_at_1_max": 0.1702882626404106, "nauc_ndcg_at_1_std": -0.19022047922326274, "nauc_ndcg_at_20_diff1": 0.3960518079661171, "nauc_ndcg_at_20_max": 0.19130027169181624, "nauc_ndcg_at_20_std": -0.18014939464801458, "nauc_ndcg_at_3_diff1": 0.422294467926566, "nauc_ndcg_at_3_max": 0.19472976699280825, "nauc_ndcg_at_3_std": -0.19475941988238804, "nauc_ndcg_at_5_diff1": 0.42150046271189157, "nauc_ndcg_at_5_max": 0.19273669097369414, "nauc_ndcg_at_5_std": -0.19416658235162243, "nauc_precision_at_1000_diff1": -0.08592574039060139, "nauc_precision_at_1000_max": 0.03273543588118617, "nauc_precision_at_1000_std": 0.034027378427526046, "nauc_precision_at_100_diff1": 0.04597533172148491, "nauc_precision_at_100_max": 0.10853805071890375, "nauc_precision_at_100_std": 0.016852971188687477, "nauc_precision_at_10_diff1": 0.20270424761964256, "nauc_precision_at_10_max": 0.11219194377794384, "nauc_precision_at_10_std": -0.15977413044551775, "nauc_precision_at_1_diff1": 0.43078365226721643, "nauc_precision_at_1_max": 0.1702882626404106, "nauc_precision_at_1_std": -0.19022047922326274, "nauc_precision_at_20_diff1": 0.1571047263945439, "nauc_precision_at_20_max": 0.10847376906241053, "nauc_precision_at_20_std": -0.1221847777154949, "nauc_precision_at_3_diff1": 0.28366520674490603, "nauc_precision_at_3_max": 0.15377715751169105, "nauc_precision_at_3_std": -0.18440380852826943, "nauc_precision_at_5_diff1": 0.2419626741051632, "nauc_precision_at_5_max": 0.13773446176999182, "nauc_precision_at_5_std": -0.1731204992204692, "nauc_recall_at_1000_diff1": 0.3794834485562372, "nauc_recall_at_1000_max": 0.85774024244626, "nauc_recall_at_1000_std": 0.8178081164635485, "nauc_recall_at_100_diff1": 0.29569892516149393, "nauc_recall_at_100_max": 0.26057031713542306, "nauc_recall_at_100_std": 0.028329931041429883, "nauc_recall_at_10_diff1": 0.348625448128896, "nauc_recall_at_10_max": 0.1643013408968958, "nauc_recall_at_10_std": -0.1839318165372178, "nauc_recall_at_1_diff1": 0.5094094888816958, "nauc_recall_at_1_max": 0.2022136942837982, "nauc_recall_at_1_std": -0.16629879431908526, "nauc_recall_at_20_diff1": 0.30229226859696084, "nauc_recall_at_20_max": 0.18109751827851614, "nauc_recall_at_20_std": -0.13702112357432608, "nauc_recall_at_3_diff1": 0.4025757277241289, "nauc_recall_at_3_max": 0.19605999728416043, "nauc_recall_at_3_std": -0.17775588225430455, "nauc_recall_at_5_diff1": 0.37889647475908744, "nauc_recall_at_5_max": 0.1769513092530404, "nauc_recall_at_5_std": -0.189916223373261, "ndcg_at_1": 0.28287, "ndcg_at_10": 0.31767, "ndcg_at_100": 0.40373, "ndcg_at_1000": 0.43976, "ndcg_at_20": 0.34737, "ndcg_at_3": 0.27816, "ndcg_at_5": 0.2897, "precision_at_1": 0.28287, "precision_at_10": 0.08073, "precision_at_100": 0.01489, "precision_at_1000": 0.00194, "precision_at_20": 0.05008, "precision_at_3": 0.18094, "precision_at_5": 0.1315, "recall_at_1": 0.15966, "recall_at_10": 0.39006, "recall_at_100": 0.75754, "recall_at_1000": 0.99791, "recall_at_20": 0.48981, "recall_at_3": 0.26937, "recall_at_5": 0.31955 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.29414, "map_at_1": 0.16169, "map_at_10": 0.2529, "map_at_100": 0.26952, "map_at_1000": 0.27127, "map_at_20": 0.26193, "map_at_3": 0.22819, "map_at_5": 0.24023, "mrr_at_1": 0.23289902280130292, "mrr_at_10": 0.2953647691432707, "mrr_at_100": 0.3071666675264507, "mrr_at_1000": 0.30820981319186874, "mrr_at_20": 0.302379485948379, "mrr_at_3": 0.27904451682953313, "mrr_at_5": 0.286699239956569, "nauc_map_at_1000_diff1": 0.402343188167998, "nauc_map_at_1000_max": 0.1299753545662161, "nauc_map_at_1000_std": -0.19019062841266368, "nauc_map_at_100_diff1": 0.4014383333829576, "nauc_map_at_100_max": 0.13032050794516703, "nauc_map_at_100_std": -0.18960081163590695, "nauc_map_at_10_diff1": 0.4094501157108844, "nauc_map_at_10_max": 0.12651540749352239, "nauc_map_at_10_std": -0.19866000806898948, "nauc_map_at_1_diff1": 0.49998577979517983, "nauc_map_at_1_max": 0.13000935252442475, "nauc_map_at_1_std": -0.15265135985215136, "nauc_map_at_20_diff1": 0.4042994016481559, "nauc_map_at_20_max": 0.12718616483049597, "nauc_map_at_20_std": -0.1940792732991164, "nauc_map_at_3_diff1": 0.41724986246003115, "nauc_map_at_3_max": 0.12281456025945924, "nauc_map_at_3_std": -0.18824030125445101, "nauc_map_at_5_diff1": 0.41152774793705993, "nauc_map_at_5_max": 0.11919557019524485, "nauc_map_at_5_std": -0.19524997617072462, "nauc_mrr_at_1000_diff1": 0.4307038972832546, "nauc_mrr_at_1000_max": 0.13352079024017607, "nauc_mrr_at_1000_std": -0.184049039415171, "nauc_mrr_at_100_diff1": 0.43026716054026354, "nauc_mrr_at_100_max": 0.13406610028872684, "nauc_mrr_at_100_std": -0.18348794449823466, "nauc_mrr_at_10_diff1": 0.43488458644787525, "nauc_mrr_at_10_max": 0.12881780828582814, "nauc_mrr_at_10_std": -0.19015371067992137, "nauc_mrr_at_1_diff1": 0.48468119540124965, "nauc_mrr_at_1_max": 0.15042533595554772, "nauc_mrr_at_1_std": -0.18437012032221006, "nauc_mrr_at_20_diff1": 0.4314267128535979, "nauc_mrr_at_20_max": 0.13322245076423717, "nauc_mrr_at_20_std": -0.18459294813723132, "nauc_mrr_at_3_diff1": 0.43897967255324455, "nauc_mrr_at_3_max": 0.1254625553396092, "nauc_mrr_at_3_std": -0.19092380547574533, "nauc_mrr_at_5_diff1": 0.43502145833562217, "nauc_mrr_at_5_max": 0.12200585923704393, "nauc_mrr_at_5_std": -0.19294483424478642, "nauc_ndcg_at_1000_diff1": 0.3880667336937146, "nauc_ndcg_at_1000_max": 0.14092945160808437, "nauc_ndcg_at_1000_std": -0.17436592159846515, "nauc_ndcg_at_100_diff1": 0.366324948159121, "nauc_ndcg_at_100_max": 0.15152727703516525, "nauc_ndcg_at_100_std": -0.15744481087079681, "nauc_ndcg_at_10_diff1": 0.39720572550680455, "nauc_ndcg_at_10_max": 0.12608780384446125, "nauc_ndcg_at_10_std": -0.20161383875574887, "nauc_ndcg_at_1_diff1": 0.48468119540124965, "nauc_ndcg_at_1_max": 0.15042533595554772, "nauc_ndcg_at_1_std": -0.18437012032221006, "nauc_ndcg_at_20_diff1": 0.38198989790320925, "nauc_ndcg_at_20_max": 0.1309411335322321, "nauc_ndcg_at_20_std": -0.1851352452028806, "nauc_ndcg_at_3_diff1": 0.4051695223146839, "nauc_ndcg_at_3_max": 0.11944364435132239, "nauc_ndcg_at_3_std": -0.19126849095602919, "nauc_ndcg_at_5_diff1": 0.39947912250237116, "nauc_ndcg_at_5_max": 0.10992273316880617, "nauc_ndcg_at_5_std": -0.20016531175412253, "nauc_precision_at_1000_diff1": -0.01258523599785369, "nauc_precision_at_1000_max": 0.13899813834379615, "nauc_precision_at_1000_std": 0.04000898712738393, "nauc_precision_at_100_diff1": 0.04345755772775644, "nauc_precision_at_100_max": 0.19226963732967228, "nauc_precision_at_100_std": 0.026170385214899197, "nauc_precision_at_10_diff1": 0.2369754676384766, "nauc_precision_at_10_max": 0.11842985597589158, "nauc_precision_at_10_std": -0.20115083032042783, "nauc_precision_at_1_diff1": 0.48468119540124965, "nauc_precision_at_1_max": 0.15042533595554772, "nauc_precision_at_1_std": -0.18437012032221006, "nauc_precision_at_20_diff1": 0.1709476117260024, "nauc_precision_at_20_max": 0.12064257063411347, "nauc_precision_at_20_std": -0.1401821039917873, "nauc_precision_at_3_diff1": 0.291841490033633, "nauc_precision_at_3_max": 0.11466029738009711, "nauc_precision_at_3_std": -0.20702151399249047, "nauc_precision_at_5_diff1": 0.26375878245224577, "nauc_precision_at_5_max": 0.09186387677675449, "nauc_precision_at_5_std": -0.21344184500612426, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.17567011906387164, "nauc_recall_at_100_max": 0.22326397283006172, "nauc_recall_at_100_std": -0.012573773594995362, "nauc_recall_at_10_diff1": 0.3302339778122505, "nauc_recall_at_10_max": 0.11908567442911973, "nauc_recall_at_10_std": -0.20567345005177817, "nauc_recall_at_1_diff1": 0.49998577979517983, "nauc_recall_at_1_max": 0.13000935252442475, "nauc_recall_at_1_std": -0.15265135985215136, "nauc_recall_at_20_diff1": 0.2808225129566676, "nauc_recall_at_20_max": 0.1309001978764897, "nauc_recall_at_20_std": -0.15195365980849182, "nauc_recall_at_3_diff1": 0.3532291466642572, "nauc_recall_at_3_max": 0.09429804779858321, "nauc_recall_at_3_std": -0.18793468626423918, "nauc_recall_at_5_diff1": 0.3371090964412974, "nauc_recall_at_5_max": 0.07987294522395673, "nauc_recall_at_5_std": -0.2053014135072843, "ndcg_at_1": 0.2329, "ndcg_at_10": 0.29414, "ndcg_at_100": 0.36767, "ndcg_at_1000": 0.41069, "ndcg_at_20": 0.32188, "ndcg_at_3": 0.25664, "ndcg_at_5": 0.26976, "precision_at_1": 0.2329, "precision_at_10": 0.06352, "precision_at_100": 0.01197, "precision_at_1000": 0.00167, "precision_at_20": 0.04055, "precision_at_3": 0.14767, "precision_at_5": 0.10293, "recall_at_1": 0.16169, "recall_at_10": 0.37021, "recall_at_100": 0.68048, "recall_at_1000": 1.0, "recall_at_20": 0.46359, "recall_at_3": 0.2719, "recall_at_5": 0.30482 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/model_meta.json ================================================ {"name": "Alibaba-NLP/gte-multilingual-base", "revision": "no_revision_available", "release_date": null, "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "loader": null} ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 50.766931772232056, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.83008, "map_at_1": 0.67544, "map_at_10": 0.78346, "map_at_100": 0.78514, "map_at_1000": 0.78514, "map_at_20": 0.78514, "map_at_3": 0.76316, "map_at_5": 0.77807, "mrr_at_1": 0.6754385964912281, "mrr_at_10": 0.7834551656920076, "mrr_at_100": 0.7851364522417152, "mrr_at_1000": 0.7851364522417152, "mrr_at_20": 0.7851364522417152, "mrr_at_3": 0.763157894736842, "mrr_at_5": 0.7780701754385964, "nauc_map_at_1000_diff1": 0.7480913240467126, "nauc_map_at_1000_max": 0.40994529514037564, "nauc_map_at_1000_std": -0.4041235885308545, "nauc_map_at_100_diff1": 0.7480913240467126, "nauc_map_at_100_max": 0.40994529514037564, "nauc_map_at_100_std": -0.4041235885308545, "nauc_map_at_10_diff1": 0.7480655535449382, "nauc_map_at_10_max": 0.4118513196432917, "nauc_map_at_10_std": -0.4096886794877292, "nauc_map_at_1_diff1": 0.7458196841797787, "nauc_map_at_1_max": 0.336170140077049, "nauc_map_at_1_std": -0.414109409881053, "nauc_map_at_20_diff1": 0.7480913240467126, "nauc_map_at_20_max": 0.40994529514037564, "nauc_map_at_20_std": -0.4041235885308545, "nauc_map_at_3_diff1": 0.7539865940326704, "nauc_map_at_3_max": 0.43539749968062214, "nauc_map_at_3_std": -0.37847225265122253, "nauc_map_at_5_diff1": 0.7459089722049166, "nauc_map_at_5_max": 0.4228032046558571, "nauc_map_at_5_std": -0.39011603199971223, "nauc_mrr_at_1000_diff1": 0.7480913240467126, "nauc_mrr_at_1000_max": 0.40994529514037564, "nauc_mrr_at_1000_std": -0.4041235885308545, "nauc_mrr_at_100_diff1": 0.7480913240467126, "nauc_mrr_at_100_max": 0.40994529514037564, "nauc_mrr_at_100_std": -0.4041235885308545, "nauc_mrr_at_10_diff1": 0.7480655535449382, "nauc_mrr_at_10_max": 0.4118513196432917, "nauc_mrr_at_10_std": -0.4096886794877292, "nauc_mrr_at_1_diff1": 0.7458196841797787, "nauc_mrr_at_1_max": 0.336170140077049, "nauc_mrr_at_1_std": -0.414109409881053, "nauc_mrr_at_20_diff1": 0.7480913240467126, "nauc_mrr_at_20_max": 0.40994529514037564, "nauc_mrr_at_20_std": -0.4041235885308545, "nauc_mrr_at_3_diff1": 0.7539865940326704, "nauc_mrr_at_3_max": 0.43539749968062214, "nauc_mrr_at_3_std": -0.37847225265122253, "nauc_mrr_at_5_diff1": 0.7459089722049166, "nauc_mrr_at_5_max": 0.4228032046558571, "nauc_mrr_at_5_std": -0.39011603199971223, "nauc_ndcg_at_1000_diff1": 0.7479959191104514, "nauc_ndcg_at_1000_max": 0.4183290864437548, "nauc_ndcg_at_1000_std": -0.40438403097706516, "nauc_ndcg_at_100_diff1": 0.7479959191104514, "nauc_ndcg_at_100_max": 0.4183290864437548, "nauc_ndcg_at_100_std": -0.40438403097706516, "nauc_ndcg_at_10_diff1": 0.7477939461929645, "nauc_ndcg_at_10_max": 0.427014191152891, "nauc_ndcg_at_10_std": -0.43143047149879826, "nauc_ndcg_at_1_diff1": 0.7458196841797787, "nauc_ndcg_at_1_max": 0.336170140077049, "nauc_ndcg_at_1_std": -0.414109409881053, "nauc_ndcg_at_20_diff1": 0.7479959191104514, "nauc_ndcg_at_20_max": 0.4183290864437548, "nauc_ndcg_at_20_std": -0.40438403097706516, "nauc_ndcg_at_3_diff1": 0.7579191887220348, "nauc_ndcg_at_3_max": 0.48018090636879757, "nauc_ndcg_at_3_std": -0.35255980074276044, "nauc_ndcg_at_5_diff1": 0.7420060164656852, "nauc_ndcg_at_5_max": 0.46033083419612225, "nauc_ndcg_at_5_std": -0.3737451683555684, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.7424388206277509, "nauc_precision_at_10_max": 0.6148801048337371, "nauc_precision_at_10_std": -1.0242575880069984, "nauc_precision_at_1_diff1": 0.7458196841797787, "nauc_precision_at_1_max": 0.336170140077049, "nauc_precision_at_1_std": -0.414109409881053, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_3_diff1": 0.7765098273273427, "nauc_precision_at_3_max": 0.690557013561025, "nauc_precision_at_3_std": -0.22618135533655087, "nauc_precision_at_5_diff1": 0.7064297107719757, "nauc_precision_at_5_max": 0.7510957493307909, "nauc_precision_at_5_std": -0.24381180954894718, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.7424388206277525, "nauc_recall_at_10_max": 0.6148801048337426, "nauc_recall_at_10_std": -1.024257588006983, "nauc_recall_at_1_diff1": 0.7458196841797787, "nauc_recall_at_1_max": 0.336170140077049, "nauc_recall_at_1_std": -0.414109409881053, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_3_diff1": 0.7765098273273437, "nauc_recall_at_3_max": 0.6905570135610254, "nauc_recall_at_3_std": -0.22618135533654984, "nauc_recall_at_5_diff1": 0.7064297107719775, "nauc_recall_at_5_max": 0.7510957493307917, "nauc_recall_at_5_std": -0.24381180954894452, "ndcg_at_1": 0.67544, "ndcg_at_10": 0.83008, "ndcg_at_100": 0.83656, "ndcg_at_1000": 0.83656, "ndcg_at_20": 0.83656, "ndcg_at_3": 0.79031, "ndcg_at_5": 0.81637, "precision_at_1": 0.67544, "precision_at_10": 0.09737, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.05, "precision_at_3": 0.28947, "precision_at_5": 0.18596, "recall_at_1": 0.67544, "recall_at_10": 0.97368, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 1.0, "recall_at_3": 0.86842, "recall_at_5": 0.92982 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 42.1163535118103, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.93164, "map_at_1": 0.87889, "map_at_10": 0.91594, "map_at_100": 0.91652, "map_at_1000": 0.91656, "map_at_20": 0.91631, "map_at_3": 0.90944, "map_at_5": 0.91344, "mrr_at_1": 0.8788888888888889, "mrr_at_10": 0.915938712522046, "mrr_at_100": 0.9165196696089475, "mrr_at_1000": 0.9165613420972883, "mrr_at_20": 0.9163125423958758, "mrr_at_3": 0.9094444444444446, "mrr_at_5": 0.9134444444444446, "nauc_map_at_1000_diff1": 0.9144018424312078, "nauc_map_at_1000_max": 0.7625594223308878, "nauc_map_at_1000_std": 0.08078533832109351, "nauc_map_at_100_diff1": 0.9144325380134781, "nauc_map_at_100_max": 0.7626093328877492, "nauc_map_at_100_std": 0.08115864225106753, "nauc_map_at_10_diff1": 0.9143139179478519, "nauc_map_at_10_max": 0.763197065026625, "nauc_map_at_10_std": 0.08254448049332629, "nauc_map_at_1_diff1": 0.9182364852426823, "nauc_map_at_1_max": 0.7360338011378074, "nauc_map_at_1_std": 0.055233653871440136, "nauc_map_at_20_diff1": 0.9145308131541325, "nauc_map_at_20_max": 0.7627389376229389, "nauc_map_at_20_std": 0.0799713735263409, "nauc_map_at_3_diff1": 0.9131876782636175, "nauc_map_at_3_max": 0.7698593668502859, "nauc_map_at_3_std": 0.08250297146424689, "nauc_map_at_5_diff1": 0.9140255060135355, "nauc_map_at_5_max": 0.7658910154050281, "nauc_map_at_5_std": 0.08177793016479339, "nauc_mrr_at_1000_diff1": 0.9144018424312078, "nauc_mrr_at_1000_max": 0.7625594223308878, "nauc_mrr_at_1000_std": 0.08078533832109351, "nauc_mrr_at_100_diff1": 0.9144325380134781, "nauc_mrr_at_100_max": 0.7626093328877492, "nauc_mrr_at_100_std": 0.08115864225106753, "nauc_mrr_at_10_diff1": 0.9143139179478519, "nauc_mrr_at_10_max": 0.763197065026625, "nauc_mrr_at_10_std": 0.08254448049332629, "nauc_mrr_at_1_diff1": 0.9182364852426823, "nauc_mrr_at_1_max": 0.7360338011378074, "nauc_mrr_at_1_std": 0.055233653871440136, "nauc_mrr_at_20_diff1": 0.9145308131541325, "nauc_mrr_at_20_max": 0.7627389376229389, "nauc_mrr_at_20_std": 0.0799713735263409, "nauc_mrr_at_3_diff1": 0.9131876782636175, "nauc_mrr_at_3_max": 0.7698593668502859, "nauc_mrr_at_3_std": 0.08250297146424689, "nauc_mrr_at_5_diff1": 0.9140255060135355, "nauc_mrr_at_5_max": 0.7658910154050281, "nauc_mrr_at_5_std": 0.08177793016479339, "nauc_ndcg_at_1000_diff1": 0.9145982546159986, "nauc_ndcg_at_1000_max": 0.7658476741472501, "nauc_ndcg_at_1000_std": 0.08783028480120193, "nauc_ndcg_at_100_diff1": 0.9152384365333261, "nauc_ndcg_at_100_max": 0.7672413292997466, "nauc_ndcg_at_100_std": 0.09739655293777892, "nauc_ndcg_at_10_diff1": 0.914857655889842, "nauc_ndcg_at_10_max": 0.769537674473529, "nauc_ndcg_at_10_std": 0.09838891524632452, "nauc_ndcg_at_1_diff1": 0.9182364852426823, "nauc_ndcg_at_1_max": 0.7360338011378074, "nauc_ndcg_at_1_std": 0.055233653871440136, "nauc_ndcg_at_20_diff1": 0.91593068556539, "nauc_ndcg_at_20_max": 0.7676635353081863, "nauc_ndcg_at_20_std": 0.08682263886524719, "nauc_ndcg_at_3_diff1": 0.9121176059457036, "nauc_ndcg_at_3_max": 0.7833482042650488, "nauc_ndcg_at_3_std": 0.09610521311242015, "nauc_ndcg_at_5_diff1": 0.9141395218023763, "nauc_ndcg_at_5_max": 0.7764607899687949, "nauc_ndcg_at_5_std": 0.09570954287447543, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9564270152505334, "nauc_precision_at_100_max": 0.8665577342047719, "nauc_precision_at_100_std": 0.7584811702458871, "nauc_precision_at_10_diff1": 0.9255628177196747, "nauc_precision_at_10_max": 0.8285610540512508, "nauc_precision_at_10_std": 0.2889563232700455, "nauc_precision_at_1_diff1": 0.9182364852426823, "nauc_precision_at_1_max": 0.7360338011378074, "nauc_precision_at_1_std": 0.055233653871440136, "nauc_precision_at_20_diff1": 0.949723479135242, "nauc_precision_at_20_max": 0.8182862888745052, "nauc_precision_at_20_std": 0.1519428284134084, "nauc_precision_at_3_diff1": 0.9077497665732935, "nauc_precision_at_3_max": 0.8447338935574222, "nauc_precision_at_3_std": 0.15945845004668682, "nauc_precision_at_5_diff1": 0.9168589004229141, "nauc_precision_at_5_max": 0.8416817707475116, "nauc_precision_at_5_std": 0.18645301257757552, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9564270152505339, "nauc_recall_at_100_max": 0.8665577342047819, "nauc_recall_at_100_std": 0.7584811702458742, "nauc_recall_at_10_diff1": 0.9255628177196826, "nauc_recall_at_10_max": 0.8285610540512551, "nauc_recall_at_10_std": 0.2889563232700448, "nauc_recall_at_1_diff1": 0.9182364852426823, "nauc_recall_at_1_max": 0.7360338011378074, "nauc_recall_at_1_std": 0.055233653871440136, "nauc_recall_at_20_diff1": 0.9497234791352488, "nauc_recall_at_20_max": 0.8182862888745368, "nauc_recall_at_20_std": 0.15194282841342197, "nauc_recall_at_3_diff1": 0.907749766573295, "nauc_recall_at_3_max": 0.8447338935574227, "nauc_recall_at_3_std": 0.15945845004668527, "nauc_recall_at_5_diff1": 0.9168589004229126, "nauc_recall_at_5_max": 0.8416817707475147, "nauc_recall_at_5_std": 0.18645301257758004, "ndcg_at_1": 0.87889, "ndcg_at_10": 0.93164, "ndcg_at_100": 0.93448, "ndcg_at_1000": 0.93539, "ndcg_at_20": 0.93303, "ndcg_at_3": 0.9185, "ndcg_at_5": 0.92577, "precision_at_1": 0.87889, "precision_at_10": 0.098, "precision_at_100": 0.00993, "precision_at_1000": 0.001, "precision_at_20": 0.04928, "precision_at_3": 0.31481, "precision_at_5": 0.19244, "recall_at_1": 0.87889, "recall_at_10": 0.98, "recall_at_100": 0.99333, "recall_at_1000": 1.0, "recall_at_20": 0.98556, "recall_at_3": 0.94444, "recall_at_5": 0.96222 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.90365, "map_at_1": 0.83556, "map_at_10": 0.88327, "map_at_100": 0.88495, "map_at_1000": 0.88497, "map_at_20": 0.88462, "map_at_3": 0.87574, "map_at_5": 0.88052, "mrr_at_1": 0.8355555555555556, "mrr_at_10": 0.8832663139329808, "mrr_at_100": 0.8849464937287592, "mrr_at_1000": 0.8849749325397649, "mrr_at_20": 0.8846182396703555, "mrr_at_3": 0.8757407407407409, "mrr_at_5": 0.8805185185185187, "nauc_map_at_1000_diff1": 0.855347628670212, "nauc_map_at_1000_max": 0.6898140243410444, "nauc_map_at_1000_std": 0.10353937270721077, "nauc_map_at_100_diff1": 0.855275356255775, "nauc_map_at_100_max": 0.6898299533073919, "nauc_map_at_100_std": 0.10369584797022877, "nauc_map_at_10_diff1": 0.8545939543607107, "nauc_map_at_10_max": 0.691361711480954, "nauc_map_at_10_std": 0.10799233041904452, "nauc_map_at_1_diff1": 0.8673830358805222, "nauc_map_at_1_max": 0.6620807511237589, "nauc_map_at_1_std": 0.05348498550416149, "nauc_map_at_20_diff1": 0.8550816588802468, "nauc_map_at_20_max": 0.6902918325884226, "nauc_map_at_20_std": 0.10508842671827126, "nauc_map_at_3_diff1": 0.8556449045847431, "nauc_map_at_3_max": 0.6940007000047526, "nauc_map_at_3_std": 0.10619835484479548, "nauc_map_at_5_diff1": 0.8549919507891112, "nauc_map_at_5_max": 0.6928210519115601, "nauc_map_at_5_std": 0.10521440661745296, "nauc_mrr_at_1000_diff1": 0.855347628670212, "nauc_mrr_at_1000_max": 0.6898140243410444, "nauc_mrr_at_1000_std": 0.10353937270721077, "nauc_mrr_at_100_diff1": 0.855275356255775, "nauc_mrr_at_100_max": 0.6898299533073919, "nauc_mrr_at_100_std": 0.10369584797022877, "nauc_mrr_at_10_diff1": 0.8545939543607107, "nauc_mrr_at_10_max": 0.691361711480954, "nauc_mrr_at_10_std": 0.10799233041904452, "nauc_mrr_at_1_diff1": 0.8673830358805222, "nauc_mrr_at_1_max": 0.6620807511237589, "nauc_mrr_at_1_std": 0.05348498550416149, "nauc_mrr_at_20_diff1": 0.8550816588802468, "nauc_mrr_at_20_max": 0.6902918325884226, "nauc_mrr_at_20_std": 0.10508842671827126, "nauc_mrr_at_3_diff1": 0.8556449045847431, "nauc_mrr_at_3_max": 0.6940007000047526, "nauc_mrr_at_3_std": 0.10619835484479548, "nauc_mrr_at_5_diff1": 0.8549919507891112, "nauc_mrr_at_5_max": 0.6928210519115601, "nauc_mrr_at_5_std": 0.10521440661745296, "nauc_ndcg_at_1000_diff1": 0.8527012462802727, "nauc_ndcg_at_1000_max": 0.6933498509696858, "nauc_ndcg_at_1000_std": 0.11443873671850584, "nauc_ndcg_at_100_diff1": 0.8508516789196892, "nauc_ndcg_at_100_max": 0.6943035655845461, "nauc_ndcg_at_100_std": 0.11991478266036447, "nauc_ndcg_at_10_diff1": 0.8477011560099511, "nauc_ndcg_at_10_max": 0.701026901481694, "nauc_ndcg_at_10_std": 0.14075623622130473, "nauc_ndcg_at_1_diff1": 0.8673830358805222, "nauc_ndcg_at_1_max": 0.6620807511237589, "nauc_ndcg_at_1_std": 0.05348498550416149, "nauc_ndcg_at_20_diff1": 0.8495343252357374, "nauc_ndcg_at_20_max": 0.6966836378939858, "nauc_ndcg_at_20_std": 0.12927198789204877, "nauc_ndcg_at_3_diff1": 0.8509468666257631, "nauc_ndcg_at_3_max": 0.7046413690365483, "nauc_ndcg_at_3_std": 0.12717513638981803, "nauc_ndcg_at_5_diff1": 0.849510553173776, "nauc_ndcg_at_5_max": 0.7041140837858083, "nauc_ndcg_at_5_std": 0.12814143423088434, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.6438842203548022, "nauc_precision_at_100_max": 0.819327731092435, "nauc_precision_at_100_std": 0.7824463118580629, "nauc_precision_at_10_diff1": 0.7782601929660737, "nauc_precision_at_10_max": 0.790242763772179, "nauc_precision_at_10_std": 0.48392468098350033, "nauc_precision_at_1_diff1": 0.8673830358805222, "nauc_precision_at_1_max": 0.6620807511237589, "nauc_precision_at_1_std": 0.05348498550416149, "nauc_precision_at_20_diff1": 0.7402627717753778, "nauc_precision_at_20_max": 0.7947512338268672, "nauc_precision_at_20_std": 0.6165466186474461, "nauc_precision_at_3_diff1": 0.8300112315457603, "nauc_precision_at_3_max": 0.7509776857602969, "nauc_precision_at_3_std": 0.22059161828983137, "nauc_precision_at_5_diff1": 0.8170961718020517, "nauc_precision_at_5_max": 0.7704388422035447, "nauc_precision_at_5_std": 0.2653127917833765, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.6438842203548035, "nauc_recall_at_100_max": 0.8193277310924408, "nauc_recall_at_100_std": 0.782446311858087, "nauc_recall_at_10_diff1": 0.778260192966078, "nauc_recall_at_10_max": 0.7902427637721744, "nauc_recall_at_10_std": 0.48392468098350666, "nauc_recall_at_1_diff1": 0.8673830358805222, "nauc_recall_at_1_max": 0.6620807511237589, "nauc_recall_at_1_std": 0.05348498550416149, "nauc_recall_at_20_diff1": 0.7402627717753743, "nauc_recall_at_20_max": 0.7947512338268609, "nauc_recall_at_20_std": 0.6165466186474606, "nauc_recall_at_3_diff1": 0.8300112315457603, "nauc_recall_at_3_max": 0.7509776857602934, "nauc_recall_at_3_std": 0.22059161828982624, "nauc_recall_at_5_diff1": 0.8170961718020519, "nauc_recall_at_5_max": 0.7704388422035484, "nauc_recall_at_5_std": 0.2653127917833834, "ndcg_at_1": 0.83556, "ndcg_at_10": 0.90365, "ndcg_at_100": 0.91028, "ndcg_at_1000": 0.911, "ndcg_at_20": 0.90829, "ndcg_at_3": 0.88803, "ndcg_at_5": 0.89668, "precision_at_1": 0.83556, "precision_at_10": 0.09667, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "precision_at_20": 0.04922, "precision_at_3": 0.30778, "precision_at_5": 0.18889, "recall_at_1": 0.83556, "recall_at_10": 0.96667, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "recall_at_20": 0.98444, "recall_at_3": 0.92333, "recall_at_5": 0.94444 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.88363, "map_at_1": 0.79667, "map_at_10": 0.85787, "map_at_100": 0.8592, "map_at_1000": 0.85925, "map_at_20": 0.85867, "map_at_3": 0.84741, "map_at_5": 0.85496, "mrr_at_1": 0.7966666666666666, "mrr_at_10": 0.8578650793650797, "mrr_at_100": 0.8592039439643446, "mrr_at_1000": 0.8592467110152766, "mrr_at_20": 0.8586734267672094, "mrr_at_3": 0.8474074074074077, "mrr_at_5": 0.8549629629629634, "nauc_map_at_1000_diff1": 0.859468841140476, "nauc_map_at_1000_max": 0.6418851630612379, "nauc_map_at_1000_std": 0.12161212115529317, "nauc_map_at_100_diff1": 0.8594694509113175, "nauc_map_at_100_max": 0.6419321905712058, "nauc_map_at_100_std": 0.12179724364431067, "nauc_map_at_10_diff1": 0.8588613413517707, "nauc_map_at_10_max": 0.641709583131691, "nauc_map_at_10_std": 0.121384800872592, "nauc_map_at_1_diff1": 0.8702250130821565, "nauc_map_at_1_max": 0.6163243388374275, "nauc_map_at_1_std": 0.07164637905101645, "nauc_map_at_20_diff1": 0.8594870506841481, "nauc_map_at_20_max": 0.6425710944098023, "nauc_map_at_20_std": 0.1237131469715452, "nauc_map_at_3_diff1": 0.8558682567291922, "nauc_map_at_3_max": 0.6492018403346966, "nauc_map_at_3_std": 0.11860755714696808, "nauc_map_at_5_diff1": 0.8589466899055904, "nauc_map_at_5_max": 0.6449249342806429, "nauc_map_at_5_std": 0.1277634095588614, "nauc_mrr_at_1000_diff1": 0.859468841140476, "nauc_mrr_at_1000_max": 0.6418851630612379, "nauc_mrr_at_1000_std": 0.12161212115529317, "nauc_mrr_at_100_diff1": 0.8594694509113175, "nauc_mrr_at_100_max": 0.6419321905712058, "nauc_mrr_at_100_std": 0.12179724364431067, "nauc_mrr_at_10_diff1": 0.8588613413517707, "nauc_mrr_at_10_max": 0.641709583131691, "nauc_mrr_at_10_std": 0.121384800872592, "nauc_mrr_at_1_diff1": 0.8702250130821565, "nauc_mrr_at_1_max": 0.6163243388374275, "nauc_mrr_at_1_std": 0.07164637905101645, "nauc_mrr_at_20_diff1": 0.8594870506841481, "nauc_mrr_at_20_max": 0.6425710944098023, "nauc_mrr_at_20_std": 0.1237131469715452, "nauc_mrr_at_3_diff1": 0.8558682567291922, "nauc_mrr_at_3_max": 0.6492018403346966, "nauc_mrr_at_3_std": 0.11860755714696808, "nauc_mrr_at_5_diff1": 0.8589466899055904, "nauc_mrr_at_5_max": 0.6449249342806429, "nauc_mrr_at_5_std": 0.1277634095588614, "nauc_ndcg_at_1000_diff1": 0.8588358520863845, "nauc_ndcg_at_1000_max": 0.645465796921435, "nauc_ndcg_at_1000_std": 0.13240406904224863, "nauc_ndcg_at_100_diff1": 0.8588633851935984, "nauc_ndcg_at_100_max": 0.6464585395890068, "nauc_ndcg_at_100_std": 0.13631972155577624, "nauc_ndcg_at_10_diff1": 0.8567728449023886, "nauc_ndcg_at_10_max": 0.6475703207508358, "nauc_ndcg_at_10_std": 0.1422797846281043, "nauc_ndcg_at_1_diff1": 0.8702250130821565, "nauc_ndcg_at_1_max": 0.6163243388374275, "nauc_ndcg_at_1_std": 0.07164637905101645, "nauc_ndcg_at_20_diff1": 0.8594313083100267, "nauc_ndcg_at_20_max": 0.6510508012250286, "nauc_ndcg_at_20_std": 0.15212758928270584, "nauc_ndcg_at_3_diff1": 0.8507464933321702, "nauc_ndcg_at_3_max": 0.6625695478441651, "nauc_ndcg_at_3_std": 0.13701665752617875, "nauc_ndcg_at_5_diff1": 0.8570505968282397, "nauc_ndcg_at_5_max": 0.6557638094232093, "nauc_ndcg_at_5_std": 0.15745646851391326, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8627450980391947, "nauc_precision_at_100_max": 0.7735760971054999, "nauc_precision_at_100_std": 0.6373482726424144, "nauc_precision_at_10_diff1": 0.8408716427747522, "nauc_precision_at_10_max": 0.6951859175042543, "nauc_precision_at_10_std": 0.3470917778876209, "nauc_precision_at_1_diff1": 0.8702250130821565, "nauc_precision_at_1_max": 0.6163243388374275, "nauc_precision_at_1_std": 0.07164637905101645, "nauc_precision_at_20_diff1": 0.8759142545907178, "nauc_precision_at_20_max": 0.7660480859010207, "nauc_precision_at_20_std": 0.5798124805477631, "nauc_precision_at_3_diff1": 0.8278734257930804, "nauc_precision_at_3_max": 0.7232185557149703, "nauc_precision_at_3_std": 0.22018970189701878, "nauc_precision_at_5_diff1": 0.8476980535804056, "nauc_precision_at_5_max": 0.7277167277167269, "nauc_precision_at_5_std": 0.36440601881777845, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8627450980392323, "nauc_recall_at_100_max": 0.7735760971055182, "nauc_recall_at_100_std": 0.6373482726423895, "nauc_recall_at_10_diff1": 0.8408716427747571, "nauc_recall_at_10_max": 0.695185917504258, "nauc_recall_at_10_std": 0.347091777887624, "nauc_recall_at_1_diff1": 0.8702250130821565, "nauc_recall_at_1_max": 0.6163243388374275, "nauc_recall_at_1_std": 0.07164637905101645, "nauc_recall_at_20_diff1": 0.8759142545907237, "nauc_recall_at_20_max": 0.7660480859010214, "nauc_recall_at_20_std": 0.5798124805477695, "nauc_recall_at_3_diff1": 0.8278734257930809, "nauc_recall_at_3_max": 0.7232185557149695, "nauc_recall_at_3_std": 0.22018970189702047, "nauc_recall_at_5_diff1": 0.8476980535804093, "nauc_recall_at_5_max": 0.7277167277167285, "nauc_recall_at_5_std": 0.3644060188177846, "ndcg_at_1": 0.79667, "ndcg_at_10": 0.88363, "ndcg_at_100": 0.89038, "ndcg_at_1000": 0.89116, "ndcg_at_20": 0.88648, "ndcg_at_3": 0.86325, "ndcg_at_5": 0.87693, "precision_at_1": 0.79667, "precision_at_10": 0.09622, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "precision_at_20": 0.04867, "precision_at_3": 0.30296, "precision_at_5": 0.18844, "recall_at_1": 0.79667, "recall_at_10": 0.96222, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "recall_at_20": 0.97333, "recall_at_3": 0.90889, "recall_at_5": 0.94222 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 185.89697933197021, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.79405, "map_at_1": 0.49267, "map_at_10": 0.75068, "map_at_100": 0.75588, "map_at_1000": 0.75626, "map_at_20": 0.75409, "map_at_3": 0.71019, "map_at_5": 0.73924, "mrr_at_1": 0.7567567567567568, "mrr_at_10": 0.8111593736593736, "mrr_at_100": 0.8129820393113771, "mrr_at_1000": 0.8131176819899872, "mrr_at_20": 0.8123800402389797, "mrr_at_3": 0.8026463963963962, "mrr_at_5": 0.809065315315315, "nauc_map_at_1000_diff1": 0.6073564845705162, "nauc_map_at_1000_max": 0.6142808799071932, "nauc_map_at_1000_std": 0.2777535403105099, "nauc_map_at_100_diff1": 0.6074610172189073, "nauc_map_at_100_max": 0.6147329654986178, "nauc_map_at_100_std": 0.27839664065266817, "nauc_map_at_10_diff1": 0.6057452723427406, "nauc_map_at_10_max": 0.6147408260448912, "nauc_map_at_10_std": 0.27396835761844074, "nauc_map_at_1_diff1": 0.6606149953272771, "nauc_map_at_1_max": 0.2789519671066579, "nauc_map_at_1_std": 0.043338484369368206, "nauc_map_at_20_diff1": 0.6050594418976999, "nauc_map_at_20_max": 0.6143563208405669, "nauc_map_at_20_std": 0.276742270435723, "nauc_map_at_3_diff1": 0.5916665330995543, "nauc_map_at_3_max": 0.5605227755844461, "nauc_map_at_3_std": 0.2104961965183136, "nauc_map_at_5_diff1": 0.6012828801837301, "nauc_map_at_5_max": 0.5977220283801926, "nauc_map_at_5_std": 0.239568588369421, "nauc_mrr_at_1000_diff1": 0.7204077645198788, "nauc_mrr_at_1000_max": 0.6874919147005658, "nauc_mrr_at_1000_std": 0.3960669270046734, "nauc_mrr_at_100_diff1": 0.7204224854441412, "nauc_mrr_at_100_max": 0.6877110638254341, "nauc_mrr_at_100_std": 0.3964536952924918, "nauc_mrr_at_10_diff1": 0.7208324628160704, "nauc_mrr_at_10_max": 0.6897440891787184, "nauc_mrr_at_10_std": 0.3963588333134652, "nauc_mrr_at_1_diff1": 0.733564685115092, "nauc_mrr_at_1_max": 0.6004409767078538, "nauc_mrr_at_1_std": 0.3056509437634208, "nauc_mrr_at_20_diff1": 0.7206297856582866, "nauc_mrr_at_20_max": 0.688336867623408, "nauc_mrr_at_20_std": 0.39657236146996266, "nauc_mrr_at_3_diff1": 0.7186594739549138, "nauc_mrr_at_3_max": 0.68914836868804, "nauc_mrr_at_3_std": 0.38332225603422865, "nauc_mrr_at_5_diff1": 0.7209733355169085, "nauc_mrr_at_5_max": 0.6888801228180421, "nauc_mrr_at_5_std": 0.39270866874049193, "nauc_ndcg_at_1000_diff1": 0.6338473482311605, "nauc_ndcg_at_1000_max": 0.6658318500784468, "nauc_ndcg_at_1000_std": 0.3613327838786333, "nauc_ndcg_at_100_diff1": 0.6359961983984979, "nauc_ndcg_at_100_max": 0.6768630279285994, "nauc_ndcg_at_100_std": 0.37843517651343045, "nauc_ndcg_at_10_diff1": 0.6302885124429262, "nauc_ndcg_at_10_max": 0.677901974773586, "nauc_ndcg_at_10_std": 0.35987771021811943, "nauc_ndcg_at_1_diff1": 0.733564685115092, "nauc_ndcg_at_1_max": 0.6004409767078538, "nauc_ndcg_at_1_std": 0.3056509437634208, "nauc_ndcg_at_20_diff1": 0.6284643385980219, "nauc_ndcg_at_20_max": 0.6768232098229742, "nauc_ndcg_at_20_std": 0.3688374813640457, "nauc_ndcg_at_3_diff1": 0.6054013951584047, "nauc_ndcg_at_3_max": 0.6092729334025931, "nauc_ndcg_at_3_std": 0.25866032885907, "nauc_ndcg_at_5_diff1": 0.6238418504187758, "nauc_ndcg_at_5_max": 0.6509906586415192, "nauc_ndcg_at_5_std": 0.30210068484335106, "nauc_precision_at_1000_diff1": -0.35990568904125725, "nauc_precision_at_1000_max": 0.0067781404379028245, "nauc_precision_at_1000_std": 0.15395949020431401, "nauc_precision_at_100_diff1": -0.23352187794170565, "nauc_precision_at_100_max": 0.15596081334229617, "nauc_precision_at_100_std": 0.267173695536987, "nauc_precision_at_10_diff1": -0.13641666339761727, "nauc_precision_at_10_max": 0.2816031935192935, "nauc_precision_at_10_std": 0.30570851296675766, "nauc_precision_at_1_diff1": 0.733564685115092, "nauc_precision_at_1_max": 0.6004409767078538, "nauc_precision_at_1_std": 0.3056509437634208, "nauc_precision_at_20_diff1": -0.18962238004033924, "nauc_precision_at_20_max": 0.22895177335346711, "nauc_precision_at_20_std": 0.29028521348267383, "nauc_precision_at_3_diff1": -0.0033508590425714883, "nauc_precision_at_3_max": 0.3897633058892868, "nauc_precision_at_3_std": 0.2660099232096745, "nauc_precision_at_5_diff1": -0.08105324547471025, "nauc_precision_at_5_max": 0.3249889835054694, "nauc_precision_at_5_std": 0.25654282884764673, "nauc_recall_at_1000_diff1": 0.5511369328228183, "nauc_recall_at_1000_max": 0.8893951650461698, "nauc_recall_at_1000_std": 0.908942266283279, "nauc_recall_at_100_diff1": 0.620790115513722, "nauc_recall_at_100_max": 0.885920841896125, "nauc_recall_at_100_std": 0.7925130360150211, "nauc_recall_at_10_diff1": 0.5943289326994741, "nauc_recall_at_10_max": 0.8028881422138773, "nauc_recall_at_10_std": 0.5280584453731186, "nauc_recall_at_1_diff1": 0.6606149953272771, "nauc_recall_at_1_max": 0.2789519671066579, "nauc_recall_at_1_std": 0.043338484369368206, "nauc_recall_at_20_diff1": 0.5813900968151912, "nauc_recall_at_20_max": 0.8287976348105712, "nauc_recall_at_20_std": 0.6138570052685672, "nauc_recall_at_3_diff1": 0.5549790197723918, "nauc_recall_at_3_max": 0.6195795055811252, "nauc_recall_at_3_std": 0.25210116535419186, "nauc_recall_at_5_diff1": 0.584423609937162, "nauc_recall_at_5_max": 0.7135500268808825, "nauc_recall_at_5_std": 0.3504720792523775, "ndcg_at_1": 0.75676, "ndcg_at_10": 0.79405, "ndcg_at_100": 0.81105, "ndcg_at_1000": 0.81929, "ndcg_at_20": 0.80289, "ndcg_at_3": 0.74791, "ndcg_at_5": 0.77754, "precision_at_1": 0.75676, "precision_at_10": 0.15743, "precision_at_100": 0.0172, "precision_at_1000": 0.00184, "precision_at_20": 0.08193, "precision_at_3": 0.44313, "precision_at_5": 0.29493, "recall_at_1": 0.49267, "recall_at_10": 0.85203, "recall_at_100": 0.9141, "recall_at_1000": 0.96796, "recall_at_20": 0.87968, "recall_at_3": 0.75176, "recall_at_5": 0.81346 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 25134.011864423752, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.70146, "map_at_1": 0.42297, "map_at_10": 0.61161, "map_at_100": 0.63802, "map_at_1000": 0.63844, "map_at_20": 0.62969, "map_at_3": 0.53676, "map_at_5": 0.57778, "mrr_at_1": 0.6713615023474179, "mrr_at_10": 0.7659344958640735, "mrr_at_100": 0.7683117539523098, "mrr_at_1000": 0.7683494097782307, "mrr_at_20": 0.7676708919666668, "mrr_at_3": 0.7511737089201878, "mrr_at_5": 0.761267605633803, "nauc_map_at_1000_diff1": 0.2860679246516418, "nauc_map_at_1000_max": 0.2416200329933295, "nauc_map_at_1000_std": -0.12005920416004798, "nauc_map_at_100_diff1": 0.28612865312726604, "nauc_map_at_100_max": 0.24147614705594259, "nauc_map_at_100_std": -0.12000538814638757, "nauc_map_at_10_diff1": 0.2910535705883505, "nauc_map_at_10_max": 0.22063655735554957, "nauc_map_at_10_std": -0.1755548069677421, "nauc_map_at_1_diff1": 0.3731225761234927, "nauc_map_at_1_max": 0.09201026764951797, "nauc_map_at_1_std": -0.3014661427052009, "nauc_map_at_20_diff1": 0.28831099175996144, "nauc_map_at_20_max": 0.2380261079942362, "nauc_map_at_20_std": -0.13345707243096736, "nauc_map_at_3_diff1": 0.3342501310457326, "nauc_map_at_3_max": 0.15872651094652668, "nauc_map_at_3_std": -0.26884596583229825, "nauc_map_at_5_diff1": 0.29839691155660714, "nauc_map_at_5_max": 0.1870288376376699, "nauc_map_at_5_std": -0.21145942033023102, "nauc_mrr_at_1000_diff1": 0.27343011112899185, "nauc_mrr_at_1000_max": 0.36215294276407056, "nauc_mrr_at_1000_std": 0.03997507028764519, "nauc_mrr_at_100_diff1": 0.2733480592732512, "nauc_mrr_at_100_max": 0.36215662012921923, "nauc_mrr_at_100_std": 0.0400697404328969, "nauc_mrr_at_10_diff1": 0.27482567942556374, "nauc_mrr_at_10_max": 0.36635220564893367, "nauc_mrr_at_10_std": 0.036866485562697504, "nauc_mrr_at_1_diff1": 0.2948130734383108, "nauc_mrr_at_1_max": 0.2926407082840507, "nauc_mrr_at_1_std": -0.027831021522469086, "nauc_mrr_at_20_diff1": 0.27309467155693556, "nauc_mrr_at_20_max": 0.36358562538828787, "nauc_mrr_at_20_std": 0.04156820247382249, "nauc_mrr_at_3_diff1": 0.2691890441095148, "nauc_mrr_at_3_max": 0.3601144018951212, "nauc_mrr_at_3_std": 0.0356925953037033, "nauc_mrr_at_5_diff1": 0.2782361790018662, "nauc_mrr_at_5_max": 0.36655032927215986, "nauc_mrr_at_5_std": 0.04156090999995617, "nauc_ndcg_at_1000_diff1": 0.27008440080369717, "nauc_ndcg_at_1000_max": 0.31432467578992335, "nauc_ndcg_at_1000_std": -0.01322753759468674, "nauc_ndcg_at_100_diff1": 0.2691417400497451, "nauc_ndcg_at_100_max": 0.3149913299281308, "nauc_ndcg_at_100_std": -0.0069897350287974995, "nauc_ndcg_at_10_diff1": 0.2759161503451715, "nauc_ndcg_at_10_max": 0.28337609820359727, "nauc_ndcg_at_10_std": -0.11079866548891888, "nauc_ndcg_at_1_diff1": 0.2948130734383108, "nauc_ndcg_at_1_max": 0.2926407082840507, "nauc_ndcg_at_1_std": -0.027831021522469086, "nauc_ndcg_at_20_diff1": 0.2719166891767834, "nauc_ndcg_at_20_max": 0.31104279428884174, "nauc_ndcg_at_20_std": -0.0325073620013048, "nauc_ndcg_at_3_diff1": 0.2971934028744493, "nauc_ndcg_at_3_max": 0.2443350194831595, "nauc_ndcg_at_3_std": -0.11285592786460469, "nauc_ndcg_at_5_diff1": 0.27619207496823056, "nauc_ndcg_at_5_max": 0.24632112632918796, "nauc_ndcg_at_5_std": -0.1027718220598243, "nauc_precision_at_1000_diff1": -0.21812881816741134, "nauc_precision_at_1000_max": 0.14106897125057397, "nauc_precision_at_1000_std": 0.48173738645886327, "nauc_precision_at_100_diff1": -0.21203964015536822, "nauc_precision_at_100_max": 0.15783084406970369, "nauc_precision_at_100_std": 0.5021751159419123, "nauc_precision_at_10_diff1": -0.2007207389719376, "nauc_precision_at_10_max": 0.23263258358188674, "nauc_precision_at_10_std": 0.4501018452604258, "nauc_precision_at_1_diff1": 0.2948130734383108, "nauc_precision_at_1_max": 0.2926407082840507, "nauc_precision_at_1_std": -0.027831021522469086, "nauc_precision_at_20_diff1": -0.20516364776757468, "nauc_precision_at_20_max": 0.1977163767715501, "nauc_precision_at_20_std": 0.4995418544872059, "nauc_precision_at_3_diff1": -0.10731185639919887, "nauc_precision_at_3_max": 0.27109790464689093, "nauc_precision_at_3_std": 0.28277292710319796, "nauc_precision_at_5_diff1": -0.20268354514790782, "nauc_precision_at_5_max": 0.23139780215041297, "nauc_precision_at_5_std": 0.40233922024606095, "nauc_recall_at_1000_diff1": 0.3625332041174769, "nauc_recall_at_1000_max": 0.8600957914042522, "nauc_recall_at_1000_std": 0.8103352826255814, "nauc_recall_at_100_diff1": 0.21183843007998174, "nauc_recall_at_100_max": 0.48029123227428006, "nauc_recall_at_100_std": 0.4243256333272366, "nauc_recall_at_10_diff1": 0.22120282376188027, "nauc_recall_at_10_max": 0.2749413623813262, "nauc_recall_at_10_std": -0.18226669196653814, "nauc_recall_at_1_diff1": 0.3731225761234927, "nauc_recall_at_1_max": 0.09201026764951797, "nauc_recall_at_1_std": -0.3014661427052009, "nauc_recall_at_20_diff1": 0.21036420602067207, "nauc_recall_at_20_max": 0.38058742085733716, "nauc_recall_at_20_std": 0.11650063573321266, "nauc_recall_at_3_diff1": 0.3251254419362985, "nauc_recall_at_3_max": 0.15496086730521202, "nauc_recall_at_3_std": -0.2878870790262251, "nauc_recall_at_5_diff1": 0.2592535399889266, "nauc_recall_at_5_max": 0.19539608299498687, "nauc_recall_at_5_std": -0.1963956729684935, "ndcg_at_1": 0.67136, "ndcg_at_10": 0.70146, "ndcg_at_100": 0.75433, "ndcg_at_1000": 0.75993, "ndcg_at_20": 0.73002, "ndcg_at_3": 0.65453, "ndcg_at_5": 0.66807, "precision_at_1": 0.67136, "precision_at_10": 0.18122, "precision_at_100": 0.02451, "precision_at_1000": 0.00255, "precision_at_20": 0.10587, "precision_at_3": 0.39124, "precision_at_5": 0.29108, "recall_at_1": 0.42297, "recall_at_10": 0.792, "recall_at_100": 0.95456, "recall_at_1000": 0.98732, "recall_at_20": 0.86726, "recall_at_3": 0.59949, "recall_at_5": 0.68371 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 15034.357434511185, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.64708, "map_at_1": 0.47823, "map_at_10": 0.59078, "map_at_100": 0.59938, "map_at_1000": 0.59956, "map_at_20": 0.59711, "map_at_3": 0.56182, "map_at_5": 0.57697, "mrr_at_1": 0.5201900237529691, "mrr_at_10": 0.6182624514572258, "mrr_at_100": 0.6236958323634181, "mrr_at_1000": 0.623854309693826, "mrr_at_20": 0.622454465076337, "mrr_at_3": 0.5942201108471894, "mrr_at_5": 0.6063341250989708, "nauc_map_at_1000_diff1": 0.3995809729123824, "nauc_map_at_1000_max": 0.3685222486712915, "nauc_map_at_1000_std": 0.06373832421801984, "nauc_map_at_100_diff1": 0.3996006455556901, "nauc_map_at_100_max": 0.36864203636141646, "nauc_map_at_100_std": 0.06383760412402077, "nauc_map_at_10_diff1": 0.40046422054500413, "nauc_map_at_10_max": 0.3670278175626621, "nauc_map_at_10_std": 0.0611611053235871, "nauc_map_at_1_diff1": 0.45659036090089766, "nauc_map_at_1_max": 0.33927633905548615, "nauc_map_at_1_std": 0.030288521635467035, "nauc_map_at_20_diff1": 0.39748224739889754, "nauc_map_at_20_max": 0.36602272288213494, "nauc_map_at_20_std": 0.062088919209027066, "nauc_map_at_3_diff1": 0.4054712860586374, "nauc_map_at_3_max": 0.3624873599798654, "nauc_map_at_3_std": 0.044772202259510875, "nauc_map_at_5_diff1": 0.39915141043021296, "nauc_map_at_5_max": 0.3718285751697345, "nauc_map_at_5_std": 0.06739584492621328, "nauc_mrr_at_1000_diff1": 0.4145924520403507, "nauc_mrr_at_1000_max": 0.39913215972590815, "nauc_mrr_at_1000_std": 0.08536808844882394, "nauc_mrr_at_100_diff1": 0.4145639732356783, "nauc_mrr_at_100_max": 0.3991859161244107, "nauc_mrr_at_100_std": 0.0854102871442121, "nauc_mrr_at_10_diff1": 0.4138086354915033, "nauc_mrr_at_10_max": 0.3989755743376254, "nauc_mrr_at_10_std": 0.08724304325932151, "nauc_mrr_at_1_diff1": 0.4647720318783489, "nauc_mrr_at_1_max": 0.3726092605121275, "nauc_mrr_at_1_std": 0.05069874346527008, "nauc_mrr_at_20_diff1": 0.4141702250960487, "nauc_mrr_at_20_max": 0.3985579489738777, "nauc_mrr_at_20_std": 0.08461936766142766, "nauc_mrr_at_3_diff1": 0.4131951592928855, "nauc_mrr_at_3_max": 0.3912898253463345, "nauc_mrr_at_3_std": 0.06924020273458875, "nauc_mrr_at_5_diff1": 0.4106291356397838, "nauc_mrr_at_5_max": 0.4018937774834494, "nauc_mrr_at_5_std": 0.08975693367426406, "nauc_ndcg_at_1000_diff1": 0.39195098358822655, "nauc_ndcg_at_1000_max": 0.3918858910986785, "nauc_ndcg_at_1000_std": 0.08771090753438557, "nauc_ndcg_at_100_diff1": 0.39104415839438417, "nauc_ndcg_at_100_max": 0.3962634998305795, "nauc_ndcg_at_100_std": 0.09272160089858074, "nauc_ndcg_at_10_diff1": 0.3863953192497146, "nauc_ndcg_at_10_max": 0.38554346803374445, "nauc_ndcg_at_10_std": 0.08406611640534893, "nauc_ndcg_at_1_diff1": 0.4647720318783489, "nauc_ndcg_at_1_max": 0.3726092605121275, "nauc_ndcg_at_1_std": 0.05069874346527008, "nauc_ndcg_at_20_diff1": 0.37950175729603264, "nauc_ndcg_at_20_max": 0.38392833756198785, "nauc_ndcg_at_20_std": 0.0820759351926107, "nauc_ndcg_at_3_diff1": 0.3912189680297108, "nauc_ndcg_at_3_max": 0.3734691298157102, "nauc_ndcg_at_3_std": 0.052651355500254036, "nauc_ndcg_at_5_diff1": 0.3814458791769543, "nauc_ndcg_at_5_max": 0.39347904278774437, "nauc_ndcg_at_5_std": 0.09381500382630131, "nauc_precision_at_1000_diff1": -0.12898648404854904, "nauc_precision_at_1000_max": 0.050173585805003536, "nauc_precision_at_1000_std": 0.13373269070320742, "nauc_precision_at_100_diff1": -0.05502828412565439, "nauc_precision_at_100_max": 0.14498281238555707, "nauc_precision_at_100_std": 0.18089885274812176, "nauc_precision_at_10_diff1": 0.08791350092212162, "nauc_precision_at_10_max": 0.25165509721637486, "nauc_precision_at_10_std": 0.1509702516631406, "nauc_precision_at_1_diff1": 0.4647720318783489, "nauc_precision_at_1_max": 0.3726092605121275, "nauc_precision_at_1_std": 0.05069874346527008, "nauc_precision_at_20_diff1": -0.03977766401214331, "nauc_precision_at_20_max": 0.16065895400343622, "nauc_precision_at_20_std": 0.14222227441113028, "nauc_precision_at_3_diff1": 0.24874213589040703, "nauc_precision_at_3_max": 0.350474552111039, "nauc_precision_at_3_std": 0.07535034822396319, "nauc_precision_at_5_diff1": 0.156423303496061, "nauc_precision_at_5_max": 0.3363413019177353, "nauc_precision_at_5_std": 0.17570990345202453, "nauc_recall_at_1000_diff1": 0.31102111214763173, "nauc_recall_at_1000_max": 0.7449072924584909, "nauc_recall_at_1000_std": 0.38932378344859375, "nauc_recall_at_100_diff1": 0.32095681471021337, "nauc_recall_at_100_max": 0.6448452329637108, "nauc_recall_at_100_std": 0.3486184419235017, "nauc_recall_at_10_diff1": 0.3086089560455287, "nauc_recall_at_10_max": 0.4185246680502123, "nauc_recall_at_10_std": 0.14169850467989772, "nauc_recall_at_1_diff1": 0.45659036090089766, "nauc_recall_at_1_max": 0.33927633905548615, "nauc_recall_at_1_std": 0.030288521635467035, "nauc_recall_at_20_diff1": 0.23899533056974023, "nauc_recall_at_20_max": 0.42735942577615504, "nauc_recall_at_20_std": 0.13915565833927118, "nauc_recall_at_3_diff1": 0.3312670437746897, "nauc_recall_at_3_max": 0.36969768105139733, "nauc_recall_at_3_std": 0.050130007567662206, "nauc_recall_at_5_diff1": 0.30225426737864636, "nauc_recall_at_5_max": 0.4308089065920804, "nauc_recall_at_5_std": 0.15938912980183326, "ndcg_at_1": 0.52019, "ndcg_at_10": 0.64708, "ndcg_at_100": 0.67978, "ndcg_at_1000": 0.68529, "ndcg_at_20": 0.66663, "ndcg_at_3": 0.59182, "ndcg_at_5": 0.61685, "precision_at_1": 0.52019, "precision_at_10": 0.09097, "precision_at_100": 0.01088, "precision_at_1000": 0.00114, "precision_at_20": 0.05, "precision_at_3": 0.2407, "precision_at_5": 0.1601, "recall_at_1": 0.47823, "recall_at_10": 0.79137, "recall_at_100": 0.92914, "recall_at_1000": 0.97348, "recall_at_20": 0.86382, "recall_at_3": 0.64766, "recall_at_5": 0.70467 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 11767.261229991913, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.43011, "map_at_1": 0.305, "map_at_10": 0.38785, "map_at_100": 0.39515, "map_at_1000": 0.39604, "map_at_20": 0.39052, "map_at_3": 0.36583, "map_at_5": 0.37733, "mrr_at_1": 0.305, "mrr_at_10": 0.3878531746031747, "mrr_at_100": 0.39515355720909306, "mrr_at_1000": 0.3960379563087161, "mrr_at_20": 0.39052307545825365, "mrr_at_3": 0.36583333333333334, "mrr_at_5": 0.37733333333333335, "nauc_map_at_1000_diff1": 0.5810056203148152, "nauc_map_at_1000_max": 0.5588819003136871, "nauc_map_at_1000_std": 0.059712449754901524, "nauc_map_at_100_diff1": 0.5805354442945942, "nauc_map_at_100_max": 0.5588426603626263, "nauc_map_at_100_std": 0.05977434254548648, "nauc_map_at_10_diff1": 0.583233343861228, "nauc_map_at_10_max": 0.5601596948204381, "nauc_map_at_10_std": 0.05652632363778106, "nauc_map_at_1_diff1": 0.6522943384528295, "nauc_map_at_1_max": 0.5608308005418264, "nauc_map_at_1_std": 0.0563526454473775, "nauc_map_at_20_diff1": 0.5804005223060995, "nauc_map_at_20_max": 0.5579725481672463, "nauc_map_at_20_std": 0.05707539378182075, "nauc_map_at_3_diff1": 0.60131329290114, "nauc_map_at_3_max": 0.5664799406416307, "nauc_map_at_3_std": 0.05032782710221144, "nauc_map_at_5_diff1": 0.5863527663604615, "nauc_map_at_5_max": 0.5583534314009817, "nauc_map_at_5_std": 0.04508261866567717, "nauc_mrr_at_1000_diff1": 0.5810056203148152, "nauc_mrr_at_1000_max": 0.5588819003136871, "nauc_mrr_at_1000_std": 0.059712449754901524, "nauc_mrr_at_100_diff1": 0.5805354442945942, "nauc_mrr_at_100_max": 0.5588426603626263, "nauc_mrr_at_100_std": 0.05977434254548648, "nauc_mrr_at_10_diff1": 0.583233343861228, "nauc_mrr_at_10_max": 0.5601596948204381, "nauc_mrr_at_10_std": 0.05652632363778106, "nauc_mrr_at_1_diff1": 0.6522943384528295, "nauc_mrr_at_1_max": 0.5608308005418264, "nauc_mrr_at_1_std": 0.0563526454473775, "nauc_mrr_at_20_diff1": 0.5804005223060995, "nauc_mrr_at_20_max": 0.5579725481672463, "nauc_mrr_at_20_std": 0.05707539378182075, "nauc_mrr_at_3_diff1": 0.60131329290114, "nauc_mrr_at_3_max": 0.5664799406416307, "nauc_mrr_at_3_std": 0.05032782710221144, "nauc_mrr_at_5_diff1": 0.5863527663604615, "nauc_mrr_at_5_max": 0.5583534314009817, "nauc_mrr_at_5_std": 0.04508261866567717, "nauc_ndcg_at_1000_diff1": 0.5571050456061191, "nauc_ndcg_at_1000_max": 0.5567212027776901, "nauc_ndcg_at_1000_std": 0.08525770642536085, "nauc_ndcg_at_100_diff1": 0.5458349516815053, "nauc_ndcg_at_100_max": 0.5571204347773431, "nauc_ndcg_at_100_std": 0.09310933703884099, "nauc_ndcg_at_10_diff1": 0.5546526730229415, "nauc_ndcg_at_10_max": 0.5557633877636018, "nauc_ndcg_at_10_std": 0.06410434020832524, "nauc_ndcg_at_1_diff1": 0.6522943384528295, "nauc_ndcg_at_1_max": 0.5608308005418264, "nauc_ndcg_at_1_std": 0.0563526454473775, "nauc_ndcg_at_20_diff1": 0.5429081173519749, "nauc_ndcg_at_20_max": 0.5475832962469729, "nauc_ndcg_at_20_std": 0.06805509553296839, "nauc_ndcg_at_3_diff1": 0.5879274570764406, "nauc_ndcg_at_3_max": 0.5663435443475426, "nauc_ndcg_at_3_std": 0.04678024829103677, "nauc_ndcg_at_5_diff1": 0.5610142142367601, "nauc_ndcg_at_5_max": 0.5515072831320688, "nauc_ndcg_at_5_std": 0.03817704892536418, "nauc_precision_at_1000_diff1": 0.2718420701613969, "nauc_precision_at_1000_max": 0.5815659597172199, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 0.35817805383022844, "nauc_precision_at_100_max": 0.571649086699255, "nauc_precision_at_100_std": 0.3839048354098527, "nauc_precision_at_10_diff1": 0.46273076950524145, "nauc_precision_at_10_max": 0.5411228225420862, "nauc_precision_at_10_std": 0.0953835394529327, "nauc_precision_at_1_diff1": 0.6522943384528295, "nauc_precision_at_1_max": 0.5608308005418264, "nauc_precision_at_1_std": 0.0563526454473775, "nauc_precision_at_20_diff1": 0.4042496940764128, "nauc_precision_at_20_max": 0.5047814723611258, "nauc_precision_at_20_std": 0.11872856646473169, "nauc_precision_at_3_diff1": 0.5506419400855919, "nauc_precision_at_3_max": 0.5653697587920712, "nauc_precision_at_3_std": 0.03617793237319149, "nauc_precision_at_5_diff1": 0.48499890648092314, "nauc_precision_at_5_max": 0.5289053542670439, "nauc_precision_at_5_std": 0.017118544095328325, "nauc_recall_at_1000_diff1": 0.271842070161403, "nauc_recall_at_1000_max": 0.5815659597172216, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_100_diff1": 0.35817805383022694, "nauc_recall_at_100_max": 0.5716490866992541, "nauc_recall_at_100_std": 0.38390483540985126, "nauc_recall_at_10_diff1": 0.4627307695052414, "nauc_recall_at_10_max": 0.5411228225420865, "nauc_recall_at_10_std": 0.09538353945293275, "nauc_recall_at_1_diff1": 0.6522943384528295, "nauc_recall_at_1_max": 0.5608308005418264, "nauc_recall_at_1_std": 0.0563526454473775, "nauc_recall_at_20_diff1": 0.4042496940764129, "nauc_recall_at_20_max": 0.5047814723611256, "nauc_recall_at_20_std": 0.11872856646473179, "nauc_recall_at_3_diff1": 0.5506419400855921, "nauc_recall_at_3_max": 0.5653697587920716, "nauc_recall_at_3_std": 0.03617793237319166, "nauc_recall_at_5_diff1": 0.48499890648092314, "nauc_recall_at_5_max": 0.5289053542670441, "nauc_recall_at_5_std": 0.017118544095328505, "ndcg_at_1": 0.305, "ndcg_at_10": 0.43011, "ndcg_at_100": 0.47499, "ndcg_at_1000": 0.49695, "ndcg_at_20": 0.44091, "ndcg_at_3": 0.38375, "ndcg_at_5": 0.4044, "precision_at_1": 0.305, "precision_at_10": 0.0565, "precision_at_100": 0.00795, "precision_at_1000": 0.00097, "precision_at_20": 0.0305, "precision_at_3": 0.145, "precision_at_5": 0.097, "recall_at_1": 0.305, "recall_at_10": 0.565, "recall_at_100": 0.795, "recall_at_1000": 0.965, "recall_at_20": 0.61, "recall_at_3": 0.435, "recall_at_5": 0.485 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.42728, "map_at_1": 0.315, "map_at_10": 0.38977, "map_at_100": 0.39754, "map_at_1000": 0.39831, "map_at_20": 0.39356, "map_at_3": 0.37, "map_at_5": 0.38475, "mrr_at_1": 0.315, "mrr_at_10": 0.3897698412698414, "mrr_at_100": 0.397541530350222, "mrr_at_1000": 0.39831436476698917, "mrr_at_20": 0.39356284197828323, "mrr_at_3": 0.37, "mrr_at_5": 0.3847500000000001, "nauc_map_at_1000_diff1": 0.6597900579101365, "nauc_map_at_1000_max": 0.4743814969593238, "nauc_map_at_1000_std": 0.04456937650037774, "nauc_map_at_100_diff1": 0.6595578028145409, "nauc_map_at_100_max": 0.4746207529903431, "nauc_map_at_100_std": 0.04469464306791652, "nauc_map_at_10_diff1": 0.6584969505864283, "nauc_map_at_10_max": 0.4726482183566381, "nauc_map_at_10_std": 0.03992466480001028, "nauc_map_at_1_diff1": 0.6806754246551459, "nauc_map_at_1_max": 0.4481595482862911, "nauc_map_at_1_std": -0.03792893019762483, "nauc_map_at_20_diff1": 0.6602038571715491, "nauc_map_at_20_max": 0.4750531319415043, "nauc_map_at_20_std": 0.04278409018479869, "nauc_map_at_3_diff1": 0.6690542521026155, "nauc_map_at_3_max": 0.4683804468855765, "nauc_map_at_3_std": 0.01923821721086023, "nauc_map_at_5_diff1": 0.6675321967565364, "nauc_map_at_5_max": 0.4740915890417542, "nauc_map_at_5_std": 0.03970725804693442, "nauc_mrr_at_1000_diff1": 0.6597900579101365, "nauc_mrr_at_1000_max": 0.4743814969593238, "nauc_mrr_at_1000_std": 0.04456937650037774, "nauc_mrr_at_100_diff1": 0.6595578028145409, "nauc_mrr_at_100_max": 0.4746207529903431, "nauc_mrr_at_100_std": 0.04469464306791652, "nauc_mrr_at_10_diff1": 0.6584969505864283, "nauc_mrr_at_10_max": 0.4726482183566381, "nauc_mrr_at_10_std": 0.03992466480001028, "nauc_mrr_at_1_diff1": 0.6806754246551459, "nauc_mrr_at_1_max": 0.4481595482862911, "nauc_mrr_at_1_std": -0.03792893019762483, "nauc_mrr_at_20_diff1": 0.6602038571715491, "nauc_mrr_at_20_max": 0.4750531319415043, "nauc_mrr_at_20_std": 0.04278409018479869, "nauc_mrr_at_3_diff1": 0.6690542521026155, "nauc_mrr_at_3_max": 0.4683804468855765, "nauc_mrr_at_3_std": 0.01923821721086023, "nauc_mrr_at_5_diff1": 0.6675321967565364, "nauc_mrr_at_5_max": 0.4740915890417542, "nauc_mrr_at_5_std": 0.03970725804693442, "nauc_ndcg_at_1000_diff1": 0.642790367735302, "nauc_ndcg_at_1000_max": 0.4799335985943758, "nauc_ndcg_at_1000_std": 0.09173020382279143, "nauc_ndcg_at_100_diff1": 0.6358413267468765, "nauc_ndcg_at_100_max": 0.482234434494922, "nauc_ndcg_at_100_std": 0.10276963770578404, "nauc_ndcg_at_10_diff1": 0.6357168723135498, "nauc_ndcg_at_10_max": 0.47785124426174147, "nauc_ndcg_at_10_std": 0.07606618544656002, "nauc_ndcg_at_1_diff1": 0.6806754246551459, "nauc_ndcg_at_1_max": 0.4481595482862911, "nauc_ndcg_at_1_std": -0.03792893019762483, "nauc_ndcg_at_20_diff1": 0.6416960939448727, "nauc_ndcg_at_20_max": 0.48480138347365553, "nauc_ndcg_at_20_std": 0.08688008419865503, "nauc_ndcg_at_3_diff1": 0.6617572971976259, "nauc_ndcg_at_3_max": 0.4706892075831608, "nauc_ndcg_at_3_std": 0.03690585168881346, "nauc_ndcg_at_5_diff1": 0.6585927852329279, "nauc_ndcg_at_5_max": 0.48058093705348043, "nauc_ndcg_at_5_std": 0.07384049008083444, "nauc_precision_at_1000_diff1": 0.3869281045751624, "nauc_precision_at_1000_max": 0.562231559290378, "nauc_precision_at_1000_std": 0.8977591036414542, "nauc_precision_at_100_diff1": 0.49931859479103574, "nauc_precision_at_100_max": 0.515585792160596, "nauc_precision_at_100_std": 0.44370295059271386, "nauc_precision_at_10_diff1": 0.55474625865067, "nauc_precision_at_10_max": 0.49297820485065996, "nauc_precision_at_10_std": 0.19283225207231972, "nauc_precision_at_1_diff1": 0.6806754246551459, "nauc_precision_at_1_max": 0.4481595482862911, "nauc_precision_at_1_std": -0.03792893019762483, "nauc_precision_at_20_diff1": 0.5748392264903835, "nauc_precision_at_20_max": 0.5196634050294592, "nauc_precision_at_20_std": 0.24749013647967752, "nauc_precision_at_3_diff1": 0.6397987680743332, "nauc_precision_at_3_max": 0.47603356475439784, "nauc_precision_at_3_std": 0.08725922639244146, "nauc_precision_at_5_diff1": 0.6306978363790986, "nauc_precision_at_5_max": 0.4993337038698742, "nauc_precision_at_5_std": 0.17841334467264405, "nauc_recall_at_1000_diff1": 0.38692810457516086, "nauc_recall_at_1000_max": 0.5622315592903827, "nauc_recall_at_1000_std": 0.8977591036414535, "nauc_recall_at_100_diff1": 0.4993185947910358, "nauc_recall_at_100_max": 0.5155857921605947, "nauc_recall_at_100_std": 0.4437029505927141, "nauc_recall_at_10_diff1": 0.5547462586506702, "nauc_recall_at_10_max": 0.49297820485066035, "nauc_recall_at_10_std": 0.19283225207231988, "nauc_recall_at_1_diff1": 0.6806754246551459, "nauc_recall_at_1_max": 0.4481595482862911, "nauc_recall_at_1_std": -0.03792893019762483, "nauc_recall_at_20_diff1": 0.5748392264903833, "nauc_recall_at_20_max": 0.5196634050294594, "nauc_recall_at_20_std": 0.2474901364796777, "nauc_recall_at_3_diff1": 0.6397987680743328, "nauc_recall_at_3_max": 0.47603356475439795, "nauc_recall_at_3_std": 0.08725922639244142, "nauc_recall_at_5_diff1": 0.6306978363790989, "nauc_recall_at_5_max": 0.4993337038698746, "nauc_recall_at_5_std": 0.17841334467264416, "ndcg_at_1": 0.315, "ndcg_at_10": 0.42728, "ndcg_at_100": 0.47081, "ndcg_at_1000": 0.49414, "ndcg_at_20": 0.44197, "ndcg_at_3": 0.38797, "ndcg_at_5": 0.41465, "precision_at_1": 0.315, "precision_at_10": 0.0545, "precision_at_100": 0.0076, "precision_at_1000": 0.00095, "precision_at_20": 0.03025, "precision_at_3": 0.14667, "precision_at_5": 0.101, "recall_at_1": 0.315, "recall_at_10": 0.545, "recall_at_100": 0.76, "recall_at_1000": 0.95, "recall_at_20": 0.605, "recall_at_3": 0.44, "recall_at_5": 0.505 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 3.1353759765625, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.80412, "map_at_1": 0.67532, "map_at_10": 0.76239, "map_at_100": 0.7653, "map_at_1000": 0.7653, "map_at_20": 0.76444, "map_at_3": 0.74459, "map_at_5": 0.75368, "mrr_at_1": 0.6753246753246753, "mrr_at_10": 0.7623891981034839, "mrr_at_100": 0.765300226623, "mrr_at_1000": 0.765300226623, "mrr_at_20": 0.7644356365135587, "mrr_at_3": 0.7445887445887447, "mrr_at_5": 0.7536796536796538, "nauc_map_at_1000_diff1": 0.5979237469677664, "nauc_map_at_1000_max": 0.2924115950702348, "nauc_map_at_1000_std": -0.17318296732546787, "nauc_map_at_100_diff1": 0.5979237469677664, "nauc_map_at_100_max": 0.2924115950702348, "nauc_map_at_100_std": -0.17318296732546787, "nauc_map_at_10_diff1": 0.596903792453234, "nauc_map_at_10_max": 0.2933935645857382, "nauc_map_at_10_std": -0.1870047943320845, "nauc_map_at_1_diff1": 0.6562465610611244, "nauc_map_at_1_max": 0.2923486241773153, "nauc_map_at_1_std": -0.14113187264225374, "nauc_map_at_20_diff1": 0.5948084784833071, "nauc_map_at_20_max": 0.2898190873177562, "nauc_map_at_20_std": -0.17399345717993836, "nauc_map_at_3_diff1": 0.5882399698411667, "nauc_map_at_3_max": 0.3008896632226807, "nauc_map_at_3_std": -0.15775222111292986, "nauc_map_at_5_diff1": 0.6011825802675416, "nauc_map_at_5_max": 0.2896018482846817, "nauc_map_at_5_std": -0.1893640157933465, "nauc_mrr_at_1000_diff1": 0.5979237469677664, "nauc_mrr_at_1000_max": 0.2924115950702348, "nauc_mrr_at_1000_std": -0.17318296732546787, "nauc_mrr_at_100_diff1": 0.5979237469677664, "nauc_mrr_at_100_max": 0.2924115950702348, "nauc_mrr_at_100_std": -0.17318296732546787, "nauc_mrr_at_10_diff1": 0.596903792453234, "nauc_mrr_at_10_max": 0.2933935645857382, "nauc_mrr_at_10_std": -0.1870047943320845, "nauc_mrr_at_1_diff1": 0.6562465610611244, "nauc_mrr_at_1_max": 0.2923486241773153, "nauc_mrr_at_1_std": -0.14113187264225374, "nauc_mrr_at_20_diff1": 0.5948084784833071, "nauc_mrr_at_20_max": 0.2898190873177562, "nauc_mrr_at_20_std": -0.17399345717993836, "nauc_mrr_at_3_diff1": 0.5882399698411667, "nauc_mrr_at_3_max": 0.3008896632226807, "nauc_mrr_at_3_std": -0.15775222111292986, "nauc_mrr_at_5_diff1": 0.6011825802675416, "nauc_mrr_at_5_max": 0.2896018482846817, "nauc_mrr_at_5_std": -0.1893640157933465, "nauc_ndcg_at_1000_diff1": 0.5863829285899035, "nauc_ndcg_at_1000_max": 0.2862629130506407, "nauc_ndcg_at_1000_std": -0.1801172856670342, "nauc_ndcg_at_100_diff1": 0.5863829285899035, "nauc_ndcg_at_100_max": 0.2862629130506407, "nauc_ndcg_at_100_std": -0.1801172856670342, "nauc_ndcg_at_10_diff1": 0.5691397820536505, "nauc_ndcg_at_10_max": 0.27936017201735985, "nauc_ndcg_at_10_std": -0.23489702342436491, "nauc_ndcg_at_1_diff1": 0.6562465610611244, "nauc_ndcg_at_1_max": 0.2923486241773153, "nauc_ndcg_at_1_std": -0.14113187264225374, "nauc_ndcg_at_20_diff1": 0.5596142899845385, "nauc_ndcg_at_20_max": 0.2643206426149416, "nauc_ndcg_at_20_std": -0.1841509467955105, "nauc_ndcg_at_3_diff1": 0.5594550811473704, "nauc_ndcg_at_3_max": 0.2957261118765833, "nauc_ndcg_at_3_std": -0.1669115648012619, "nauc_ndcg_at_5_diff1": 0.5816286642082346, "nauc_ndcg_at_5_max": 0.2715504531622847, "nauc_ndcg_at_5_std": -0.231577091224693, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.31051762722087517, "nauc_precision_at_10_max": 0.13450912346094948, "nauc_precision_at_10_std": -0.7262196475190367, "nauc_precision_at_1_diff1": 0.6562465610611244, "nauc_precision_at_1_max": 0.2923486241773153, "nauc_precision_at_1_std": -0.14113187264225374, "nauc_precision_at_20_diff1": -0.012909070638074602, "nauc_precision_at_20_max": -0.20163456083414846, "nauc_precision_at_20_std": -0.2453969933433873, "nauc_precision_at_3_diff1": 0.44361177715468525, "nauc_precision_at_3_max": 0.2714359789930319, "nauc_precision_at_3_std": -0.20462224480229224, "nauc_precision_at_5_diff1": 0.4887066625206391, "nauc_precision_at_5_max": 0.1715034869029589, "nauc_precision_at_5_std": -0.45582172416288225, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.31051762722087156, "nauc_recall_at_10_max": 0.13450912346094304, "nauc_recall_at_10_std": -0.7262196475190429, "nauc_recall_at_1_diff1": 0.6562465610611244, "nauc_recall_at_1_max": 0.2923486241773153, "nauc_recall_at_1_std": -0.14113187264225374, "nauc_recall_at_20_diff1": -0.012909070638077955, "nauc_recall_at_20_max": -0.20163456083415854, "nauc_recall_at_20_std": -0.2453969933433915, "nauc_recall_at_3_diff1": 0.4436117771546839, "nauc_recall_at_3_max": 0.27143597899302946, "nauc_recall_at_3_std": -0.2046222448022942, "nauc_recall_at_5_diff1": 0.4887066625206389, "nauc_recall_at_5_max": 0.1715034869029567, "nauc_recall_at_5_std": -0.4558217241628853, "ndcg_at_1": 0.67532, "ndcg_at_10": 0.80412, "ndcg_at_100": 0.81803, "ndcg_at_1000": 0.81803, "ndcg_at_20": 0.81099, "ndcg_at_3": 0.76685, "ndcg_at_5": 0.78306, "precision_at_1": 0.67532, "precision_at_10": 0.09351, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04805, "precision_at_3": 0.27706, "precision_at_5": 0.17403, "recall_at_1": 0.67532, "recall_at_10": 0.93506, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.96104, "recall_at_3": 0.83117, "recall_at_5": 0.87013 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 14.820614337921143, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.36075, "map_at_1": 0.20869, "map_at_10": 0.31251, "map_at_100": 0.32829, "map_at_1000": 0.3295, "map_at_20": 0.32044, "map_at_3": 0.27927, "map_at_5": 0.29813, "mrr_at_1": 0.2981651376146789, "mrr_at_10": 0.36110868404446383, "mrr_at_100": 0.37293569979489216, "mrr_at_1000": 0.37373096839382697, "mrr_at_20": 0.36738264151456124, "mrr_at_3": 0.3399592252803263, "mrr_at_5": 0.3518093781855251, "nauc_map_at_1000_diff1": 0.4362409385792419, "nauc_map_at_1000_max": 0.1800486478837926, "nauc_map_at_1000_std": -0.26553196504621107, "nauc_map_at_100_diff1": 0.43620257998412865, "nauc_map_at_100_max": 0.18041121730191945, "nauc_map_at_100_std": -0.2647152938362735, "nauc_map_at_10_diff1": 0.4365198864276503, "nauc_map_at_10_max": 0.17654502923161086, "nauc_map_at_10_std": -0.27053686219361156, "nauc_map_at_1_diff1": 0.4775455448698503, "nauc_map_at_1_max": 0.14025200235237892, "nauc_map_at_1_std": -0.24531006529699495, "nauc_map_at_20_diff1": 0.4372089574951557, "nauc_map_at_20_max": 0.17841296368749265, "nauc_map_at_20_std": -0.27054609403498514, "nauc_map_at_3_diff1": 0.43961166664937285, "nauc_map_at_3_max": 0.1732670244304699, "nauc_map_at_3_std": -0.2687298321809951, "nauc_map_at_5_diff1": 0.43149891318406053, "nauc_map_at_5_max": 0.1698123078445682, "nauc_map_at_5_std": -0.27170327177369624, "nauc_mrr_at_1000_diff1": 0.4624744241639784, "nauc_mrr_at_1000_max": 0.1731885093497636, "nauc_mrr_at_1000_std": -0.26682701100639095, "nauc_mrr_at_100_diff1": 0.46224820778311054, "nauc_mrr_at_100_max": 0.17381712611397718, "nauc_mrr_at_100_std": -0.2658464744036903, "nauc_mrr_at_10_diff1": 0.4646238026569186, "nauc_mrr_at_10_max": 0.17108141117685932, "nauc_mrr_at_10_std": -0.2709506520398262, "nauc_mrr_at_1_diff1": 0.4795989047978672, "nauc_mrr_at_1_max": 0.1613152106252518, "nauc_mrr_at_1_std": -0.2794729237371355, "nauc_mrr_at_20_diff1": 0.4625648951242374, "nauc_mrr_at_20_max": 0.17240416545488282, "nauc_mrr_at_20_std": -0.26933954811207755, "nauc_mrr_at_3_diff1": 0.46485773566608873, "nauc_mrr_at_3_max": 0.16259516847114316, "nauc_mrr_at_3_std": -0.2792100954377753, "nauc_mrr_at_5_diff1": 0.4634993684884953, "nauc_mrr_at_5_max": 0.16570511771454952, "nauc_mrr_at_5_std": -0.2776998684810224, "nauc_ndcg_at_1000_diff1": 0.4369497458665596, "nauc_ndcg_at_1000_max": 0.1915032282436996, "nauc_ndcg_at_1000_std": -0.24616957979834705, "nauc_ndcg_at_100_diff1": 0.4309501960061425, "nauc_ndcg_at_100_max": 0.2039078472412195, "nauc_ndcg_at_100_std": -0.22259433873748213, "nauc_ndcg_at_10_diff1": 0.4391876806744528, "nauc_ndcg_at_10_max": 0.18210353909233828, "nauc_ndcg_at_10_std": -0.26517297046778904, "nauc_ndcg_at_1_diff1": 0.4739748753019021, "nauc_ndcg_at_1_max": 0.16470793869325873, "nauc_ndcg_at_1_std": -0.2736602237293781, "nauc_ndcg_at_20_diff1": 0.43610809749086343, "nauc_ndcg_at_20_max": 0.18705830469730586, "nauc_ndcg_at_20_std": -0.262436971587048, "nauc_ndcg_at_3_diff1": 0.4404524148865511, "nauc_ndcg_at_3_max": 0.16685181632799478, "nauc_ndcg_at_3_std": -0.2760472904229075, "nauc_ndcg_at_5_diff1": 0.43175905189077163, "nauc_ndcg_at_5_max": 0.16802255171664907, "nauc_ndcg_at_5_std": -0.27313213116694535, "nauc_precision_at_1000_diff1": 0.005738353521907607, "nauc_precision_at_1000_max": 0.12700703924892656, "nauc_precision_at_1000_std": 0.07119374603768011, "nauc_precision_at_100_diff1": 0.11322430246123903, "nauc_precision_at_100_max": 0.20523244705487917, "nauc_precision_at_100_std": 0.06400716793210835, "nauc_precision_at_10_diff1": 0.28455008810078686, "nauc_precision_at_10_max": 0.1643627103941248, "nauc_precision_at_10_std": -0.21860014184635101, "nauc_precision_at_1_diff1": 0.4739748753019021, "nauc_precision_at_1_max": 0.16470793869325873, "nauc_precision_at_1_std": -0.2736602237293781, "nauc_precision_at_20_diff1": 0.25222565983172496, "nauc_precision_at_20_max": 0.16477777191116325, "nauc_precision_at_20_std": -0.18651987868268488, "nauc_precision_at_3_diff1": 0.3378333070589199, "nauc_precision_at_3_max": 0.18122173410071726, "nauc_precision_at_3_std": -0.2567043215963174, "nauc_precision_at_5_diff1": 0.290729541778719, "nauc_precision_at_5_max": 0.1554550049956641, "nauc_precision_at_5_std": -0.24180845484190416, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.3078252171240789, "nauc_recall_at_100_max": 0.3384084172097074, "nauc_recall_at_100_std": 0.07928212220425672, "nauc_recall_at_10_diff1": 0.3988096195861225, "nauc_recall_at_10_max": 0.19381012460429922, "nauc_recall_at_10_std": -0.23436828228388612, "nauc_recall_at_1_diff1": 0.4775455448698503, "nauc_recall_at_1_max": 0.14025200235237892, "nauc_recall_at_1_std": -0.24531006529699495, "nauc_recall_at_20_diff1": 0.37800523343844544, "nauc_recall_at_20_max": 0.2084767289752829, "nauc_recall_at_20_std": -0.22265254957557698, "nauc_recall_at_3_diff1": 0.4118684124430158, "nauc_recall_at_3_max": 0.15919744597863614, "nauc_recall_at_3_std": -0.26957065331521735, "nauc_recall_at_5_diff1": 0.3872357077075556, "nauc_recall_at_5_max": 0.15746330901970873, "nauc_recall_at_5_std": -0.2632903690243722, "ndcg_at_1": 0.29969, "ndcg_at_10": 0.36075, "ndcg_at_100": 0.438, "ndcg_at_1000": 0.46551, "ndcg_at_20": 0.38676, "ndcg_at_3": 0.30866, "ndcg_at_5": 0.33199, "precision_at_1": 0.29969, "precision_at_10": 0.07339, "precision_at_100": 0.01275, "precision_at_1000": 0.00156, "precision_at_20": 0.04373, "precision_at_3": 0.16769, "precision_at_5": 0.12049, "recall_at_1": 0.20869, "recall_at_10": 0.4565, "recall_at_100": 0.80196, "recall_at_1000": 1.0, "recall_at_20": 0.55059, "recall_at_3": 0.32164, "recall_at_5": 0.37734 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.34097, "map_at_1": 0.15329, "map_at_10": 0.28211, "map_at_100": 0.30033, "map_at_1000": 0.30226, "map_at_20": 0.29105, "map_at_3": 0.23869, "map_at_5": 0.26591, "mrr_at_1": 0.2599388379204893, "mrr_at_10": 0.34867057424396874, "mrr_at_100": 0.35994203378643586, "mrr_at_1000": 0.36064997729462694, "mrr_at_20": 0.3539873685171496, "mrr_at_3": 0.3246687054026505, "mrr_at_5": 0.33904179408766577, "nauc_map_at_1000_diff1": 0.4022637216689636, "nauc_map_at_1000_max": 0.011405819834518141, "nauc_map_at_1000_std": -0.3304653513589999, "nauc_map_at_100_diff1": 0.4015497416795462, "nauc_map_at_100_max": 0.00999834034490326, "nauc_map_at_100_std": -0.3315000593794023, "nauc_map_at_10_diff1": 0.4050857717148575, "nauc_map_at_10_max": 0.007792945212886593, "nauc_map_at_10_std": -0.33401676582266493, "nauc_map_at_1_diff1": 0.49852568637580613, "nauc_map_at_1_max": 0.02404894735225604, "nauc_map_at_1_std": -0.2958887726957041, "nauc_map_at_20_diff1": 0.40291247339467057, "nauc_map_at_20_max": 0.008587253687752358, "nauc_map_at_20_std": -0.33421441567561455, "nauc_map_at_3_diff1": 0.4222013776089469, "nauc_map_at_3_max": 0.010136085993912492, "nauc_map_at_3_std": -0.3217815240658463, "nauc_map_at_5_diff1": 0.40647453183337523, "nauc_map_at_5_max": 0.00784040840222447, "nauc_map_at_5_std": -0.333646684697192, "nauc_mrr_at_1000_diff1": 0.42172847950756187, "nauc_mrr_at_1000_max": 0.030659369617223816, "nauc_mrr_at_1000_std": -0.3294606914183562, "nauc_mrr_at_100_diff1": 0.42121093117322805, "nauc_mrr_at_100_max": 0.030771466732591353, "nauc_mrr_at_100_std": -0.32908633822077565, "nauc_mrr_at_10_diff1": 0.42269462574679667, "nauc_mrr_at_10_max": 0.02890850927456576, "nauc_mrr_at_10_std": -0.33308852547287354, "nauc_mrr_at_1_diff1": 0.4932698402485779, "nauc_mrr_at_1_max": 0.057147760742959436, "nauc_mrr_at_1_std": -0.33036442287893925, "nauc_mrr_at_20_diff1": 0.4222852986333038, "nauc_mrr_at_20_max": 0.029630418083217754, "nauc_mrr_at_20_std": -0.33077727396662693, "nauc_mrr_at_3_diff1": 0.4221909842276611, "nauc_mrr_at_3_max": 0.027951377591010493, "nauc_mrr_at_3_std": -0.33298500148454946, "nauc_mrr_at_5_diff1": 0.42174716322716244, "nauc_mrr_at_5_max": 0.026063202507198405, "nauc_mrr_at_5_std": -0.33793652527089124, "nauc_ndcg_at_1000_diff1": 0.389985021344595, "nauc_ndcg_at_1000_max": 0.025478227758486245, "nauc_ndcg_at_1000_std": -0.3135560630292576, "nauc_ndcg_at_100_diff1": 0.3736204768886421, "nauc_ndcg_at_100_max": 0.01822183922412112, "nauc_ndcg_at_100_std": -0.3119570550697157, "nauc_ndcg_at_10_diff1": 0.3877510519245756, "nauc_ndcg_at_10_max": 0.004903649764391959, "nauc_ndcg_at_10_std": -0.33622046849680726, "nauc_ndcg_at_1_diff1": 0.4932698402485779, "nauc_ndcg_at_1_max": 0.057147760742959436, "nauc_ndcg_at_1_std": -0.33036442287893925, "nauc_ndcg_at_20_diff1": 0.382337268651163, "nauc_ndcg_at_20_max": 0.005347611162799741, "nauc_ndcg_at_20_std": -0.3333227743975014, "nauc_ndcg_at_3_diff1": 0.3958505613149547, "nauc_ndcg_at_3_max": 0.01333717863280632, "nauc_ndcg_at_3_std": -0.3269820535215641, "nauc_ndcg_at_5_diff1": 0.3891041993512005, "nauc_ndcg_at_5_max": 0.004872403771688151, "nauc_ndcg_at_5_std": -0.33796576122118294, "nauc_precision_at_1000_diff1": 0.018385980950516967, "nauc_precision_at_1000_max": 0.10829418337507772, "nauc_precision_at_1000_std": 0.012377008907229219, "nauc_precision_at_100_diff1": 0.052371030353125736, "nauc_precision_at_100_max": 0.07423047732844952, "nauc_precision_at_100_std": -0.0750545936290214, "nauc_precision_at_10_diff1": 0.19232194764074734, "nauc_precision_at_10_max": -0.0033784513521296126, "nauc_precision_at_10_std": -0.2798787278299724, "nauc_precision_at_1_diff1": 0.4932698402485779, "nauc_precision_at_1_max": 0.057147760742959436, "nauc_precision_at_1_std": -0.33036442287893925, "nauc_precision_at_20_diff1": 0.1628337505030489, "nauc_precision_at_20_max": 0.008810919798039354, "nauc_precision_at_20_std": -0.24592783621714245, "nauc_precision_at_3_diff1": 0.2603627393385583, "nauc_precision_at_3_max": 0.00461726406111989, "nauc_precision_at_3_std": -0.3148008782795308, "nauc_precision_at_5_diff1": 0.21110877531171163, "nauc_precision_at_5_max": -0.0035837827562565593, "nauc_precision_at_5_std": -0.3142870970521295, "nauc_recall_at_1000_diff1": 0.5720905885610953, "nauc_recall_at_1000_max": 0.459646771815093, "nauc_recall_at_1000_std": 0.35316247214157614, "nauc_recall_at_100_diff1": 0.16627657178276536, "nauc_recall_at_100_max": 0.01909966925378612, "nauc_recall_at_100_std": -0.16809671666048576, "nauc_recall_at_10_diff1": 0.3097083306895549, "nauc_recall_at_10_max": -0.0150857823864439, "nauc_recall_at_10_std": -0.3107430801334924, "nauc_recall_at_1_diff1": 0.49852568637580613, "nauc_recall_at_1_max": 0.02404894735225604, "nauc_recall_at_1_std": -0.2958887726957041, "nauc_recall_at_20_diff1": 0.286070471392095, "nauc_recall_at_20_max": -0.022532940637716797, "nauc_recall_at_20_std": -0.30186268870653005, "nauc_recall_at_3_diff1": 0.33628511779923886, "nauc_recall_at_3_max": -0.006004096452956056, "nauc_recall_at_3_std": -0.29302186995925117, "nauc_recall_at_5_diff1": 0.31865956050985655, "nauc_recall_at_5_max": -0.0127304827540012, "nauc_recall_at_5_std": -0.3175634383646668, "ndcg_at_1": 0.25994, "ndcg_at_10": 0.34097, "ndcg_at_100": 0.42407, "ndcg_at_1000": 0.45495, "ndcg_at_20": 0.36704, "ndcg_at_3": 0.28859, "ndcg_at_5": 0.31071, "precision_at_1": 0.25994, "precision_at_10": 0.09098, "precision_at_100": 0.01578, "precision_at_1000": 0.00195, "precision_at_20": 0.05413, "precision_at_3": 0.19113, "precision_at_5": 0.14618, "recall_at_1": 0.15329, "recall_at_10": 0.4367, "recall_at_100": 0.79322, "recall_at_1000": 0.99592, "recall_at_20": 0.52339, "recall_at_3": 0.29177, "recall_at_5": 0.36307 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.31949, "map_at_1": 0.17277, "map_at_10": 0.2702, "map_at_100": 0.28856, "map_at_1000": 0.28995, "map_at_20": 0.27957, "map_at_3": 0.24088, "map_at_5": 0.25708, "mrr_at_1": 0.24267100977198697, "mrr_at_10": 0.31691872188614845, "mrr_at_100": 0.328413410224604, "mrr_at_1000": 0.329229369589827, "mrr_at_20": 0.3228330336041533, "mrr_at_3": 0.2926167209554832, "mrr_at_5": 0.3045874049945711, "nauc_map_at_1000_diff1": 0.41505090394048866, "nauc_map_at_1000_max": 0.13241336388494718, "nauc_map_at_1000_std": -0.19201020800120797, "nauc_map_at_100_diff1": 0.4145118451334163, "nauc_map_at_100_max": 0.13241089525446442, "nauc_map_at_100_std": -0.19148643172776678, "nauc_map_at_10_diff1": 0.41401562649660273, "nauc_map_at_10_max": 0.13134661521994792, "nauc_map_at_10_std": -0.19904013040457355, "nauc_map_at_1_diff1": 0.4925810221309357, "nauc_map_at_1_max": 0.12167560986697107, "nauc_map_at_1_std": -0.16440428374787602, "nauc_map_at_20_diff1": 0.4140400331213615, "nauc_map_at_20_max": 0.1287694349146764, "nauc_map_at_20_std": -0.2002561977628548, "nauc_map_at_3_diff1": 0.42367078522749024, "nauc_map_at_3_max": 0.1335653702272776, "nauc_map_at_3_std": -0.19622013982861383, "nauc_map_at_5_diff1": 0.41654803369714793, "nauc_map_at_5_max": 0.1286532915530522, "nauc_map_at_5_std": -0.2001912647201993, "nauc_mrr_at_1000_diff1": 0.45183780463327433, "nauc_mrr_at_1000_max": 0.14496507835048258, "nauc_mrr_at_1000_std": -0.1765942333681556, "nauc_mrr_at_100_diff1": 0.45163588502609076, "nauc_mrr_at_100_max": 0.14538390414927446, "nauc_mrr_at_100_std": -0.17569220940165448, "nauc_mrr_at_10_diff1": 0.4529239223346786, "nauc_mrr_at_10_max": 0.14284857231467374, "nauc_mrr_at_10_std": -0.18221641068311575, "nauc_mrr_at_1_diff1": 0.4933141110384315, "nauc_mrr_at_1_max": 0.1470220161647062, "nauc_mrr_at_1_std": -0.18184622537720502, "nauc_mrr_at_20_diff1": 0.45170537665016325, "nauc_mrr_at_20_max": 0.14353492025833842, "nauc_mrr_at_20_std": -0.18058264159515347, "nauc_mrr_at_3_diff1": 0.4569265963743151, "nauc_mrr_at_3_max": 0.13946392741551988, "nauc_mrr_at_3_std": -0.18887643809079077, "nauc_mrr_at_5_diff1": 0.45306740774381393, "nauc_mrr_at_5_max": 0.14065915445105157, "nauc_mrr_at_5_std": -0.185550522817219, "nauc_ndcg_at_1000_diff1": 0.41635722113714646, "nauc_ndcg_at_1000_max": 0.14770010488401367, "nauc_ndcg_at_1000_std": -0.16435648756657403, "nauc_ndcg_at_100_diff1": 0.40619177981217786, "nauc_ndcg_at_100_max": 0.1556090922438858, "nauc_ndcg_at_100_std": -0.14259999248239374, "nauc_ndcg_at_10_diff1": 0.40757123471236806, "nauc_ndcg_at_10_max": 0.13381733405090998, "nauc_ndcg_at_10_std": -0.19595078611831573, "nauc_ndcg_at_1_diff1": 0.4933141110384315, "nauc_ndcg_at_1_max": 0.1470220161647062, "nauc_ndcg_at_1_std": -0.18184622537720502, "nauc_ndcg_at_20_diff1": 0.4048954589210238, "nauc_ndcg_at_20_max": 0.1297128681452499, "nauc_ndcg_at_20_std": -0.19617046481394884, "nauc_ndcg_at_3_diff1": 0.4234307694909729, "nauc_ndcg_at_3_max": 0.13498935897305633, "nauc_ndcg_at_3_std": -0.1983067983598234, "nauc_ndcg_at_5_diff1": 0.41299148536389446, "nauc_ndcg_at_5_max": 0.12908388749749589, "nauc_ndcg_at_5_std": -0.2007748056600153, "nauc_precision_at_1000_diff1": 0.04861193672132261, "nauc_precision_at_1000_max": 0.16256377281794238, "nauc_precision_at_1000_std": 0.14917420534621734, "nauc_precision_at_100_diff1": 0.11837321725063916, "nauc_precision_at_100_max": 0.20102015016691627, "nauc_precision_at_100_std": 0.14444275898938244, "nauc_precision_at_10_diff1": 0.26536633318911446, "nauc_precision_at_10_max": 0.11926061953347789, "nauc_precision_at_10_std": -0.15367486499405267, "nauc_precision_at_1_diff1": 0.4933141110384315, "nauc_precision_at_1_max": 0.1470220161647062, "nauc_precision_at_1_std": -0.18184622537720502, "nauc_precision_at_20_diff1": 0.22872657885495534, "nauc_precision_at_20_max": 0.0963507558339687, "nauc_precision_at_20_std": -0.12574891572105426, "nauc_precision_at_3_diff1": 0.31992163766273557, "nauc_precision_at_3_max": 0.13552665742834516, "nauc_precision_at_3_std": -0.19881480307611593, "nauc_precision_at_5_diff1": 0.28623210182482245, "nauc_precision_at_5_max": 0.11026835194152086, "nauc_precision_at_5_std": -0.19152693283731337, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.27966581240649957, "nauc_recall_at_100_max": 0.23425675276197166, "nauc_recall_at_100_std": 0.10921505542278978, "nauc_recall_at_10_diff1": 0.33956407535116556, "nauc_recall_at_10_max": 0.12826659208335056, "nauc_recall_at_10_std": -0.1943975091252743, "nauc_recall_at_1_diff1": 0.4925810221309357, "nauc_recall_at_1_max": 0.12167560986697107, "nauc_recall_at_1_std": -0.16440428374787602, "nauc_recall_at_20_diff1": 0.317679503564471, "nauc_recall_at_20_max": 0.11352166597469808, "nauc_recall_at_20_std": -0.19408222099233943, "nauc_recall_at_3_diff1": 0.3757273582663656, "nauc_recall_at_3_max": 0.12682016391977954, "nauc_recall_at_3_std": -0.20341052110217941, "nauc_recall_at_5_diff1": 0.3598059148605256, "nauc_recall_at_5_max": 0.1185482342600925, "nauc_recall_at_5_std": -0.20587844250384485, "ndcg_at_1": 0.24267, "ndcg_at_10": 0.31949, "ndcg_at_100": 0.40277, "ndcg_at_1000": 0.43378, "ndcg_at_20": 0.34741, "ndcg_at_3": 0.26984, "ndcg_at_5": 0.29087, "precision_at_1": 0.24267, "precision_at_10": 0.06954, "precision_at_100": 0.01326, "precision_at_1000": 0.00167, "precision_at_20": 0.043, "precision_at_3": 0.15255, "precision_at_5": 0.11107, "recall_at_1": 0.17277, "recall_at_10": 0.41503, "recall_at_100": 0.77696, "recall_at_1000": 1.0, "recall_at_20": 0.51226, "recall_at_3": 0.29015, "recall_at_5": 0.33854 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/model_meta.json ================================================ {"name": "BAAI/bge-m3", "revision": "no_revision_available", "release_date": null, "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "loader": null} ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 119.05822992324829, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.76535, "map_at_1": 0.58772, "map_at_10": 0.71412, "map_at_100": 0.71727, "map_at_1000": 0.71731, "map_at_20": 0.71588, "map_at_3": 0.69298, "map_at_5": 0.70614, "mrr_at_1": 0.5877192982456141, "mrr_at_10": 0.7141186299081036, "mrr_at_100": 0.7172715798404293, "mrr_at_1000": 0.7173069505252058, "mrr_at_20": 0.7158782372598161, "mrr_at_3": 0.6929824561403509, "mrr_at_5": 0.706140350877193, "nauc_map_at_1000_diff1": 0.6871804548539637, "nauc_map_at_1000_max": 0.1420323864306385, "nauc_map_at_1000_std": -0.4359167933175919, "nauc_map_at_100_diff1": 0.6869888986630723, "nauc_map_at_100_max": 0.14202935472851613, "nauc_map_at_100_std": -0.4361190145277671, "nauc_map_at_10_diff1": 0.6821625682937265, "nauc_map_at_10_max": 0.14282670462128133, "nauc_map_at_10_std": -0.44046306719237877, "nauc_map_at_1_diff1": 0.7565205727235682, "nauc_map_at_1_max": 0.14499014445168806, "nauc_map_at_1_std": -0.4021330787124016, "nauc_map_at_20_diff1": 0.683899005732777, "nauc_map_at_20_max": 0.14037491725835372, "nauc_map_at_20_std": -0.43770795307708554, "nauc_map_at_3_diff1": 0.6857726824218638, "nauc_map_at_3_max": 0.11412180292020804, "nauc_map_at_3_std": -0.44606450164059286, "nauc_map_at_5_diff1": 0.6882116080015668, "nauc_map_at_5_max": 0.13770668919721318, "nauc_map_at_5_std": -0.43159780499446615, "nauc_mrr_at_1000_diff1": 0.6871804548539637, "nauc_mrr_at_1000_max": 0.1420323864306385, "nauc_mrr_at_1000_std": -0.4359167933175919, "nauc_mrr_at_100_diff1": 0.6869888986630723, "nauc_mrr_at_100_max": 0.14202935472851613, "nauc_mrr_at_100_std": -0.4361190145277671, "nauc_mrr_at_10_diff1": 0.6821625682937265, "nauc_mrr_at_10_max": 0.14282670462128133, "nauc_mrr_at_10_std": -0.44046306719237877, "nauc_mrr_at_1_diff1": 0.7565205727235682, "nauc_mrr_at_1_max": 0.14499014445168806, "nauc_mrr_at_1_std": -0.4021330787124016, "nauc_mrr_at_20_diff1": 0.683899005732777, "nauc_mrr_at_20_max": 0.14037491725835372, "nauc_mrr_at_20_std": -0.43770795307708554, "nauc_mrr_at_3_diff1": 0.6857726824218638, "nauc_mrr_at_3_max": 0.11412180292020804, "nauc_mrr_at_3_std": -0.44606450164059286, "nauc_mrr_at_5_diff1": 0.6882116080015668, "nauc_mrr_at_5_max": 0.13770668919721318, "nauc_mrr_at_5_std": -0.43159780499446615, "nauc_ndcg_at_1000_diff1": 0.6725739924867542, "nauc_ndcg_at_1000_max": 0.14475508849337584, "nauc_ndcg_at_1000_std": -0.4473199404587193, "nauc_ndcg_at_100_diff1": 0.6650462445483546, "nauc_ndcg_at_100_max": 0.14461796408541713, "nauc_ndcg_at_100_std": -0.4552912559815156, "nauc_ndcg_at_10_diff1": 0.6424956614815897, "nauc_ndcg_at_10_max": 0.15106663392054379, "nauc_ndcg_at_10_std": -0.4748615636013759, "nauc_ndcg_at_1_diff1": 0.7565205727235682, "nauc_ndcg_at_1_max": 0.14499014445168806, "nauc_ndcg_at_1_std": -0.4021330787124016, "nauc_ndcg_at_20_diff1": 0.6488798602618329, "nauc_ndcg_at_20_max": 0.14049065208385828, "nauc_ndcg_at_20_std": -0.46283163692940793, "nauc_ndcg_at_3_diff1": 0.6574451317211064, "nauc_ndcg_at_3_max": 0.09375508448638213, "nauc_ndcg_at_3_std": -0.4739113630902631, "nauc_ndcg_at_5_diff1": 0.6595895244317779, "nauc_ndcg_at_5_max": 0.13802228056251378, "nauc_ndcg_at_5_std": -0.4481832149672461, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": -0.5637901180614636, "nauc_precision_at_100_max": 0.12223365206034731, "nauc_precision_at_100_std": -1.7565359911142433, "nauc_precision_at_10_diff1": 0.29489340133999126, "nauc_precision_at_10_max": 0.2401706780084637, "nauc_precision_at_10_std": -0.7962881634526331, "nauc_precision_at_1_diff1": 0.7565205727235682, "nauc_precision_at_1_max": 0.14499014445168806, "nauc_precision_at_1_std": -0.4021330787124016, "nauc_precision_at_20_diff1": 0.21143457644733152, "nauc_precision_at_20_max": 0.1353309980220975, "nauc_precision_at_20_std": -0.7794901519427275, "nauc_precision_at_3_diff1": 0.538958335146876, "nauc_precision_at_3_max": 0.0048859242585519765, "nauc_precision_at_3_std": -0.5951020938202185, "nauc_precision_at_5_diff1": 0.5045809293735882, "nauc_precision_at_5_max": 0.14478965602215157, "nauc_precision_at_5_std": -0.538854179166969, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": -0.5637901180615443, "nauc_recall_at_100_max": 0.12223365206031499, "nauc_recall_at_100_std": -1.756535991114336, "nauc_recall_at_10_diff1": 0.2948934013399936, "nauc_recall_at_10_max": 0.24017067800846637, "nauc_recall_at_10_std": -0.7962881634526358, "nauc_recall_at_1_diff1": 0.7565205727235682, "nauc_recall_at_1_max": 0.14499014445168806, "nauc_recall_at_1_std": -0.4021330787124016, "nauc_recall_at_20_diff1": 0.21143457644733485, "nauc_recall_at_20_max": 0.13533099802210283, "nauc_recall_at_20_std": -0.7794901519427204, "nauc_recall_at_3_diff1": 0.5389583351468771, "nauc_recall_at_3_max": 0.004885924258552919, "nauc_recall_at_3_std": -0.5951020938202172, "nauc_recall_at_5_diff1": 0.5045809293735892, "nauc_recall_at_5_max": 0.14478965602215235, "nauc_recall_at_5_std": -0.5388541791669698, "ndcg_at_1": 0.58772, "ndcg_at_10": 0.76535, "ndcg_at_100": 0.78048, "ndcg_at_1000": 0.78158, "ndcg_at_20": 0.77193, "ndcg_at_3": 0.72472, "ndcg_at_5": 0.74739, "precision_at_1": 0.58772, "precision_at_10": 0.09211, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "precision_at_20": 0.04737, "precision_at_3": 0.27193, "precision_at_5": 0.17368, "recall_at_1": 0.58772, "recall_at_10": 0.92105, "recall_at_100": 0.99123, "recall_at_1000": 1.0, "recall_at_20": 0.94737, "recall_at_3": 0.81579, "recall_at_5": 0.86842 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 195.57231187820435, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.95001, "map_at_1": 0.90778, "map_at_10": 0.93791, "map_at_100": 0.93841, "map_at_1000": 0.93842, "map_at_20": 0.93826, "map_at_3": 0.93259, "map_at_5": 0.93676, "mrr_at_1": 0.9077777777777778, "mrr_at_10": 0.937910052910053, "mrr_at_100": 0.9384057688805332, "mrr_at_1000": 0.9384191334793509, "mrr_at_20": 0.9382570947570948, "mrr_at_3": 0.9325925925925926, "mrr_at_5": 0.9367592592592593, "nauc_map_at_1000_diff1": 0.9142665361908336, "nauc_map_at_1000_max": 0.7417200323164244, "nauc_map_at_1000_std": -0.022857413046968294, "nauc_map_at_100_diff1": 0.9142851384743285, "nauc_map_at_100_max": 0.7417760734113105, "nauc_map_at_100_std": -0.02263547539591741, "nauc_map_at_10_diff1": 0.9134947687355001, "nauc_map_at_10_max": 0.741494495824178, "nauc_map_at_10_std": -0.024850704377757292, "nauc_map_at_1_diff1": 0.9127602848368274, "nauc_map_at_1_max": 0.7206529198024576, "nauc_map_at_1_std": -0.05017267951357163, "nauc_map_at_20_diff1": 0.9144115754201003, "nauc_map_at_20_max": 0.7421258068927056, "nauc_map_at_20_std": -0.021967674396242153, "nauc_map_at_3_diff1": 0.9122854269913103, "nauc_map_at_3_max": 0.7422276602948861, "nauc_map_at_3_std": -0.023631247370742646, "nauc_map_at_5_diff1": 0.9141119600597686, "nauc_map_at_5_max": 0.7437446428058785, "nauc_map_at_5_std": -0.020491515298165976, "nauc_mrr_at_1000_diff1": 0.9142665361908336, "nauc_mrr_at_1000_max": 0.7417200323164244, "nauc_mrr_at_1000_std": -0.022857413046968294, "nauc_mrr_at_100_diff1": 0.9142851384743285, "nauc_mrr_at_100_max": 0.7417760734113105, "nauc_mrr_at_100_std": -0.02263547539591741, "nauc_mrr_at_10_diff1": 0.9134947687355001, "nauc_mrr_at_10_max": 0.741494495824178, "nauc_mrr_at_10_std": -0.024850704377757292, "nauc_mrr_at_1_diff1": 0.9127602848368274, "nauc_mrr_at_1_max": 0.7206529198024576, "nauc_mrr_at_1_std": -0.05017267951357163, "nauc_mrr_at_20_diff1": 0.9144115754201003, "nauc_mrr_at_20_max": 0.7421258068927056, "nauc_mrr_at_20_std": -0.021967674396242153, "nauc_mrr_at_3_diff1": 0.9122854269913103, "nauc_mrr_at_3_max": 0.7422276602948861, "nauc_mrr_at_3_std": -0.023631247370742646, "nauc_mrr_at_5_diff1": 0.9141119600597686, "nauc_mrr_at_5_max": 0.7437446428058785, "nauc_mrr_at_5_std": -0.020491515298165976, "nauc_ndcg_at_1000_diff1": 0.9148444589331953, "nauc_ndcg_at_1000_max": 0.7452873562263655, "nauc_ndcg_at_1000_std": -0.017786190784458585, "nauc_ndcg_at_100_diff1": 0.9153780312139375, "nauc_ndcg_at_100_max": 0.7468833487427657, "nauc_ndcg_at_100_std": -0.011408889996697627, "nauc_ndcg_at_10_diff1": 0.9129157695282673, "nauc_ndcg_at_10_max": 0.7474421693216621, "nauc_ndcg_at_10_std": -0.021544911168402143, "nauc_ndcg_at_1_diff1": 0.9127602848368274, "nauc_ndcg_at_1_max": 0.7206529198024576, "nauc_ndcg_at_1_std": -0.05017267951357163, "nauc_ndcg_at_20_diff1": 0.9164480561151565, "nauc_ndcg_at_20_max": 0.7495752715796, "nauc_ndcg_at_20_std": -0.008984213063139969, "nauc_ndcg_at_3_diff1": 0.9111897107164737, "nauc_ndcg_at_3_max": 0.749687976964171, "nauc_ndcg_at_3_std": -0.017244343945449332, "nauc_ndcg_at_5_diff1": 0.9145423364478309, "nauc_ndcg_at_5_max": 0.7534919419643613, "nauc_ndcg_at_5_std": -0.008650222933101416, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.9029333955804523, "nauc_precision_at_10_max": 0.8148537192654776, "nauc_precision_at_10_std": -0.00042794895737402667, "nauc_precision_at_1_diff1": 0.9127602848368274, "nauc_precision_at_1_max": 0.7206529198024576, "nauc_precision_at_1_std": -0.05017267951357163, "nauc_precision_at_20_diff1": 0.9673202614378968, "nauc_precision_at_20_max": 0.8850373482726227, "nauc_precision_at_20_std": 0.27357609710550823, "nauc_precision_at_3_diff1": 0.9052954515139436, "nauc_precision_at_3_max": 0.7854875283446744, "nauc_precision_at_3_std": 0.011497932506332782, "nauc_precision_at_5_diff1": 0.9180549412747513, "nauc_precision_at_5_max": 0.8355201729814675, "nauc_precision_at_5_std": 0.09275640080592862, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 1.0, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 1.0, "nauc_recall_at_10_diff1": 0.9029333955804517, "nauc_recall_at_10_max": 0.8148537192654788, "nauc_recall_at_10_std": -0.00042794895735932413, "nauc_recall_at_1_diff1": 0.9127602848368274, "nauc_recall_at_1_max": 0.7206529198024576, "nauc_recall_at_1_std": -0.05017267951357163, "nauc_recall_at_20_diff1": 0.9673202614379147, "nauc_recall_at_20_max": 0.8850373482726468, "nauc_recall_at_20_std": 0.2735760971055196, "nauc_recall_at_3_diff1": 0.9052954515139391, "nauc_recall_at_3_max": 0.7854875283446687, "nauc_recall_at_3_std": 0.01149793250633776, "nauc_recall_at_5_diff1": 0.9180549412747542, "nauc_recall_at_5_max": 0.8355201729814765, "nauc_recall_at_5_std": 0.0927564008059334, "ndcg_at_1": 0.90778, "ndcg_at_10": 0.95001, "ndcg_at_100": 0.95238, "ndcg_at_1000": 0.95268, "ndcg_at_20": 0.95119, "ndcg_at_3": 0.93997, "ndcg_at_5": 0.94739, "precision_at_1": 0.90778, "precision_at_10": 0.09867, "precision_at_100": 0.00998, "precision_at_1000": 0.001, "precision_at_20": 0.04956, "precision_at_3": 0.32037, "precision_at_5": 0.19578, "recall_at_1": 0.90778, "recall_at_10": 0.98667, "recall_at_100": 0.99778, "recall_at_1000": 1.0, "recall_at_20": 0.99111, "recall_at_3": 0.96111, "recall_at_5": 0.97889 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.9533, "map_at_1": 0.90778, "map_at_10": 0.94111, "map_at_100": 0.94148, "map_at_1000": 0.94149, "map_at_20": 0.94134, "map_at_3": 0.93833, "map_at_5": 0.93944, "mrr_at_1": 0.9077777777777778, "mrr_at_10": 0.9411089065255732, "mrr_at_100": 0.9414824263539285, "mrr_at_1000": 0.9414930083645107, "mrr_at_20": 0.9413410894660895, "mrr_at_3": 0.9383333333333335, "mrr_at_5": 0.9394444444444445, "nauc_map_at_1000_diff1": 0.9132390834178515, "nauc_map_at_1000_max": 0.7034851682270743, "nauc_map_at_1000_std": -0.08533923507942101, "nauc_map_at_100_diff1": 0.9132547728061845, "nauc_map_at_100_max": 0.7035387884114396, "nauc_map_at_100_std": -0.08514296803424397, "nauc_map_at_10_diff1": 0.9131790376965144, "nauc_map_at_10_max": 0.7036961829241862, "nauc_map_at_10_std": -0.08741887301346339, "nauc_map_at_1_diff1": 0.9127602848368274, "nauc_map_at_1_max": 0.6805766483300142, "nauc_map_at_1_std": -0.08971460069971847, "nauc_map_at_20_diff1": 0.9133632060050118, "nauc_map_at_20_max": 0.7039100639974116, "nauc_map_at_20_std": -0.08463651070667616, "nauc_map_at_3_diff1": 0.913022265963442, "nauc_map_at_3_max": 0.7048799499779878, "nauc_map_at_3_std": -0.08527995782897725, "nauc_map_at_5_diff1": 0.9138998106888043, "nauc_map_at_5_max": 0.7084357698227083, "nauc_map_at_5_std": -0.08271157596575836, "nauc_mrr_at_1000_diff1": 0.9132390834178515, "nauc_mrr_at_1000_max": 0.7034851682270743, "nauc_mrr_at_1000_std": -0.08533923507942101, "nauc_mrr_at_100_diff1": 0.9132547728061845, "nauc_mrr_at_100_max": 0.7035387884114396, "nauc_mrr_at_100_std": -0.08514296803424397, "nauc_mrr_at_10_diff1": 0.9131790376965144, "nauc_mrr_at_10_max": 0.7036961829241862, "nauc_mrr_at_10_std": -0.08741887301346339, "nauc_mrr_at_1_diff1": 0.9127602848368274, "nauc_mrr_at_1_max": 0.6805766483300142, "nauc_mrr_at_1_std": -0.08971460069971847, "nauc_mrr_at_20_diff1": 0.9133632060050118, "nauc_mrr_at_20_max": 0.7039100639974116, "nauc_mrr_at_20_std": -0.08463651070667616, "nauc_mrr_at_3_diff1": 0.913022265963442, "nauc_mrr_at_3_max": 0.7048799499779878, "nauc_mrr_at_3_std": -0.08527995782897725, "nauc_mrr_at_5_diff1": 0.9138998106888043, "nauc_mrr_at_5_max": 0.7084357698227083, "nauc_mrr_at_5_std": -0.08271157596575836, "nauc_ndcg_at_1000_diff1": 0.9136586516663353, "nauc_ndcg_at_1000_max": 0.7067346358618749, "nauc_ndcg_at_1000_std": -0.08426696192429434, "nauc_ndcg_at_100_diff1": 0.9139767092971215, "nauc_ndcg_at_100_max": 0.7078149442970522, "nauc_ndcg_at_100_std": -0.08027282252636965, "nauc_ndcg_at_10_diff1": 0.9137398261541717, "nauc_ndcg_at_10_max": 0.7089914704706038, "nauc_ndcg_at_10_std": -0.09076166542013302, "nauc_ndcg_at_1_diff1": 0.9127602848368274, "nauc_ndcg_at_1_max": 0.6805766483300142, "nauc_ndcg_at_1_std": -0.08971460069971847, "nauc_ndcg_at_20_diff1": 0.9147962958143541, "nauc_ndcg_at_20_max": 0.7105830753311271, "nauc_ndcg_at_20_std": -0.07929133989085652, "nauc_ndcg_at_3_diff1": 0.9139755965104456, "nauc_ndcg_at_3_max": 0.7145165315074726, "nauc_ndcg_at_3_std": -0.08322391069107185, "nauc_ndcg_at_5_diff1": 0.9158174675677071, "nauc_ndcg_at_5_max": 0.7221978200003155, "nauc_ndcg_at_5_std": -0.07786087673971333, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.9214130096483081, "nauc_precision_at_10_max": 0.7707749766573202, "nauc_precision_at_10_std": -0.16692602967115508, "nauc_precision_at_1_diff1": 0.9127602848368274, "nauc_precision_at_1_max": 0.6805766483300142, "nauc_precision_at_1_std": -0.08971460069971847, "nauc_precision_at_20_diff1": 0.9564270152505291, "nauc_precision_at_20_max": 0.8513849984437983, "nauc_precision_at_20_std": 0.08667911609085283, "nauc_precision_at_3_diff1": 0.9208294428882671, "nauc_precision_at_3_max": 0.7763383131030174, "nauc_precision_at_3_std": -0.06977513227512878, "nauc_precision_at_5_diff1": 0.9319561157796464, "nauc_precision_at_5_max": 0.8293884220354816, "nauc_precision_at_5_std": -0.03867880485527598, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 1.0, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 1.0, "nauc_recall_at_10_diff1": 0.9214130096482933, "nauc_recall_at_10_max": 0.7707749766573149, "nauc_recall_at_10_std": -0.16692602967111914, "nauc_recall_at_1_diff1": 0.9127602848368274, "nauc_recall_at_1_max": 0.6805766483300142, "nauc_recall_at_1_std": -0.08971460069971847, "nauc_recall_at_20_diff1": 0.9564270152505339, "nauc_recall_at_20_max": 0.8513849984438123, "nauc_recall_at_20_std": 0.08667911609088264, "nauc_recall_at_3_diff1": 0.9208294428882646, "nauc_recall_at_3_max": 0.7763383131030149, "nauc_recall_at_3_std": -0.06977513227513614, "nauc_recall_at_5_diff1": 0.9319561157796439, "nauc_recall_at_5_max": 0.8293884220354797, "nauc_recall_at_5_std": -0.03867880485527428, "ndcg_at_1": 0.90778, "ndcg_at_10": 0.9533, "ndcg_at_100": 0.95517, "ndcg_at_1000": 0.95533, "ndcg_at_20": 0.95414, "ndcg_at_3": 0.94739, "ndcg_at_5": 0.94931, "precision_at_1": 0.90778, "precision_at_10": 0.099, "precision_at_100": 0.00999, "precision_at_1000": 0.001, "precision_at_20": 0.04967, "precision_at_3": 0.32444, "precision_at_5": 0.19556, "recall_at_1": 0.90778, "recall_at_10": 0.99, "recall_at_100": 0.99889, "recall_at_1000": 1.0, "recall_at_20": 0.99333, "recall_at_3": 0.97333, "recall_at_5": 0.97778 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.92542, "map_at_1": 0.86889, "map_at_10": 0.90865, "map_at_100": 0.90952, "map_at_1000": 0.90955, "map_at_20": 0.90934, "map_at_3": 0.90167, "map_at_5": 0.90644, "mrr_at_1": 0.8688888888888889, "mrr_at_10": 0.9086534391534392, "mrr_at_100": 0.9095160509349679, "mrr_at_1000": 0.9095506290733619, "mrr_at_20": 0.9093372640192817, "mrr_at_3": 0.901666666666667, "mrr_at_5": 0.9064444444444446, "nauc_map_at_1000_diff1": 0.8655071414135598, "nauc_map_at_1000_max": 0.712880044053201, "nauc_map_at_1000_std": 0.07778526277498768, "nauc_map_at_100_diff1": 0.8655016531537733, "nauc_map_at_100_max": 0.712992288401549, "nauc_map_at_100_std": 0.07814578592941561, "nauc_map_at_10_diff1": 0.8655264893655531, "nauc_map_at_10_max": 0.7144121638788207, "nauc_map_at_10_std": 0.08036728429197564, "nauc_map_at_1_diff1": 0.8740098646066407, "nauc_map_at_1_max": 0.6846061506802336, "nauc_map_at_1_std": 0.016475036338989085, "nauc_map_at_20_diff1": 0.8655606749633783, "nauc_map_at_20_max": 0.7133069583346271, "nauc_map_at_20_std": 0.07895951036393203, "nauc_map_at_3_diff1": 0.8629935666435051, "nauc_map_at_3_max": 0.7143413754778334, "nauc_map_at_3_std": 0.0809241912518571, "nauc_map_at_5_diff1": 0.8652061970583541, "nauc_map_at_5_max": 0.7163535037743536, "nauc_map_at_5_std": 0.07951637719503098, "nauc_mrr_at_1000_diff1": 0.8655071414135598, "nauc_mrr_at_1000_max": 0.712880044053201, "nauc_mrr_at_1000_std": 0.07778526277498768, "nauc_mrr_at_100_diff1": 0.8655016531537733, "nauc_mrr_at_100_max": 0.712992288401549, "nauc_mrr_at_100_std": 0.07814578592941561, "nauc_mrr_at_10_diff1": 0.8655264893655531, "nauc_mrr_at_10_max": 0.7144121638788207, "nauc_mrr_at_10_std": 0.08036728429197564, "nauc_mrr_at_1_diff1": 0.8740098646066407, "nauc_mrr_at_1_max": 0.6846061506802336, "nauc_mrr_at_1_std": 0.016475036338989085, "nauc_mrr_at_20_diff1": 0.8655606749633783, "nauc_mrr_at_20_max": 0.7133069583346271, "nauc_mrr_at_20_std": 0.07895951036393203, "nauc_mrr_at_3_diff1": 0.8629935666435051, "nauc_mrr_at_3_max": 0.7143413754778334, "nauc_mrr_at_3_std": 0.0809241912518571, "nauc_mrr_at_5_diff1": 0.8652061970583541, "nauc_mrr_at_5_max": 0.7163535037743536, "nauc_mrr_at_5_std": 0.07951637719503098, "nauc_ndcg_at_1000_diff1": 0.8647471328268296, "nauc_ndcg_at_1000_max": 0.7185203395380872, "nauc_ndcg_at_1000_std": 0.09124983250276489, "nauc_ndcg_at_100_diff1": 0.8644783270567897, "nauc_ndcg_at_100_max": 0.7215603085795872, "nauc_ndcg_at_100_std": 0.10106429785744557, "nauc_ndcg_at_10_diff1": 0.8648663071038122, "nauc_ndcg_at_10_max": 0.7282776538048082, "nauc_ndcg_at_10_std": 0.11073311323860291, "nauc_ndcg_at_1_diff1": 0.8740098646066407, "nauc_ndcg_at_1_max": 0.6846061506802336, "nauc_ndcg_at_1_std": 0.016475036338989085, "nauc_ndcg_at_20_diff1": 0.8649366008500634, "nauc_ndcg_at_20_max": 0.7236249483095551, "nauc_ndcg_at_20_std": 0.10537703607809089, "nauc_ndcg_at_3_diff1": 0.8591692392430215, "nauc_ndcg_at_3_max": 0.72692936085964, "nauc_ndcg_at_3_std": 0.10883567883912774, "nauc_ndcg_at_5_diff1": 0.8637479893020074, "nauc_ndcg_at_5_max": 0.7331969061294942, "nauc_ndcg_at_5_std": 0.10888095163739013, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8366013071895334, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.8646125116713362, "nauc_precision_at_10_max": 0.8768618558534468, "nauc_precision_at_10_std": 0.43799742119069335, "nauc_precision_at_1_diff1": 0.8740098646066407, "nauc_precision_at_1_max": 0.6846061506802336, "nauc_precision_at_1_std": 0.016475036338989085, "nauc_precision_at_20_diff1": 0.8651960784313695, "nauc_precision_at_20_max": 0.8897447868036057, "nauc_precision_at_20_std": 0.5705726735138443, "nauc_precision_at_3_diff1": 0.8417278861230014, "nauc_precision_at_3_max": 0.7857407113788957, "nauc_precision_at_3_std": 0.23955745820341165, "nauc_precision_at_5_diff1": 0.8566760037348254, "nauc_precision_at_5_max": 0.8495664932639713, "nauc_precision_at_5_std": 0.30673602774442654, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8366013071895522, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 1.0, "nauc_recall_at_10_diff1": 0.8646125116713329, "nauc_recall_at_10_max": 0.8768618558534561, "nauc_recall_at_10_std": 0.4379974211906987, "nauc_recall_at_1_diff1": 0.8740098646066407, "nauc_recall_at_1_max": 0.6846061506802336, "nauc_recall_at_1_std": 0.016475036338989085, "nauc_recall_at_20_diff1": 0.865196078431372, "nauc_recall_at_20_max": 0.8897447868036054, "nauc_recall_at_20_std": 0.5705726735138528, "nauc_recall_at_3_diff1": 0.8417278861230021, "nauc_recall_at_3_max": 0.7857407113788921, "nauc_recall_at_3_std": 0.23955745820340837, "nauc_recall_at_5_diff1": 0.8566760037348257, "nauc_recall_at_5_max": 0.8495664932639706, "nauc_recall_at_5_std": 0.3067360277444287, "ndcg_at_1": 0.86889, "ndcg_at_10": 0.92542, "ndcg_at_100": 0.92934, "ndcg_at_1000": 0.93009, "ndcg_at_20": 0.92794, "ndcg_at_3": 0.91184, "ndcg_at_5": 0.92026, "precision_at_1": 0.86889, "precision_at_10": 0.09767, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "precision_at_20": 0.04933, "precision_at_3": 0.3137, "precision_at_5": 0.19222, "recall_at_1": 0.86889, "recall_at_10": 0.97667, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "recall_at_20": 0.98667, "recall_at_3": 0.94111, "recall_at_5": 0.96111 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 1484.8957543373108, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.79072, "map_at_1": 0.4751, "map_at_10": 0.74355, "map_at_100": 0.74902, "map_at_1000": 0.74932, "map_at_20": 0.7475, "map_at_3": 0.69864, "map_at_5": 0.73103, "mrr_at_1": 0.7280405405405406, "mrr_at_10": 0.7977303196053193, "mrr_at_100": 0.7993979797532479, "mrr_at_1000": 0.7995630295310928, "mrr_at_20": 0.798864165188178, "mrr_at_3": 0.7916666666666664, "mrr_at_5": 0.795636261261261, "nauc_map_at_1000_diff1": 0.6256649071324872, "nauc_map_at_1000_max": 0.5735375619379824, "nauc_map_at_1000_std": 0.22083743098839143, "nauc_map_at_100_diff1": 0.6258479659620486, "nauc_map_at_100_max": 0.5740550390532451, "nauc_map_at_100_std": 0.22152263331314953, "nauc_map_at_10_diff1": 0.6217411286741686, "nauc_map_at_10_max": 0.5702714855001783, "nauc_map_at_10_std": 0.21405796044522293, "nauc_map_at_1_diff1": 0.6002554239795219, "nauc_map_at_1_max": 0.234901175002455, "nauc_map_at_1_std": 0.034169324518353, "nauc_map_at_20_diff1": 0.6260613170820447, "nauc_map_at_20_max": 0.5737386933721409, "nauc_map_at_20_std": 0.21992702151623847, "nauc_map_at_3_diff1": 0.6090197689169069, "nauc_map_at_3_max": 0.49284382384621306, "nauc_map_at_3_std": 0.138624455301081, "nauc_map_at_5_diff1": 0.6185927547856056, "nauc_map_at_5_max": 0.5615815304157588, "nauc_map_at_5_std": 0.20245448375917285, "nauc_mrr_at_1000_diff1": 0.6668721433088087, "nauc_mrr_at_1000_max": 0.6446490938197565, "nauc_mrr_at_1000_std": 0.31776904035044184, "nauc_mrr_at_100_diff1": 0.6669711329326476, "nauc_mrr_at_100_max": 0.6448963717970834, "nauc_mrr_at_100_std": 0.31813572678586155, "nauc_mrr_at_10_diff1": 0.6665297306611556, "nauc_mrr_at_10_max": 0.6461626693027981, "nauc_mrr_at_10_std": 0.3196263666901991, "nauc_mrr_at_1_diff1": 0.6575720836305169, "nauc_mrr_at_1_max": 0.5533847493305769, "nauc_mrr_at_1_std": 0.21355645302950124, "nauc_mrr_at_20_diff1": 0.6667875799287543, "nauc_mrr_at_20_max": 0.6455155944464043, "nauc_mrr_at_20_std": 0.31852476694902354, "nauc_mrr_at_3_diff1": 0.664813000551536, "nauc_mrr_at_3_max": 0.643043305388793, "nauc_mrr_at_3_std": 0.3097808850229901, "nauc_mrr_at_5_diff1": 0.665857829582622, "nauc_mrr_at_5_max": 0.6450324247131991, "nauc_mrr_at_5_std": 0.31935362969536607, "nauc_ndcg_at_1000_diff1": 0.6452554189620043, "nauc_ndcg_at_1000_max": 0.6337749441549703, "nauc_ndcg_at_1000_std": 0.303874650663312, "nauc_ndcg_at_100_diff1": 0.6491652007587659, "nauc_ndcg_at_100_max": 0.6441858281974558, "nauc_ndcg_at_100_std": 0.31742026395480366, "nauc_ndcg_at_10_diff1": 0.6391422675672194, "nauc_ndcg_at_10_max": 0.6366084275304457, "nauc_ndcg_at_10_std": 0.2931836617472829, "nauc_ndcg_at_1_diff1": 0.6575720836305169, "nauc_ndcg_at_1_max": 0.5533847493305769, "nauc_ndcg_at_1_std": 0.21355645302950124, "nauc_ndcg_at_20_diff1": 0.6492444931741755, "nauc_ndcg_at_20_max": 0.6459156570602925, "nauc_ndcg_at_20_std": 0.3088994793914235, "nauc_ndcg_at_3_diff1": 0.6159064468715444, "nauc_ndcg_at_3_max": 0.549490455209115, "nauc_ndcg_at_3_std": 0.19277974263365644, "nauc_ndcg_at_5_diff1": 0.632330726369529, "nauc_ndcg_at_5_max": 0.6188063762925213, "nauc_ndcg_at_5_std": 0.269801271967624, "nauc_precision_at_1000_diff1": -0.29138047819005797, "nauc_precision_at_1000_max": 0.101601813226311, "nauc_precision_at_1000_std": 0.1763922980088165, "nauc_precision_at_100_diff1": -0.18031829366040822, "nauc_precision_at_100_max": 0.2170414669213159, "nauc_precision_at_100_std": 0.25866935327202856, "nauc_precision_at_10_diff1": -0.0938064195201447, "nauc_precision_at_10_max": 0.30520467264375906, "nauc_precision_at_10_std": 0.2559624998139914, "nauc_precision_at_1_diff1": 0.6575720836305169, "nauc_precision_at_1_max": 0.5533847493305769, "nauc_precision_at_1_std": 0.21355645302950124, "nauc_precision_at_20_diff1": -0.1260731942177778, "nauc_precision_at_20_max": 0.27243280987859214, "nauc_precision_at_20_std": 0.26381077730402147, "nauc_precision_at_3_diff1": 0.05273177364101314, "nauc_precision_at_3_max": 0.3895696620910321, "nauc_precision_at_3_std": 0.21741897358471318, "nauc_precision_at_5_diff1": -0.032521923578669315, "nauc_precision_at_5_max": 0.3683805962204715, "nauc_precision_at_5_std": 0.26718712482468526, "nauc_recall_at_1000_diff1": 0.7311646540840526, "nauc_recall_at_1000_max": 0.9428956332658632, "nauc_recall_at_1000_std": 0.8378377024188404, "nauc_recall_at_100_diff1": 0.7345940983451282, "nauc_recall_at_100_max": 0.9204853995815641, "nauc_recall_at_100_std": 0.7451243993738742, "nauc_recall_at_10_diff1": 0.6400880030997321, "nauc_recall_at_10_max": 0.7782417771679832, "nauc_recall_at_10_std": 0.46177014614662737, "nauc_recall_at_1_diff1": 0.6002554239795219, "nauc_recall_at_1_max": 0.234901175002455, "nauc_recall_at_1_std": 0.034169324518353, "nauc_recall_at_20_diff1": 0.7061094411625372, "nauc_recall_at_20_max": 0.8660046059800297, "nauc_recall_at_20_std": 0.5751746991889914, "nauc_recall_at_3_diff1": 0.5739425851828479, "nauc_recall_at_3_max": 0.5294684112762419, "nauc_recall_at_3_std": 0.18366776112042058, "nauc_recall_at_5_diff1": 0.6126662940491471, "nauc_recall_at_5_max": 0.6963000057908852, "nauc_recall_at_5_std": 0.362227933418644, "ndcg_at_1": 0.72804, "ndcg_at_10": 0.79072, "ndcg_at_100": 0.80673, "ndcg_at_1000": 0.81307, "ndcg_at_20": 0.79961, "ndcg_at_3": 0.7428, "ndcg_at_5": 0.77321, "precision_at_1": 0.72804, "precision_at_10": 0.16014, "precision_at_100": 0.01736, "precision_at_1000": 0.00183, "precision_at_20": 0.08336, "precision_at_3": 0.44651, "precision_at_5": 0.29932, "recall_at_1": 0.4751, "recall_at_10": 0.86162, "recall_at_100": 0.91864, "recall_at_1000": 0.95721, "recall_at_20": 0.88824, "recall_at_3": 0.75862, "recall_at_5": 0.82077 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 226490.0695669651, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.70315, "map_at_1": 0.40696, "map_at_10": 0.6179, "map_at_100": 0.64309, "map_at_1000": 0.64353, "map_at_20": 0.6354, "map_at_3": 0.5396, "map_at_5": 0.5818, "mrr_at_1": 0.647887323943662, "mrr_at_10": 0.7541135703107532, "mrr_at_100": 0.756483656164785, "mrr_at_1000": 0.7565706179045278, "mrr_at_20": 0.7558957279027702, "mrr_at_3": 0.7394366197183099, "mrr_at_5": 0.7488262910798122, "nauc_map_at_1000_diff1": 0.42552998881302734, "nauc_map_at_1000_max": 0.43253852127285747, "nauc_map_at_1000_std": -0.048738283171735924, "nauc_map_at_100_diff1": 0.425107385005373, "nauc_map_at_100_max": 0.43288751763034894, "nauc_map_at_100_std": -0.04827078902742531, "nauc_map_at_10_diff1": 0.4223931151651558, "nauc_map_at_10_max": 0.41416943314831794, "nauc_map_at_10_std": -0.09765345839536647, "nauc_map_at_1_diff1": 0.5476774384667021, "nauc_map_at_1_max": 0.3508466130362578, "nauc_map_at_1_std": -0.2035097473946885, "nauc_map_at_20_diff1": 0.41952567372103966, "nauc_map_at_20_max": 0.4277687815742127, "nauc_map_at_20_std": -0.058148737142206766, "nauc_map_at_3_diff1": 0.4709814401670962, "nauc_map_at_3_max": 0.4046661811300015, "nauc_map_at_3_std": -0.18553573987315636, "nauc_map_at_5_diff1": 0.4558106421319416, "nauc_map_at_5_max": 0.3971314966952542, "nauc_map_at_5_std": -0.15454760317419264, "nauc_mrr_at_1000_diff1": 0.4714198679370516, "nauc_mrr_at_1000_max": 0.49974094019049387, "nauc_mrr_at_1000_std": 0.059360318840965084, "nauc_mrr_at_100_diff1": 0.4714650897451011, "nauc_mrr_at_100_max": 0.49982591467441995, "nauc_mrr_at_100_std": 0.05959962814788641, "nauc_mrr_at_10_diff1": 0.4679289815151978, "nauc_mrr_at_10_max": 0.5010831390131519, "nauc_mrr_at_10_std": 0.06168711909655768, "nauc_mrr_at_1_diff1": 0.5042396590385281, "nauc_mrr_at_1_max": 0.4699305141604478, "nauc_mrr_at_1_std": -0.0011007059701792971, "nauc_mrr_at_20_diff1": 0.4704854162022878, "nauc_mrr_at_20_max": 0.5008331396297026, "nauc_mrr_at_20_std": 0.060925795418258, "nauc_mrr_at_3_diff1": 0.4715046578891891, "nauc_mrr_at_3_max": 0.49816203311296314, "nauc_mrr_at_3_std": 0.056350371986931214, "nauc_mrr_at_5_diff1": 0.4723870570529245, "nauc_mrr_at_5_max": 0.4955394810752652, "nauc_mrr_at_5_std": 0.06356591587965502, "nauc_ndcg_at_1000_diff1": 0.4270525162587747, "nauc_ndcg_at_1000_max": 0.47117300959402636, "nauc_ndcg_at_1000_std": 0.03133906346076797, "nauc_ndcg_at_100_diff1": 0.4221656058478231, "nauc_ndcg_at_100_max": 0.4800851305303125, "nauc_ndcg_at_100_std": 0.04498541606625714, "nauc_ndcg_at_10_diff1": 0.3965948030560033, "nauc_ndcg_at_10_max": 0.4512772471953279, "nauc_ndcg_at_10_std": -0.030608424277932752, "nauc_ndcg_at_1_diff1": 0.5042396590385281, "nauc_ndcg_at_1_max": 0.4699305141604478, "nauc_ndcg_at_1_std": -0.0011007059701792971, "nauc_ndcg_at_20_diff1": 0.39626112547057285, "nauc_ndcg_at_20_max": 0.4760189355895963, "nauc_ndcg_at_20_std": 0.03248405807057921, "nauc_ndcg_at_3_diff1": 0.42166580498704165, "nauc_ndcg_at_3_max": 0.45149429818153103, "nauc_ndcg_at_3_std": -0.0408513220270249, "nauc_ndcg_at_5_diff1": 0.4269384493145147, "nauc_ndcg_at_5_max": 0.42644511900266857, "nauc_ndcg_at_5_std": -0.03984430540195149, "nauc_precision_at_1000_diff1": -0.24101868411855218, "nauc_precision_at_1000_max": -0.04746443318515683, "nauc_precision_at_1000_std": 0.41741552513094327, "nauc_precision_at_100_diff1": -0.23606184742838943, "nauc_precision_at_100_max": -0.017038071863325368, "nauc_precision_at_100_std": 0.4375692179815892, "nauc_precision_at_10_diff1": -0.23798899035846333, "nauc_precision_at_10_max": 0.04073411138282004, "nauc_precision_at_10_std": 0.40078496264694746, "nauc_precision_at_1_diff1": 0.5042396590385281, "nauc_precision_at_1_max": 0.4699305141604478, "nauc_precision_at_1_std": -0.0011007059701792971, "nauc_precision_at_20_diff1": -0.25405923535139, "nauc_precision_at_20_max": 0.023578697930631626, "nauc_precision_at_20_std": 0.4569720970184273, "nauc_precision_at_3_diff1": -0.052995738627394005, "nauc_precision_at_3_max": 0.18323842116419178, "nauc_precision_at_3_std": 0.2240819386079469, "nauc_precision_at_5_diff1": -0.14718211901562456, "nauc_precision_at_5_max": 0.0717424078506237, "nauc_precision_at_5_std": 0.32151182016566493, "nauc_recall_at_1000_diff1": 0.806368320522017, "nauc_recall_at_1000_max": -0.17087593368954954, "nauc_recall_at_1000_std": -1.1451833892298617, "nauc_recall_at_100_diff1": 0.2909290823307582, "nauc_recall_at_100_max": 0.6688047794752549, "nauc_recall_at_100_std": 0.43562146184686673, "nauc_recall_at_10_diff1": 0.20633514494858715, "nauc_recall_at_10_max": 0.4057250627983415, "nauc_recall_at_10_std": -0.05566438793011242, "nauc_recall_at_1_diff1": 0.5476774384667021, "nauc_recall_at_1_max": 0.3508466130362578, "nauc_recall_at_1_std": -0.2035097473946885, "nauc_recall_at_20_diff1": 0.16102371618571604, "nauc_recall_at_20_max": 0.5229582968117912, "nauc_recall_at_20_std": 0.17547160506393064, "nauc_recall_at_3_diff1": 0.3820991451441941, "nauc_recall_at_3_max": 0.37984007561200095, "nauc_recall_at_3_std": -0.19212144775977344, "nauc_recall_at_5_diff1": 0.35722728457669023, "nauc_recall_at_5_max": 0.35453563586445336, "nauc_recall_at_5_std": -0.15577293164732453, "ndcg_at_1": 0.64789, "ndcg_at_10": 0.70315, "ndcg_at_100": 0.75526, "ndcg_at_1000": 0.76135, "ndcg_at_20": 0.73133, "ndcg_at_3": 0.6588, "ndcg_at_5": 0.67139, "precision_at_1": 0.64789, "precision_at_10": 0.18638, "precision_at_100": 0.02474, "precision_at_1000": 0.00256, "precision_at_20": 0.10775, "precision_at_3": 0.40063, "precision_at_5": 0.29859, "recall_at_1": 0.40696, "recall_at_10": 0.79588, "recall_at_100": 0.95957, "recall_at_1000": 0.99804, "recall_at_20": 0.87194, "recall_at_3": 0.60968, "recall_at_5": 0.69135 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 225582.5098335743, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.47521, "map_at_1": 0.27633, "map_at_10": 0.40312, "map_at_100": 0.41514, "map_at_1000": 0.41546, "map_at_20": 0.41135, "map_at_3": 0.36012, "map_at_5": 0.38971, "mrr_at_1": 0.30403800475059384, "mrr_at_10": 0.42929155827018056, "mrr_at_100": 0.43882159362643647, "mrr_at_1000": 0.4390708750947643, "mrr_at_20": 0.43607832888459863, "mrr_at_3": 0.39034045922406985, "mrr_at_5": 0.4170625494853524, "nauc_map_at_1000_diff1": 0.3299997110422999, "nauc_map_at_1000_max": 0.261136550440722, "nauc_map_at_1000_std": 0.05180398576481644, "nauc_map_at_100_diff1": 0.32981208083213237, "nauc_map_at_100_max": 0.26113801470672554, "nauc_map_at_100_std": 0.051673677500214926, "nauc_map_at_10_diff1": 0.3274851176386591, "nauc_map_at_10_max": 0.2603323830317976, "nauc_map_at_10_std": 0.052308397860183264, "nauc_map_at_1_diff1": 0.41465785426818985, "nauc_map_at_1_max": 0.27030540018926097, "nauc_map_at_1_std": 0.0023165462854145853, "nauc_map_at_20_diff1": 0.32880687706482875, "nauc_map_at_20_max": 0.2629650012217612, "nauc_map_at_20_std": 0.05446051110700691, "nauc_map_at_3_diff1": 0.3428481637942387, "nauc_map_at_3_max": 0.26321278862175634, "nauc_map_at_3_std": 0.037101244696859936, "nauc_map_at_5_diff1": 0.33321956753870174, "nauc_map_at_5_max": 0.2629455113862495, "nauc_map_at_5_std": 0.05460387559470636, "nauc_mrr_at_1000_diff1": 0.3391131477393978, "nauc_mrr_at_1000_max": 0.2670718808327621, "nauc_mrr_at_1000_std": 0.05745966983171127, "nauc_mrr_at_100_diff1": 0.3390230568793171, "nauc_mrr_at_100_max": 0.2671436426531771, "nauc_mrr_at_100_std": 0.05736360861869989, "nauc_mrr_at_10_diff1": 0.3344011804166142, "nauc_mrr_at_10_max": 0.2657666006840405, "nauc_mrr_at_10_std": 0.057863489730523174, "nauc_mrr_at_1_diff1": 0.41442850939921844, "nauc_mrr_at_1_max": 0.27186743449287076, "nauc_mrr_at_1_std": 0.01296059900788334, "nauc_mrr_at_20_diff1": 0.338040888416676, "nauc_mrr_at_20_max": 0.26892076225670264, "nauc_mrr_at_20_std": 0.059408352473913524, "nauc_mrr_at_3_diff1": 0.34798351395796856, "nauc_mrr_at_3_max": 0.2721745101928153, "nauc_mrr_at_3_std": 0.054692191143277176, "nauc_mrr_at_5_diff1": 0.3376418122676429, "nauc_mrr_at_5_max": 0.2634763841561849, "nauc_mrr_at_5_std": 0.06183139175418699, "nauc_ndcg_at_1000_diff1": 0.31131009883990385, "nauc_ndcg_at_1000_max": 0.26439124461019764, "nauc_ndcg_at_1000_std": 0.07304740928882791, "nauc_ndcg_at_100_diff1": 0.3060948038438666, "nauc_ndcg_at_100_max": 0.2645714917339233, "nauc_ndcg_at_100_std": 0.07049845288026353, "nauc_ndcg_at_10_diff1": 0.2924432543443178, "nauc_ndcg_at_10_max": 0.2632135618758708, "nauc_ndcg_at_10_std": 0.07745730443694289, "nauc_ndcg_at_1_diff1": 0.41442850939921844, "nauc_ndcg_at_1_max": 0.27186743449287076, "nauc_ndcg_at_1_std": 0.01296059900788334, "nauc_ndcg_at_20_diff1": 0.2993053088686414, "nauc_ndcg_at_20_max": 0.2748616805397411, "nauc_ndcg_at_20_std": 0.08624189874787164, "nauc_ndcg_at_3_diff1": 0.3266546843467852, "nauc_ndcg_at_3_max": 0.26790901916180276, "nauc_ndcg_at_3_std": 0.05528896643741061, "nauc_ndcg_at_5_diff1": 0.3068610853041326, "nauc_ndcg_at_5_max": 0.2638375185069462, "nauc_ndcg_at_5_std": 0.08316288426103477, "nauc_precision_at_1000_diff1": -0.05523684016254207, "nauc_precision_at_1000_max": -0.00905274759700128, "nauc_precision_at_1000_std": 0.07357648643285562, "nauc_precision_at_100_diff1": -0.027874601394909285, "nauc_precision_at_100_max": 0.0446628567171925, "nauc_precision_at_100_std": 0.07016491966862602, "nauc_precision_at_10_diff1": 0.11500240615914001, "nauc_precision_at_10_max": 0.21040947801283377, "nauc_precision_at_10_std": 0.1464462638185187, "nauc_precision_at_1_diff1": 0.41442850939921844, "nauc_precision_at_1_max": 0.27186743449287076, "nauc_precision_at_1_std": 0.01296059900788334, "nauc_precision_at_20_diff1": 0.062028655994787185, "nauc_precision_at_20_max": 0.19068781355702719, "nauc_precision_at_20_std": 0.17273256146657345, "nauc_precision_at_3_diff1": 0.2602570091547143, "nauc_precision_at_3_max": 0.2657809243311167, "nauc_precision_at_3_std": 0.1105366063394043, "nauc_precision_at_5_diff1": 0.1944037995411402, "nauc_precision_at_5_max": 0.24159428718600895, "nauc_precision_at_5_std": 0.17169251233558955, "nauc_recall_at_1000_diff1": 0.02465665919311491, "nauc_recall_at_1000_max": 0.33345290954641715, "nauc_recall_at_1000_std": 0.5754950856097731, "nauc_recall_at_100_diff1": 0.09651938189716085, "nauc_recall_at_100_max": 0.27347112901513265, "nauc_recall_at_100_std": 0.17782919554033105, "nauc_recall_at_10_diff1": 0.13957480788444254, "nauc_recall_at_10_max": 0.25693549389031883, "nauc_recall_at_10_std": 0.1576834345452784, "nauc_recall_at_1_diff1": 0.41465785426818985, "nauc_recall_at_1_max": 0.27030540018926097, "nauc_recall_at_1_std": 0.0023165462854145853, "nauc_recall_at_20_diff1": 0.13755908199795955, "nauc_recall_at_20_max": 0.3302629769637712, "nauc_recall_at_20_std": 0.23341298688647236, "nauc_recall_at_3_diff1": 0.2669687987201209, "nauc_recall_at_3_max": 0.2635911241634212, "nauc_recall_at_3_std": 0.08708729704891159, "nauc_recall_at_5_diff1": 0.2075990915648664, "nauc_recall_at_5_max": 0.25456082319944046, "nauc_recall_at_5_std": 0.1620135372926199, "ndcg_at_1": 0.30404, "ndcg_at_10": 0.47521, "ndcg_at_100": 0.5268, "ndcg_at_1000": 0.5359, "ndcg_at_20": 0.50275, "ndcg_at_3": 0.39307, "ndcg_at_5": 0.44387, "precision_at_1": 0.30404, "precision_at_10": 0.07625, "precision_at_100": 0.01048, "precision_at_1000": 0.00113, "precision_at_20": 0.04442, "precision_at_3": 0.16865, "precision_at_5": 0.12922, "recall_at_1": 0.27633, "recall_at_10": 0.67181, "recall_at_100": 0.89667, "recall_at_1000": 0.96873, "recall_at_20": 0.77553, "recall_at_3": 0.45962, "recall_at_5": 0.57878 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 1957.7613937854767, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.31021, "map_at_1": 0.205, "map_at_10": 0.27098, "map_at_100": 0.27647, "map_at_1000": 0.27733, "map_at_20": 0.2736, "map_at_3": 0.24583, "map_at_5": 0.26408, "mrr_at_1": 0.205, "mrr_at_10": 0.2709781746031746, "mrr_at_100": 0.2764738908569317, "mrr_at_1000": 0.27733006788463876, "mrr_at_20": 0.2735993867243867, "mrr_at_3": 0.24583333333333332, "mrr_at_5": 0.26408333333333334, "nauc_map_at_1000_diff1": 0.39733314773068384, "nauc_map_at_1000_max": 0.26177504420530523, "nauc_map_at_1000_std": -0.044565445932742434, "nauc_map_at_100_diff1": 0.3973711015291718, "nauc_map_at_100_max": 0.26210930689594614, "nauc_map_at_100_std": -0.044480465313211245, "nauc_map_at_10_diff1": 0.4008119062964794, "nauc_map_at_10_max": 0.26615517939284056, "nauc_map_at_10_std": -0.048709690232786436, "nauc_map_at_1_diff1": 0.3974833515872953, "nauc_map_at_1_max": 0.2091824090433059, "nauc_map_at_1_std": -0.07337293243902232, "nauc_map_at_20_diff1": 0.39855645189686256, "nauc_map_at_20_max": 0.26309233062394216, "nauc_map_at_20_std": -0.0470281024793479, "nauc_map_at_3_diff1": 0.38411975469841636, "nauc_map_at_3_max": 0.23797842781734818, "nauc_map_at_3_std": -0.08279732915833832, "nauc_map_at_5_diff1": 0.3972286051496386, "nauc_map_at_5_max": 0.2603135968220476, "nauc_map_at_5_std": -0.06161473773449569, "nauc_mrr_at_1000_diff1": 0.39733314773068384, "nauc_mrr_at_1000_max": 0.26177504420530523, "nauc_mrr_at_1000_std": -0.044565445932742434, "nauc_mrr_at_100_diff1": 0.3973711015291718, "nauc_mrr_at_100_max": 0.26210930689594614, "nauc_mrr_at_100_std": -0.044480465313211245, "nauc_mrr_at_10_diff1": 0.4008119062964794, "nauc_mrr_at_10_max": 0.26615517939284056, "nauc_mrr_at_10_std": -0.048709690232786436, "nauc_mrr_at_1_diff1": 0.3974833515872953, "nauc_mrr_at_1_max": 0.2091824090433059, "nauc_mrr_at_1_std": -0.07337293243902232, "nauc_mrr_at_20_diff1": 0.39855645189686256, "nauc_mrr_at_20_max": 0.26309233062394216, "nauc_mrr_at_20_std": -0.0470281024793479, "nauc_mrr_at_3_diff1": 0.38411975469841636, "nauc_mrr_at_3_max": 0.23797842781734818, "nauc_mrr_at_3_std": -0.08279732915833832, "nauc_mrr_at_5_diff1": 0.3972286051496386, "nauc_mrr_at_5_max": 0.2603135968220476, "nauc_mrr_at_5_std": -0.06161473773449569, "nauc_ndcg_at_1000_diff1": 0.4013539559811185, "nauc_ndcg_at_1000_max": 0.28507892906757715, "nauc_ndcg_at_1000_std": 0.004858667645440177, "nauc_ndcg_at_100_diff1": 0.4007852359443604, "nauc_ndcg_at_100_max": 0.2907197959727122, "nauc_ndcg_at_100_std": 0.007853949474176185, "nauc_ndcg_at_10_diff1": 0.4149383023022595, "nauc_ndcg_at_10_max": 0.3045199903629755, "nauc_ndcg_at_10_std": -0.015284828398756922, "nauc_ndcg_at_1_diff1": 0.3974833515872953, "nauc_ndcg_at_1_max": 0.2091824090433059, "nauc_ndcg_at_1_std": -0.07337293243902232, "nauc_ndcg_at_20_diff1": 0.40616856971054405, "nauc_ndcg_at_20_max": 0.29242695558692794, "nauc_ndcg_at_20_std": -0.01101570303459534, "nauc_ndcg_at_3_diff1": 0.3840734360437966, "nauc_ndcg_at_3_max": 0.2527827522900115, "nauc_ndcg_at_3_std": -0.08424756880596201, "nauc_ndcg_at_5_diff1": 0.4059240405340309, "nauc_ndcg_at_5_max": 0.29096068689802457, "nauc_ndcg_at_5_std": -0.04612859569696686, "nauc_precision_at_1000_diff1": 0.4244877773067686, "nauc_precision_at_1000_max": 0.3884131694220722, "nauc_precision_at_1000_std": 0.353002684753426, "nauc_precision_at_100_diff1": 0.4059412515587299, "nauc_precision_at_100_max": 0.37999808156308357, "nauc_precision_at_100_std": 0.21763191098452753, "nauc_precision_at_10_diff1": 0.458027996534631, "nauc_precision_at_10_max": 0.4135774724952613, "nauc_precision_at_10_std": 0.08749405611031875, "nauc_precision_at_1_diff1": 0.3974833515872953, "nauc_precision_at_1_max": 0.2091824090433059, "nauc_precision_at_1_std": -0.07337293243902232, "nauc_precision_at_20_diff1": 0.425010732093914, "nauc_precision_at_20_max": 0.36834527622758645, "nauc_precision_at_20_std": 0.1013638014727735, "nauc_precision_at_3_diff1": 0.385060685909143, "nauc_precision_at_3_max": 0.293442147897053, "nauc_precision_at_3_std": -0.0876761441366157, "nauc_precision_at_5_diff1": 0.43126170920335316, "nauc_precision_at_5_max": 0.37529738945539676, "nauc_precision_at_5_std": -0.0007714298679494246, "nauc_recall_at_1000_diff1": 0.4244877773067688, "nauc_recall_at_1000_max": 0.38841316942207216, "nauc_recall_at_1000_std": 0.35300268475342617, "nauc_recall_at_100_diff1": 0.40594125155873034, "nauc_recall_at_100_max": 0.37999808156308335, "nauc_recall_at_100_std": 0.21763191098452705, "nauc_recall_at_10_diff1": 0.458027996534631, "nauc_recall_at_10_max": 0.413577472495261, "nauc_recall_at_10_std": 0.08749405611031859, "nauc_recall_at_1_diff1": 0.3974833515872953, "nauc_recall_at_1_max": 0.2091824090433059, "nauc_recall_at_1_std": -0.07337293243902232, "nauc_recall_at_20_diff1": 0.42501073209391427, "nauc_recall_at_20_max": 0.36834527622758656, "nauc_recall_at_20_std": 0.10136380147277349, "nauc_recall_at_3_diff1": 0.3850606859091431, "nauc_recall_at_3_max": 0.293442147897053, "nauc_recall_at_3_std": -0.08767614413661576, "nauc_recall_at_5_diff1": 0.4312617092033527, "nauc_recall_at_5_max": 0.3752973894553965, "nauc_recall_at_5_std": -0.0007714298679495149, "ndcg_at_1": 0.205, "ndcg_at_10": 0.31021, "ndcg_at_100": 0.34194, "ndcg_at_1000": 0.36992, "ndcg_at_20": 0.32013, "ndcg_at_3": 0.26089, "ndcg_at_5": 0.29381, "precision_at_1": 0.205, "precision_at_10": 0.0435, "precision_at_100": 0.00595, "precision_at_1000": 0.00083, "precision_at_20": 0.02375, "precision_at_3": 0.10167, "precision_at_5": 0.077, "recall_at_1": 0.205, "recall_at_10": 0.435, "recall_at_100": 0.595, "recall_at_1000": 0.825, "recall_at_20": 0.475, "recall_at_3": 0.305, "recall_at_5": 0.385 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.28278, "map_at_1": 0.2, "map_at_10": 0.25367, "map_at_100": 0.26251, "map_at_1000": 0.26325, "map_at_20": 0.25857, "map_at_3": 0.23833, "map_at_5": 0.24833, "mrr_at_1": 0.2, "mrr_at_10": 0.253670634920635, "mrr_at_100": 0.2625110520095054, "mrr_at_1000": 0.26325325927271204, "mrr_at_20": 0.2585687943592357, "mrr_at_3": 0.23833333333333337, "mrr_at_5": 0.24833333333333335, "nauc_map_at_1000_diff1": 0.48846433045230764, "nauc_map_at_1000_max": 0.4184821449135949, "nauc_map_at_1000_std": 0.1649120543143561, "nauc_map_at_100_diff1": 0.48850419034873516, "nauc_map_at_100_max": 0.4187393743703572, "nauc_map_at_100_std": 0.1651000090790014, "nauc_map_at_10_diff1": 0.4951649412044444, "nauc_map_at_10_max": 0.4173850927312253, "nauc_map_at_10_std": 0.15971050008080603, "nauc_map_at_1_diff1": 0.5464401033257992, "nauc_map_at_1_max": 0.40082337746206026, "nauc_map_at_1_std": 0.1043186955117856, "nauc_map_at_20_diff1": 0.49048308444617456, "nauc_map_at_20_max": 0.41788675027001815, "nauc_map_at_20_std": 0.16211881844640397, "nauc_map_at_3_diff1": 0.4937193447187161, "nauc_map_at_3_max": 0.4130072283244158, "nauc_map_at_3_std": 0.1410825210753104, "nauc_map_at_5_diff1": 0.4950184057607674, "nauc_map_at_5_max": 0.42583483715307674, "nauc_map_at_5_std": 0.16011398415838624, "nauc_mrr_at_1000_diff1": 0.48846433045230764, "nauc_mrr_at_1000_max": 0.4184821449135949, "nauc_mrr_at_1000_std": 0.1649120543143561, "nauc_mrr_at_100_diff1": 0.48850419034873516, "nauc_mrr_at_100_max": 0.4187393743703572, "nauc_mrr_at_100_std": 0.1651000090790014, "nauc_mrr_at_10_diff1": 0.4951649412044444, "nauc_mrr_at_10_max": 0.4173850927312253, "nauc_mrr_at_10_std": 0.15971050008080603, "nauc_mrr_at_1_diff1": 0.5464401033257992, "nauc_mrr_at_1_max": 0.40082337746206026, "nauc_mrr_at_1_std": 0.1043186955117856, "nauc_mrr_at_20_diff1": 0.49048308444617456, "nauc_mrr_at_20_max": 0.41788675027001815, "nauc_mrr_at_20_std": 0.16211881844640397, "nauc_mrr_at_3_diff1": 0.4937193447187161, "nauc_mrr_at_3_max": 0.4130072283244158, "nauc_mrr_at_3_std": 0.1410825210753104, "nauc_mrr_at_5_diff1": 0.4950184057607674, "nauc_mrr_at_5_max": 0.42583483715307674, "nauc_mrr_at_5_std": 0.16011398415838624, "nauc_ndcg_at_1000_diff1": 0.45934372208001717, "nauc_ndcg_at_1000_max": 0.4215169378811277, "nauc_ndcg_at_1000_std": 0.20931476899785414, "nauc_ndcg_at_100_diff1": 0.45624813807719783, "nauc_ndcg_at_100_max": 0.4275818663860078, "nauc_ndcg_at_100_std": 0.21518755563634898, "nauc_ndcg_at_10_diff1": 0.4825968395141318, "nauc_ndcg_at_10_max": 0.4189614797028018, "nauc_ndcg_at_10_std": 0.18252879839759328, "nauc_ndcg_at_1_diff1": 0.5464401033257992, "nauc_ndcg_at_1_max": 0.40082337746206026, "nauc_ndcg_at_1_std": 0.1043186955117856, "nauc_ndcg_at_20_diff1": 0.4673813466344015, "nauc_ndcg_at_20_max": 0.42296503599069235, "nauc_ndcg_at_20_std": 0.1929808659951319, "nauc_ndcg_at_3_diff1": 0.47916085101108324, "nauc_ndcg_at_3_max": 0.4152641986255343, "nauc_ndcg_at_3_std": 0.15033600889958793, "nauc_ndcg_at_5_diff1": 0.4814091000301, "nauc_ndcg_at_5_max": 0.4386330550531778, "nauc_ndcg_at_5_std": 0.18384208900679808, "nauc_precision_at_1000_diff1": 0.31238761238761237, "nauc_precision_at_1000_max": 0.4142482517482516, "nauc_precision_at_1000_std": 0.4742382617382624, "nauc_precision_at_100_diff1": 0.3347757406062608, "nauc_precision_at_100_max": 0.462818084493124, "nauc_precision_at_100_std": 0.42115547303721634, "nauc_precision_at_10_diff1": 0.45217391304347826, "nauc_precision_at_10_max": 0.4194105194593717, "nauc_precision_at_10_std": 0.24364110079791576, "nauc_precision_at_1_diff1": 0.5464401033257992, "nauc_precision_at_1_max": 0.40082337746206026, "nauc_precision_at_1_std": 0.1043186955117856, "nauc_precision_at_20_diff1": 0.39996601151686717, "nauc_precision_at_20_max": 0.43821612753524697, "nauc_precision_at_20_std": 0.2850326485525485, "nauc_precision_at_3_diff1": 0.4409502355672158, "nauc_precision_at_3_max": 0.4208480419762924, "nauc_precision_at_3_std": 0.1743472109665811, "nauc_precision_at_5_diff1": 0.44671921587898206, "nauc_precision_at_5_max": 0.4748554011249165, "nauc_precision_at_5_std": 0.24908193825578231, "nauc_recall_at_1000_diff1": 0.31238761238761215, "nauc_recall_at_1000_max": 0.41424825174825103, "nauc_recall_at_1000_std": 0.47423826173826156, "nauc_recall_at_100_diff1": 0.33477574060626, "nauc_recall_at_100_max": 0.4628180844931236, "nauc_recall_at_100_std": 0.42115547303721584, "nauc_recall_at_10_diff1": 0.4521739130434784, "nauc_recall_at_10_max": 0.4194105194593716, "nauc_recall_at_10_std": 0.24364110079791582, "nauc_recall_at_1_diff1": 0.5464401033257992, "nauc_recall_at_1_max": 0.40082337746206026, "nauc_recall_at_1_std": 0.1043186955117856, "nauc_recall_at_20_diff1": 0.3999660115168666, "nauc_recall_at_20_max": 0.4382161275352466, "nauc_recall_at_20_std": 0.2850326485525483, "nauc_recall_at_3_diff1": 0.44095023556721585, "nauc_recall_at_3_max": 0.4208480419762923, "nauc_recall_at_3_std": 0.17434721096658118, "nauc_recall_at_5_diff1": 0.4467192158789817, "nauc_recall_at_5_max": 0.47485540112491614, "nauc_recall_at_5_std": 0.2490819382557821, "ndcg_at_1": 0.2, "ndcg_at_10": 0.28278, "ndcg_at_100": 0.32982, "ndcg_at_1000": 0.35357, "ndcg_at_20": 0.30054, "ndcg_at_3": 0.25155, "ndcg_at_5": 0.26983, "precision_at_1": 0.2, "precision_at_10": 0.0375, "precision_at_100": 0.00605, "precision_at_1000": 0.0008, "precision_at_20": 0.02225, "precision_at_3": 0.09667, "precision_at_5": 0.067, "recall_at_1": 0.2, "recall_at_10": 0.375, "recall_at_100": 0.605, "recall_at_1000": 0.8, "recall_at_20": 0.445, "recall_at_3": 0.29, "recall_at_5": 0.335 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 13.14871883392334, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.87102, "map_at_1": 0.76623, "map_at_10": 0.83824, "map_at_100": 0.8405, "map_at_1000": 0.8405, "map_at_20": 0.8405, "map_at_3": 0.829, "map_at_5": 0.83225, "mrr_at_1": 0.7662337662337663, "mrr_at_10": 0.8382395382395383, "mrr_at_100": 0.8405024268660632, "mrr_at_1000": 0.8405024268660632, "mrr_at_20": 0.8405024268660632, "mrr_at_3": 0.8290043290043291, "mrr_at_5": 0.8322510822510824, "nauc_map_at_1000_diff1": 0.6885097620218021, "nauc_map_at_1000_max": 0.4904532674828099, "nauc_map_at_1000_std": -0.04312965967394911, "nauc_map_at_100_diff1": 0.6885097620218021, "nauc_map_at_100_max": 0.4904532674828099, "nauc_map_at_100_std": -0.04312965967394911, "nauc_map_at_10_diff1": 0.6872038137909682, "nauc_map_at_10_max": 0.48687056680246493, "nauc_map_at_10_std": -0.05217570853794742, "nauc_map_at_1_diff1": 0.7055578519524833, "nauc_map_at_1_max": 0.4691591722758418, "nauc_map_at_1_std": -0.04903115664558869, "nauc_map_at_20_diff1": 0.6885097620218021, "nauc_map_at_20_max": 0.4904532674828099, "nauc_map_at_20_std": -0.04312965967394911, "nauc_map_at_3_diff1": 0.6938494791877855, "nauc_map_at_3_max": 0.5041937160694657, "nauc_map_at_3_std": -0.01740315649056545, "nauc_map_at_5_diff1": 0.6874582527449108, "nauc_map_at_5_max": 0.4938432184645834, "nauc_map_at_5_std": -0.03864255840167695, "nauc_mrr_at_1000_diff1": 0.6885097620218021, "nauc_mrr_at_1000_max": 0.4904532674828099, "nauc_mrr_at_1000_std": -0.04312965967394911, "nauc_mrr_at_100_diff1": 0.6885097620218021, "nauc_mrr_at_100_max": 0.4904532674828099, "nauc_mrr_at_100_std": -0.04312965967394911, "nauc_mrr_at_10_diff1": 0.6872038137909682, "nauc_mrr_at_10_max": 0.48687056680246493, "nauc_mrr_at_10_std": -0.05217570853794742, "nauc_mrr_at_1_diff1": 0.7055578519524833, "nauc_mrr_at_1_max": 0.4691591722758418, "nauc_mrr_at_1_std": -0.04903115664558869, "nauc_mrr_at_20_diff1": 0.6885097620218021, "nauc_mrr_at_20_max": 0.4904532674828099, "nauc_mrr_at_20_std": -0.04312965967394911, "nauc_mrr_at_3_diff1": 0.6938494791877855, "nauc_mrr_at_3_max": 0.5041937160694657, "nauc_mrr_at_3_std": -0.01740315649056545, "nauc_mrr_at_5_diff1": 0.6874582527449108, "nauc_mrr_at_5_max": 0.4938432184645834, "nauc_mrr_at_5_std": -0.03864255840167695, "nauc_ndcg_at_1000_diff1": 0.6875068929206645, "nauc_ndcg_at_1000_max": 0.4907520578278997, "nauc_ndcg_at_1000_std": -0.04447977952738645, "nauc_ndcg_at_100_diff1": 0.6875068929206645, "nauc_ndcg_at_100_max": 0.4907520578278997, "nauc_ndcg_at_100_std": -0.04447977952738645, "nauc_ndcg_at_10_diff1": 0.6828535625360586, "nauc_ndcg_at_10_max": 0.4753715542957344, "nauc_ndcg_at_10_std": -0.08166804488905967, "nauc_ndcg_at_1_diff1": 0.7055578519524833, "nauc_ndcg_at_1_max": 0.4691591722758418, "nauc_ndcg_at_1_std": -0.04903115664558869, "nauc_ndcg_at_20_diff1": 0.6875068929206645, "nauc_ndcg_at_20_max": 0.4907520578278997, "nauc_ndcg_at_20_std": -0.04447977952738645, "nauc_ndcg_at_3_diff1": 0.6967417535519655, "nauc_ndcg_at_3_max": 0.5162820063696817, "nauc_ndcg_at_3_std": 0.001959353725228958, "nauc_ndcg_at_5_diff1": 0.6842494443336714, "nauc_ndcg_at_5_max": 0.4963559043702825, "nauc_ndcg_at_5_std": -0.03915356739636289, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.613314205220816, "nauc_precision_at_10_max": 0.21977808756952869, "nauc_precision_at_10_std": -0.68631307805364, "nauc_precision_at_1_diff1": 0.7055578519524833, "nauc_precision_at_1_max": 0.4691591722758418, "nauc_precision_at_1_std": -0.04903115664558869, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_3_diff1": 0.7154433535920243, "nauc_precision_at_3_max": 0.5750317724577289, "nauc_precision_at_3_std": 0.10172403925188567, "nauc_precision_at_5_diff1": 0.6680172458573604, "nauc_precision_at_5_max": 0.5042037345340171, "nauc_precision_at_5_std": -0.047988620872798975, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.6133142052208155, "nauc_recall_at_10_max": 0.21977808756952488, "nauc_recall_at_10_std": -0.6863130780536503, "nauc_recall_at_1_diff1": 0.7055578519524833, "nauc_recall_at_1_max": 0.4691591722758418, "nauc_recall_at_1_std": -0.04903115664558869, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_3_diff1": 0.7154433535920213, "nauc_recall_at_3_max": 0.5750317724577277, "nauc_recall_at_3_std": 0.10172403925188503, "nauc_recall_at_5_diff1": 0.668017245857358, "nauc_recall_at_5_max": 0.5042037345340157, "nauc_recall_at_5_std": -0.04798862087280273, "ndcg_at_1": 0.76623, "ndcg_at_10": 0.87102, "ndcg_at_100": 0.87815, "ndcg_at_1000": 0.87815, "ndcg_at_20": 0.87815, "ndcg_at_3": 0.84957, "ndcg_at_5": 0.85516, "precision_at_1": 0.76623, "precision_at_10": 0.0974, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.05, "precision_at_3": 0.30303, "precision_at_5": 0.18442, "recall_at_1": 0.76623, "recall_at_10": 0.97403, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 1.0, "recall_at_3": 0.90909, "recall_at_5": 0.92208 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 214.9201488494873, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.37446, "map_at_1": 0.20971, "map_at_10": 0.32866, "map_at_100": 0.34424, "map_at_1000": 0.34533, "map_at_20": 0.33707, "map_at_3": 0.29453, "map_at_5": 0.31455, "mrr_at_1": 0.29357798165137616, "mrr_at_10": 0.36200548517062264, "mrr_at_100": 0.3740163116039349, "mrr_at_1000": 0.37468599603491515, "mrr_at_20": 0.3687663363276522, "mrr_at_3": 0.3409785932721714, "mrr_at_5": 0.35275229357798166, "nauc_map_at_1000_diff1": 0.45378110647853365, "nauc_map_at_1000_max": 0.04759413522164465, "nauc_map_at_1000_std": -0.3401700100287993, "nauc_map_at_100_diff1": 0.45329104168210915, "nauc_map_at_100_max": 0.047579865387336544, "nauc_map_at_100_std": -0.3399619310023172, "nauc_map_at_10_diff1": 0.4577169109716571, "nauc_map_at_10_max": 0.05097338145091819, "nauc_map_at_10_std": -0.3413191852269897, "nauc_map_at_1_diff1": 0.5518348816563906, "nauc_map_at_1_max": 0.07377101119089663, "nauc_map_at_1_std": -0.26809843855821786, "nauc_map_at_20_diff1": 0.4547596157257102, "nauc_map_at_20_max": 0.04688698286524278, "nauc_map_at_20_std": -0.34268186516724314, "nauc_map_at_3_diff1": 0.4679841190429562, "nauc_map_at_3_max": 0.05790504514306734, "nauc_map_at_3_std": -0.32209965854683414, "nauc_map_at_5_diff1": 0.460133850677349, "nauc_map_at_5_max": 0.05327053889146215, "nauc_map_at_5_std": -0.33567484138364745, "nauc_mrr_at_1000_diff1": 0.4818611900563693, "nauc_mrr_at_1000_max": 0.03688776037602183, "nauc_mrr_at_1000_std": -0.35643108371521776, "nauc_mrr_at_100_diff1": 0.4814378572863629, "nauc_mrr_at_100_max": 0.0367993573850964, "nauc_mrr_at_100_std": -0.3561719362447453, "nauc_mrr_at_10_diff1": 0.48258660834366796, "nauc_mrr_at_10_max": 0.036186629121457, "nauc_mrr_at_10_std": -0.35958431245602557, "nauc_mrr_at_1_diff1": 0.5230353749741389, "nauc_mrr_at_1_max": 0.06587751414523899, "nauc_mrr_at_1_std": -0.32859487926805514, "nauc_mrr_at_20_diff1": 0.4819017718965853, "nauc_mrr_at_20_max": 0.036475821434962016, "nauc_mrr_at_20_std": -0.35816082778792685, "nauc_mrr_at_3_diff1": 0.4905198332221737, "nauc_mrr_at_3_max": 0.03661467102314838, "nauc_mrr_at_3_std": -0.35640572814472793, "nauc_mrr_at_5_diff1": 0.4824162768724931, "nauc_mrr_at_5_max": 0.03364822894570434, "nauc_mrr_at_5_std": -0.35963166357650084, "nauc_ndcg_at_1000_diff1": 0.4450897476457732, "nauc_ndcg_at_1000_max": 0.04253233686878224, "nauc_ndcg_at_1000_std": -0.34135743053752615, "nauc_ndcg_at_100_diff1": 0.4300598727282433, "nauc_ndcg_at_100_max": 0.044135555523664306, "nauc_ndcg_at_100_std": -0.3286139221465386, "nauc_ndcg_at_10_diff1": 0.44711999598286095, "nauc_ndcg_at_10_max": 0.04090220393579857, "nauc_ndcg_at_10_std": -0.3584111558393882, "nauc_ndcg_at_1_diff1": 0.5230353749741389, "nauc_ndcg_at_1_max": 0.06587751414523899, "nauc_ndcg_at_1_std": -0.32859487926805514, "nauc_ndcg_at_20_diff1": 0.4387416070736086, "nauc_ndcg_at_20_max": 0.03230834151415491, "nauc_ndcg_at_20_std": -0.3587366944538616, "nauc_ndcg_at_3_diff1": 0.4641232978935688, "nauc_ndcg_at_3_max": 0.04028056576504315, "nauc_ndcg_at_3_std": -0.3446656104836864, "nauc_ndcg_at_5_diff1": 0.45040047273630857, "nauc_ndcg_at_5_max": 0.03979649691333102, "nauc_ndcg_at_5_std": -0.3511801978433322, "nauc_precision_at_1000_diff1": -0.07041101680846983, "nauc_precision_at_1000_max": 0.014657691604807463, "nauc_precision_at_1000_std": -0.039924232399408184, "nauc_precision_at_100_diff1": -0.0056135207397094685, "nauc_precision_at_100_max": 0.027912903091879524, "nauc_precision_at_100_std": -0.08066141684496765, "nauc_precision_at_10_diff1": 0.20485786635579623, "nauc_precision_at_10_max": -0.019954336112085008, "nauc_precision_at_10_std": -0.3488177923175864, "nauc_precision_at_1_diff1": 0.5230353749741389, "nauc_precision_at_1_max": 0.06587751414523899, "nauc_precision_at_1_std": -0.32859487926805514, "nauc_precision_at_20_diff1": 0.145619780468102, "nauc_precision_at_20_max": -0.03706454738456665, "nauc_precision_at_20_std": -0.3138145777743116, "nauc_precision_at_3_diff1": 0.27939462010075594, "nauc_precision_at_3_max": 0.01284089151169814, "nauc_precision_at_3_std": -0.34079680000131574, "nauc_precision_at_5_diff1": 0.23749631814377248, "nauc_precision_at_5_max": -0.009766776981789392, "nauc_precision_at_5_std": -0.35023524187203975, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.2152830103417114, "nauc_recall_at_100_max": 0.07106920383398822, "nauc_recall_at_100_std": -0.12740884920285253, "nauc_recall_at_10_diff1": 0.38412046069100564, "nauc_recall_at_10_max": 0.03512918479884001, "nauc_recall_at_10_std": -0.36799641807074623, "nauc_recall_at_1_diff1": 0.5518348816563906, "nauc_recall_at_1_max": 0.07377101119089663, "nauc_recall_at_1_std": -0.26809843855821786, "nauc_recall_at_20_diff1": 0.3465280191808201, "nauc_recall_at_20_max": 0.0017577641688437504, "nauc_recall_at_20_std": -0.3661202915437602, "nauc_recall_at_3_diff1": 0.42395320185818214, "nauc_recall_at_3_max": 0.027697685968404766, "nauc_recall_at_3_std": -0.3374561320362361, "nauc_recall_at_5_diff1": 0.39403075751656697, "nauc_recall_at_5_max": 0.02406606359316884, "nauc_recall_at_5_std": -0.3555034100963647, "ndcg_at_1": 0.29358, "ndcg_at_10": 0.37446, "ndcg_at_100": 0.44885, "ndcg_at_1000": 0.47516, "ndcg_at_20": 0.40232, "ndcg_at_3": 0.32326, "ndcg_at_5": 0.34683, "precision_at_1": 0.29358, "precision_at_10": 0.07768, "precision_at_100": 0.01292, "precision_at_1000": 0.00156, "precision_at_20": 0.04633, "precision_at_3": 0.18298, "precision_at_5": 0.1315, "recall_at_1": 0.20971, "recall_at_10": 0.47617, "recall_at_100": 0.80466, "recall_at_1000": 1.0, "recall_at_20": 0.5777, "recall_at_3": 0.34182, "recall_at_5": 0.40038 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.34661, "map_at_1": 0.16825, "map_at_10": 0.29036, "map_at_100": 0.30859, "map_at_1000": 0.31046, "map_at_20": 0.3002, "map_at_3": 0.24643, "map_at_5": 0.27468, "mrr_at_1": 0.2874617737003058, "mrr_at_10": 0.36174518227270513, "mrr_at_100": 0.3740905454658665, "mrr_at_1000": 0.37486441693210293, "mrr_at_20": 0.36896087973093267, "mrr_at_3": 0.33893985728848136, "mrr_at_5": 0.35178389398572896, "nauc_map_at_1000_diff1": 0.43408674116439533, "nauc_map_at_1000_max": -0.0781045934625065, "nauc_map_at_1000_std": -0.37262055431721036, "nauc_map_at_100_diff1": 0.43372688722543273, "nauc_map_at_100_max": -0.07898705574706322, "nauc_map_at_100_std": -0.3734511070798701, "nauc_map_at_10_diff1": 0.43007088282208156, "nauc_map_at_10_max": -0.08084375661708175, "nauc_map_at_10_std": -0.3755659759251913, "nauc_map_at_1_diff1": 0.4718554401290633, "nauc_map_at_1_max": -0.08010336469225167, "nauc_map_at_1_std": -0.3086076796871873, "nauc_map_at_20_diff1": 0.43273126305857496, "nauc_map_at_20_max": -0.08154413984121149, "nauc_map_at_20_std": -0.37713432219940535, "nauc_map_at_3_diff1": 0.4399922767182238, "nauc_map_at_3_max": -0.07177036482821501, "nauc_map_at_3_std": -0.35609704330938696, "nauc_map_at_5_diff1": 0.43315207872628014, "nauc_map_at_5_max": -0.07962376430405066, "nauc_map_at_5_std": -0.36861286758266093, "nauc_mrr_at_1000_diff1": 0.45778654097969523, "nauc_mrr_at_1000_max": -0.09093283852005311, "nauc_mrr_at_1000_std": -0.3980434677180335, "nauc_mrr_at_100_diff1": 0.45751526696935674, "nauc_mrr_at_100_max": -0.0909091440945732, "nauc_mrr_at_100_std": -0.39788117539104273, "nauc_mrr_at_10_diff1": 0.4566004428578541, "nauc_mrr_at_10_max": -0.09457397072149339, "nauc_mrr_at_10_std": -0.4026325473909036, "nauc_mrr_at_1_diff1": 0.5200549198032715, "nauc_mrr_at_1_max": -0.07890395420526032, "nauc_mrr_at_1_std": -0.393363625612812, "nauc_mrr_at_20_diff1": 0.45713824341360926, "nauc_mrr_at_20_max": -0.09116271873911638, "nauc_mrr_at_20_std": -0.3992770051600864, "nauc_mrr_at_3_diff1": 0.4611576322989618, "nauc_mrr_at_3_max": -0.09627581748308975, "nauc_mrr_at_3_std": -0.4030908830145351, "nauc_mrr_at_5_diff1": 0.45570439185164613, "nauc_mrr_at_5_max": -0.09489554045913565, "nauc_mrr_at_5_std": -0.4010519332946782, "nauc_ndcg_at_1000_diff1": 0.4282778137273464, "nauc_ndcg_at_1000_max": -0.07094144336277669, "nauc_ndcg_at_1000_std": -0.36650615049900964, "nauc_ndcg_at_100_diff1": 0.41884799526273186, "nauc_ndcg_at_100_max": -0.06995363016792937, "nauc_ndcg_at_100_std": -0.359887810898336, "nauc_ndcg_at_10_diff1": 0.41241717995825367, "nauc_ndcg_at_10_max": -0.0917564028796353, "nauc_ndcg_at_10_std": -0.39161872772526274, "nauc_ndcg_at_1_diff1": 0.5200549198032715, "nauc_ndcg_at_1_max": -0.07890395420526032, "nauc_ndcg_at_1_std": -0.393363625612812, "nauc_ndcg_at_20_diff1": 0.4187269134432203, "nauc_ndcg_at_20_max": -0.08777624732484189, "nauc_ndcg_at_20_std": -0.39029048439084557, "nauc_ndcg_at_3_diff1": 0.4305826194695503, "nauc_ndcg_at_3_max": -0.07609209333894651, "nauc_ndcg_at_3_std": -0.37738703403188706, "nauc_ndcg_at_5_diff1": 0.4153195435356488, "nauc_ndcg_at_5_max": -0.08866109540688547, "nauc_ndcg_at_5_std": -0.38023370215240054, "nauc_precision_at_1000_diff1": 0.09602906643187797, "nauc_precision_at_1000_max": 0.061603874890195504, "nauc_precision_at_1000_std": -0.032617277454407304, "nauc_precision_at_100_diff1": 0.1657852134906329, "nauc_precision_at_100_max": 0.04464208789278082, "nauc_precision_at_100_std": -0.10333452416246913, "nauc_precision_at_10_diff1": 0.2751306967754088, "nauc_precision_at_10_max": -0.06871625850600073, "nauc_precision_at_10_std": -0.3455839419676085, "nauc_precision_at_1_diff1": 0.5200549198032715, "nauc_precision_at_1_max": -0.07890395420526032, "nauc_precision_at_1_std": -0.393363625612812, "nauc_precision_at_20_diff1": 0.26429363273796425, "nauc_precision_at_20_max": -0.056867504122808527, "nauc_precision_at_20_std": -0.31859774863833207, "nauc_precision_at_3_diff1": 0.3585490915750111, "nauc_precision_at_3_max": -0.0588168533717345, "nauc_precision_at_3_std": -0.3778525156033515, "nauc_precision_at_5_diff1": 0.3173381453275914, "nauc_precision_at_5_max": -0.07158141598094207, "nauc_precision_at_5_std": -0.36335703054042195, "nauc_recall_at_1000_diff1": -0.10407317508841295, "nauc_recall_at_1000_max": 0.5028113070253024, "nauc_recall_at_1000_std": 0.44697916727698483, "nauc_recall_at_100_diff1": 0.26253714857740146, "nauc_recall_at_100_max": 0.007743556269314414, "nauc_recall_at_100_std": -0.1732423165494968, "nauc_recall_at_10_diff1": 0.31657628588215375, "nauc_recall_at_10_max": -0.10287505538155159, "nauc_recall_at_10_std": -0.38093458994929014, "nauc_recall_at_1_diff1": 0.4718554401290633, "nauc_recall_at_1_max": -0.08010336469225167, "nauc_recall_at_1_std": -0.3086076796871873, "nauc_recall_at_20_diff1": 0.3251959720635749, "nauc_recall_at_20_max": -0.08663571831697997, "nauc_recall_at_20_std": -0.3685351766024191, "nauc_recall_at_3_diff1": 0.350720002293693, "nauc_recall_at_3_max": -0.0770006441235346, "nauc_recall_at_3_std": -0.3379478997653626, "nauc_recall_at_5_diff1": 0.32461586682936044, "nauc_recall_at_5_max": -0.09297069153847383, "nauc_recall_at_5_std": -0.35374272867378154, "ndcg_at_1": 0.28746, "ndcg_at_10": 0.34661, "ndcg_at_100": 0.42667, "ndcg_at_1000": 0.46022, "ndcg_at_20": 0.37589, "ndcg_at_3": 0.29564, "ndcg_at_5": 0.31791, "precision_at_1": 0.28746, "precision_at_10": 0.08899, "precision_at_100": 0.01535, "precision_at_1000": 0.00194, "precision_at_20": 0.0539, "precision_at_3": 0.18858, "precision_at_5": 0.14495, "recall_at_1": 0.16825, "recall_at_10": 0.43537, "recall_at_100": 0.76952, "recall_at_1000": 0.99529, "recall_at_20": 0.53272, "recall_at_3": 0.29225, "recall_at_5": 0.36555 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.36844, "map_at_1": 0.20053, "map_at_10": 0.32119, "map_at_100": 0.33609, "map_at_1000": 0.33724, "map_at_20": 0.32875, "map_at_3": 0.28589, "map_at_5": 0.30422, "mrr_at_1": 0.2915309446254072, "mrr_at_10": 0.3576728193992037, "mrr_at_100": 0.3685340664387878, "mrr_at_1000": 0.36922795246376183, "mrr_at_20": 0.36325312691972717, "mrr_at_3": 0.3357763300760045, "mrr_at_5": 0.346199782844734, "nauc_map_at_1000_diff1": 0.41813669791893776, "nauc_map_at_1000_max": 0.007552584722714954, "nauc_map_at_1000_std": -0.3026040760903253, "nauc_map_at_100_diff1": 0.4177465362193772, "nauc_map_at_100_max": 0.007214830327060678, "nauc_map_at_100_std": -0.30268745302145583, "nauc_map_at_10_diff1": 0.42085448669537834, "nauc_map_at_10_max": 0.006364125031470106, "nauc_map_at_10_std": -0.3071472255158821, "nauc_map_at_1_diff1": 0.5184323474689903, "nauc_map_at_1_max": 0.014963896113339063, "nauc_map_at_1_std": -0.2440981121144059, "nauc_map_at_20_diff1": 0.4194286322842389, "nauc_map_at_20_max": 0.004354642450157929, "nauc_map_at_20_std": -0.307072227884935, "nauc_map_at_3_diff1": 0.44586295812962795, "nauc_map_at_3_max": 0.007930967745808305, "nauc_map_at_3_std": -0.29710924898322916, "nauc_map_at_5_diff1": 0.4311737957820591, "nauc_map_at_5_max": 0.007918955890765625, "nauc_map_at_5_std": -0.30633520852076374, "nauc_mrr_at_1000_diff1": 0.4354023503587192, "nauc_mrr_at_1000_max": -0.003970421913251222, "nauc_mrr_at_1000_std": -0.3113503346789748, "nauc_mrr_at_100_diff1": 0.43518787677162674, "nauc_mrr_at_100_max": -0.004117761167703212, "nauc_mrr_at_100_std": -0.3112766826824572, "nauc_mrr_at_10_diff1": 0.43652695829120913, "nauc_mrr_at_10_max": -0.0036706389480374666, "nauc_mrr_at_10_std": -0.3130101204856395, "nauc_mrr_at_1_diff1": 0.45804622982087095, "nauc_mrr_at_1_max": 0.0022866283980352733, "nauc_mrr_at_1_std": -0.2981204258269174, "nauc_mrr_at_20_diff1": 0.4358619448426231, "nauc_mrr_at_20_max": -0.004360869558686554, "nauc_mrr_at_20_std": -0.31266435497381806, "nauc_mrr_at_3_diff1": 0.44785727048628854, "nauc_mrr_at_3_max": -0.014145524448956039, "nauc_mrr_at_3_std": -0.3222094215558068, "nauc_mrr_at_5_diff1": 0.43954828980579475, "nauc_mrr_at_5_max": -0.006563909112455232, "nauc_mrr_at_5_std": -0.3182430061373452, "nauc_ndcg_at_1000_diff1": 0.40844284469814046, "nauc_ndcg_at_1000_max": 0.015080435112036725, "nauc_ndcg_at_1000_std": -0.291890093061604, "nauc_ndcg_at_100_diff1": 0.39704412813599954, "nauc_ndcg_at_100_max": 0.016089111193302506, "nauc_ndcg_at_100_std": -0.28219762613772403, "nauc_ndcg_at_10_diff1": 0.4088979768044545, "nauc_ndcg_at_10_max": 0.004666657059697, "nauc_ndcg_at_10_std": -0.31202664936338564, "nauc_ndcg_at_1_diff1": 0.45804622982087095, "nauc_ndcg_at_1_max": 0.0022866283980352733, "nauc_ndcg_at_1_std": -0.2981204258269174, "nauc_ndcg_at_20_diff1": 0.4032049470009599, "nauc_ndcg_at_20_max": -0.0006181694712085585, "nauc_ndcg_at_20_std": -0.3103864548925818, "nauc_ndcg_at_3_diff1": 0.4385036165208426, "nauc_ndcg_at_3_max": -0.004331691755456782, "nauc_ndcg_at_3_std": -0.31799870435750993, "nauc_ndcg_at_5_diff1": 0.422456674845733, "nauc_ndcg_at_5_max": 0.004322515747886231, "nauc_ndcg_at_5_std": -0.31603953506255617, "nauc_precision_at_1000_diff1": -0.08207023394380356, "nauc_precision_at_1000_max": 0.12415361386838557, "nauc_precision_at_1000_std": 0.04949077736997206, "nauc_precision_at_100_diff1": -0.024188274262316435, "nauc_precision_at_100_max": 0.09925301307678765, "nauc_precision_at_100_std": -0.013622993433942163, "nauc_precision_at_10_diff1": 0.14368637929984812, "nauc_precision_at_10_max": -0.01062236695603982, "nauc_precision_at_10_std": -0.2751113441776455, "nauc_precision_at_1_diff1": 0.45804622982087095, "nauc_precision_at_1_max": 0.0022866283980352733, "nauc_precision_at_1_std": -0.2981204258269174, "nauc_precision_at_20_diff1": 0.1082263641660178, "nauc_precision_at_20_max": -0.015697333715391372, "nauc_precision_at_20_std": -0.2429625882645431, "nauc_precision_at_3_diff1": 0.2532752428987566, "nauc_precision_at_3_max": -0.01810692192437379, "nauc_precision_at_3_std": -0.3180735176178936, "nauc_precision_at_5_diff1": 0.19985716355588498, "nauc_precision_at_5_max": -0.011542622259030643, "nauc_precision_at_5_std": -0.3098687288272108, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.24172121926780435, "nauc_recall_at_100_max": 0.0724716126973392, "nauc_recall_at_100_std": -0.10402632689466423, "nauc_recall_at_10_diff1": 0.3500669517021082, "nauc_recall_at_10_max": 0.012805663309776125, "nauc_recall_at_10_std": -0.29977093252907944, "nauc_recall_at_1_diff1": 0.5184323474689903, "nauc_recall_at_1_max": 0.014963896113339063, "nauc_recall_at_1_std": -0.2440981121144059, "nauc_recall_at_20_diff1": 0.3195452308319116, "nauc_recall_at_20_max": -0.007130058012837736, "nauc_recall_at_20_std": -0.290770239297093, "nauc_recall_at_3_diff1": 0.4266731339339648, "nauc_recall_at_3_max": -0.005380600005323247, "nauc_recall_at_3_std": -0.31113757836239436, "nauc_recall_at_5_diff1": 0.3850481536457508, "nauc_recall_at_5_max": 0.008066667725823217, "nauc_recall_at_5_std": -0.31577804754380534, "ndcg_at_1": 0.29153, "ndcg_at_10": 0.36844, "ndcg_at_100": 0.44152, "ndcg_at_1000": 0.4695, "ndcg_at_20": 0.39218, "ndcg_at_3": 0.31583, "ndcg_at_5": 0.33543, "precision_at_1": 0.29153, "precision_at_10": 0.08062, "precision_at_100": 0.01357, "precision_at_1000": 0.00167, "precision_at_20": 0.04739, "precision_at_3": 0.18187, "precision_at_5": 0.13029, "recall_at_1": 0.20053, "recall_at_10": 0.46927, "recall_at_100": 0.79471, "recall_at_1000": 1.0, "recall_at_20": 0.55277, "recall_at_3": 0.32904, "recall_at_5": 0.37916 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/model_meta.json ================================================ {"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "loader": null} ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 1549.6882767677307, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.70782, "map_at_1": 0.54386, "map_at_10": 0.65351, "map_at_100": 0.65816, "map_at_1000": 0.65822, "map_at_20": 0.65658, "map_at_3": 0.62427, "map_at_5": 0.64313, "mrr_at_1": 0.543859649122807, "mrr_at_10": 0.6535087719298245, "mrr_at_100": 0.6581566406059242, "mrr_at_1000": 0.6582175567852613, "mrr_at_20": 0.6565841687552212, "mrr_at_3": 0.6242690058479534, "mrr_at_5": 0.6431286549707603, "nauc_map_at_1000_diff1": 0.4589811760453532, "nauc_map_at_1000_max": 0.10234092569414859, "nauc_map_at_1000_std": -0.356311102381068, "nauc_map_at_100_diff1": 0.45883771525940487, "nauc_map_at_100_max": 0.10218882872908919, "nauc_map_at_100_std": -0.3563584758354272, "nauc_map_at_10_diff1": 0.45785560488749766, "nauc_map_at_10_max": 0.1083158493065166, "nauc_map_at_10_std": -0.3542947811883265, "nauc_map_at_1_diff1": 0.45100389919557227, "nauc_map_at_1_max": 0.07773262230195767, "nauc_map_at_1_std": -0.29191787775456185, "nauc_map_at_20_diff1": 0.45889280176699143, "nauc_map_at_20_max": 0.1041613429769374, "nauc_map_at_20_std": -0.3562801591947266, "nauc_map_at_3_diff1": 0.4581271634272649, "nauc_map_at_3_max": 0.07010811847042181, "nauc_map_at_3_std": -0.36784309569200174, "nauc_map_at_5_diff1": 0.46628806215271035, "nauc_map_at_5_max": 0.0998584281053696, "nauc_map_at_5_std": -0.37400894462811973, "nauc_mrr_at_1000_diff1": 0.4589811760453532, "nauc_mrr_at_1000_max": 0.10234092569414859, "nauc_mrr_at_1000_std": -0.356311102381068, "nauc_mrr_at_100_diff1": 0.45883771525940487, "nauc_mrr_at_100_max": 0.10218882872908919, "nauc_mrr_at_100_std": -0.3563584758354272, "nauc_mrr_at_10_diff1": 0.45785560488749766, "nauc_mrr_at_10_max": 0.1083158493065166, "nauc_mrr_at_10_std": -0.3542947811883265, "nauc_mrr_at_1_diff1": 0.45100389919557227, "nauc_mrr_at_1_max": 0.07773262230195767, "nauc_mrr_at_1_std": -0.29191787775456185, "nauc_mrr_at_20_diff1": 0.45889280176699143, "nauc_mrr_at_20_max": 0.1041613429769374, "nauc_mrr_at_20_std": -0.3562801591947266, "nauc_mrr_at_3_diff1": 0.4581271634272649, "nauc_mrr_at_3_max": 0.07010811847042181, "nauc_mrr_at_3_std": -0.36784309569200174, "nauc_mrr_at_5_diff1": 0.46628806215271035, "nauc_mrr_at_5_max": 0.0998584281053696, "nauc_mrr_at_5_std": -0.37400894462811973, "nauc_ndcg_at_1000_diff1": 0.4579233057407457, "nauc_ndcg_at_1000_max": 0.11353070121982693, "nauc_ndcg_at_1000_std": -0.36344840933846423, "nauc_ndcg_at_100_diff1": 0.45430840688203755, "nauc_ndcg_at_100_max": 0.10962723815228848, "nauc_ndcg_at_100_std": -0.3646029972953465, "nauc_ndcg_at_10_diff1": 0.4503205658127994, "nauc_ndcg_at_10_max": 0.1462947779227858, "nauc_ndcg_at_10_std": -0.3628340802117219, "nauc_ndcg_at_1_diff1": 0.45100389919557227, "nauc_ndcg_at_1_max": 0.07773262230195767, "nauc_ndcg_at_1_std": -0.29191787775456185, "nauc_ndcg_at_20_diff1": 0.4570695553298544, "nauc_ndcg_at_20_max": 0.13210328427557108, "nauc_ndcg_at_20_std": -0.3660073237106156, "nauc_ndcg_at_3_diff1": 0.45833625612690976, "nauc_ndcg_at_3_max": 0.0708825922620583, "nauc_ndcg_at_3_std": -0.3897144449317193, "nauc_ndcg_at_5_diff1": 0.47257932918610845, "nauc_ndcg_at_5_max": 0.1250269996217293, "nauc_ndcg_at_5_std": -0.40588530243383514, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": -0.16932645024141074, "nauc_precision_at_100_max": -0.5637901180614636, "nauc_precision_at_100_std": -0.5637901180614636, "nauc_precision_at_10_diff1": 0.3830070681610025, "nauc_precision_at_10_max": 0.418519573108106, "nauc_precision_at_10_std": -0.4009186122719858, "nauc_precision_at_1_diff1": 0.45100389919557227, "nauc_precision_at_1_max": 0.07773262230195767, "nauc_precision_at_1_std": -0.29191787775456185, "nauc_precision_at_20_diff1": 0.4392864027297254, "nauc_precision_at_20_max": 0.41063793833902507, "nauc_precision_at_20_std": -0.4208684992861658, "nauc_precision_at_3_diff1": 0.45816122675541565, "nauc_precision_at_3_max": 0.07498683816017587, "nauc_precision_at_3_std": -0.4649198530675704, "nauc_precision_at_5_diff1": 0.5008433631808221, "nauc_precision_at_5_max": 0.24698005260037342, "nauc_precision_at_5_std": -0.5508019949459134, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": -0.16932645024147489, "nauc_recall_at_100_max": -0.5637901180615443, "nauc_recall_at_100_std": -0.5637901180615443, "nauc_recall_at_10_diff1": 0.38300706816100444, "nauc_recall_at_10_max": 0.4185195731081076, "nauc_recall_at_10_std": -0.40091861227198716, "nauc_recall_at_1_diff1": 0.45100389919557227, "nauc_recall_at_1_max": 0.07773262230195767, "nauc_recall_at_1_std": -0.29191787775456185, "nauc_recall_at_20_diff1": 0.43928640272972663, "nauc_recall_at_20_max": 0.4106379383390254, "nauc_recall_at_20_std": -0.420868499286167, "nauc_recall_at_3_diff1": 0.45816122675541615, "nauc_recall_at_3_max": 0.07498683816017519, "nauc_recall_at_3_std": -0.4649198530675706, "nauc_recall_at_5_diff1": 0.5008433631808211, "nauc_recall_at_5_max": 0.24698005260037348, "nauc_recall_at_5_std": -0.5508019949459128, "ndcg_at_1": 0.54386, "ndcg_at_10": 0.70782, "ndcg_at_100": 0.73155, "ndcg_at_1000": 0.73278, "ndcg_at_20": 0.71895, "ndcg_at_3": 0.6509, "ndcg_at_5": 0.68413, "precision_at_1": 0.54386, "precision_at_10": 0.08772, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "precision_at_20": 0.04605, "precision_at_3": 0.24269, "precision_at_5": 0.1614, "recall_at_1": 0.54386, "recall_at_10": 0.87719, "recall_at_100": 0.99123, "recall_at_1000": 1.0, "recall_at_20": 0.92105, "recall_at_3": 0.72807, "recall_at_5": 0.80702 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 1465.829463481903, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.91747, "map_at_1": 0.86, "map_at_10": 0.90096, "map_at_100": 0.90212, "map_at_1000": 0.90215, "map_at_20": 0.90174, "map_at_3": 0.89463, "map_at_5": 0.89835, "mrr_at_1": 0.86, "mrr_at_10": 0.900962081128748, "mrr_at_100": 0.9021249834348359, "mrr_at_1000": 0.9021471859406125, "mrr_at_20": 0.9017424595194203, "mrr_at_3": 0.8946296296296297, "mrr_at_5": 0.8983518518518521, "nauc_map_at_1000_diff1": 0.9038265405786627, "nauc_map_at_1000_max": 0.7775056855610126, "nauc_map_at_1000_std": -0.03270651100548468, "nauc_map_at_100_diff1": 0.9038184890206553, "nauc_map_at_100_max": 0.7775375908773677, "nauc_map_at_100_std": -0.03250325754067984, "nauc_map_at_10_diff1": 0.9039191701071102, "nauc_map_at_10_max": 0.7790969647189588, "nauc_map_at_10_std": -0.028729734505420247, "nauc_map_at_1_diff1": 0.9094614567456455, "nauc_map_at_1_max": 0.75417609656213, "nauc_map_at_1_std": -0.12773851862115637, "nauc_map_at_20_diff1": 0.9038802928670955, "nauc_map_at_20_max": 0.7782253662276372, "nauc_map_at_20_std": -0.03015435395933517, "nauc_map_at_3_diff1": 0.9007654653790561, "nauc_map_at_3_max": 0.7781202095312141, "nauc_map_at_3_std": -0.02215962536115016, "nauc_map_at_5_diff1": 0.9019619140503947, "nauc_map_at_5_max": 0.7794158343254726, "nauc_map_at_5_std": -0.02349717216127941, "nauc_mrr_at_1000_diff1": 0.9038265405786627, "nauc_mrr_at_1000_max": 0.7775056855610126, "nauc_mrr_at_1000_std": -0.03270651100548468, "nauc_mrr_at_100_diff1": 0.9038184890206553, "nauc_mrr_at_100_max": 0.7775375908773677, "nauc_mrr_at_100_std": -0.03250325754067984, "nauc_mrr_at_10_diff1": 0.9039191701071102, "nauc_mrr_at_10_max": 0.7790969647189588, "nauc_mrr_at_10_std": -0.028729734505420247, "nauc_mrr_at_1_diff1": 0.9094614567456455, "nauc_mrr_at_1_max": 0.75417609656213, "nauc_mrr_at_1_std": -0.12773851862115637, "nauc_mrr_at_20_diff1": 0.9038802928670955, "nauc_mrr_at_20_max": 0.7782253662276372, "nauc_mrr_at_20_std": -0.03015435395933517, "nauc_mrr_at_3_diff1": 0.9007654653790561, "nauc_mrr_at_3_max": 0.7781202095312141, "nauc_mrr_at_3_std": -0.02215962536115016, "nauc_mrr_at_5_diff1": 0.9019619140503947, "nauc_mrr_at_5_max": 0.7794158343254726, "nauc_mrr_at_5_std": -0.02349717216127941, "nauc_ndcg_at_1000_diff1": 0.9037169251607843, "nauc_ndcg_at_1000_max": 0.7825042250574558, "nauc_ndcg_at_1000_std": -0.013103733510511814, "nauc_ndcg_at_100_diff1": 0.90350685579764, "nauc_ndcg_at_100_max": 0.7833044667596207, "nauc_ndcg_at_100_std": -0.007790985921977495, "nauc_ndcg_at_10_diff1": 0.9047554274230343, "nauc_ndcg_at_10_max": 0.7929516389052672, "nauc_ndcg_at_10_std": 0.017937956553844697, "nauc_ndcg_at_1_diff1": 0.9094614567456455, "nauc_ndcg_at_1_max": 0.75417609656213, "nauc_ndcg_at_1_std": -0.12773851862115637, "nauc_ndcg_at_20_diff1": 0.9044326135405133, "nauc_ndcg_at_20_max": 0.7893189448516453, "nauc_ndcg_at_20_std": 0.013078954190118957, "nauc_ndcg_at_3_diff1": 0.8974391156578337, "nauc_ndcg_at_3_max": 0.7892974922303669, "nauc_ndcg_at_3_std": 0.027141537353611352, "nauc_ndcg_at_5_diff1": 0.8995251483219957, "nauc_ndcg_at_5_max": 0.7929255775725709, "nauc_ndcg_at_5_std": 0.028589749196155496, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8692810457516348, "nauc_precision_at_100_max": 0.9128540305010695, "nauc_precision_at_100_std": 0.8638344226579172, "nauc_precision_at_10_diff1": 0.918300653594768, "nauc_precision_at_10_max": 0.914195563282784, "nauc_precision_at_10_std": 0.41237644483080255, "nauc_precision_at_1_diff1": 0.9094614567456455, "nauc_precision_at_1_max": 0.75417609656213, "nauc_precision_at_1_std": -0.12773851862115637, "nauc_precision_at_20_diff1": 0.920116194625998, "nauc_precision_at_20_max": 0.930750077808904, "nauc_precision_at_20_std": 0.5774198568316234, "nauc_precision_at_3_diff1": 0.882701103040088, "nauc_precision_at_3_max": 0.839900932124267, "nauc_precision_at_3_std": 0.25035211824843107, "nauc_precision_at_5_diff1": 0.8863990040460619, "nauc_precision_at_5_max": 0.8712625791057137, "nauc_precision_at_5_std": 0.3274198568316172, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8692810457516407, "nauc_recall_at_100_max": 0.9128540305010676, "nauc_recall_at_100_std": 0.8638344226579231, "nauc_recall_at_10_diff1": 0.9183006535947711, "nauc_recall_at_10_max": 0.9141955632827841, "nauc_recall_at_10_std": 0.4123764448308052, "nauc_recall_at_1_diff1": 0.9094614567456455, "nauc_recall_at_1_max": 0.75417609656213, "nauc_recall_at_1_std": -0.12773851862115637, "nauc_recall_at_20_diff1": 0.9201161946259977, "nauc_recall_at_20_max": 0.9307500778089021, "nauc_recall_at_20_std": 0.5774198568316253, "nauc_recall_at_3_diff1": 0.8827011030400855, "nauc_recall_at_3_max": 0.8399009321242628, "nauc_recall_at_3_std": 0.25035211824842885, "nauc_recall_at_5_diff1": 0.8863990040460626, "nauc_recall_at_5_max": 0.8712625791057151, "nauc_recall_at_5_std": 0.32741985683162, "ndcg_at_1": 0.86, "ndcg_at_10": 0.91747, "ndcg_at_100": 0.92349, "ndcg_at_1000": 0.92395, "ndcg_at_20": 0.92048, "ndcg_at_3": 0.90493, "ndcg_at_5": 0.91149, "precision_at_1": 0.86, "precision_at_10": 0.09678, "precision_at_100": 0.00997, "precision_at_1000": 0.001, "precision_at_20": 0.049, "precision_at_3": 0.31148, "precision_at_5": 0.19, "recall_at_1": 0.86, "recall_at_10": 0.96778, "recall_at_100": 0.99667, "recall_at_1000": 1.0, "recall_at_20": 0.98, "recall_at_3": 0.93444, "recall_at_5": 0.95 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.89583, "map_at_1": 0.82444, "map_at_10": 0.8743, "map_at_100": 0.87606, "map_at_1000": 0.87609, "map_at_20": 0.87558, "map_at_3": 0.86704, "map_at_5": 0.87109, "mrr_at_1": 0.8244444444444444, "mrr_at_10": 0.8743029100529099, "mrr_at_100": 0.8760601012965096, "mrr_at_1000": 0.8760935672797767, "mrr_at_20": 0.8755839329277152, "mrr_at_3": 0.867037037037037, "mrr_at_5": 0.8710925925925928, "nauc_map_at_1000_diff1": 0.8709388580881812, "nauc_map_at_1000_max": 0.7257094345122158, "nauc_map_at_1000_std": -0.15810898862814932, "nauc_map_at_100_diff1": 0.8709262914659613, "nauc_map_at_100_max": 0.7256623523411022, "nauc_map_at_100_std": -0.15801895449476316, "nauc_map_at_10_diff1": 0.8704691614053939, "nauc_map_at_10_max": 0.7273011370477138, "nauc_map_at_10_std": -0.15290381074544582, "nauc_map_at_1_diff1": 0.8785551478415506, "nauc_map_at_1_max": 0.6949085148217262, "nauc_map_at_1_std": -0.21110157923079828, "nauc_map_at_20_diff1": 0.8709799295251514, "nauc_map_at_20_max": 0.7264313439462893, "nauc_map_at_20_std": -0.15571474736075042, "nauc_map_at_3_diff1": 0.8700221104222086, "nauc_map_at_3_max": 0.7320877164020355, "nauc_map_at_3_std": -0.16073965722079653, "nauc_map_at_5_diff1": 0.869566359696301, "nauc_map_at_5_max": 0.7302110743273366, "nauc_map_at_5_std": -0.15265026941719037, "nauc_mrr_at_1000_diff1": 0.8709388580881812, "nauc_mrr_at_1000_max": 0.7257094345122158, "nauc_mrr_at_1000_std": -0.15810898862814932, "nauc_mrr_at_100_diff1": 0.8709262914659613, "nauc_mrr_at_100_max": 0.7256623523411022, "nauc_mrr_at_100_std": -0.15801895449476316, "nauc_mrr_at_10_diff1": 0.8704691614053939, "nauc_mrr_at_10_max": 0.7273011370477138, "nauc_mrr_at_10_std": -0.15290381074544582, "nauc_mrr_at_1_diff1": 0.8785551478415506, "nauc_mrr_at_1_max": 0.6949085148217262, "nauc_mrr_at_1_std": -0.21110157923079828, "nauc_mrr_at_20_diff1": 0.8709799295251514, "nauc_mrr_at_20_max": 0.7264313439462893, "nauc_mrr_at_20_std": -0.15571474736075042, "nauc_mrr_at_3_diff1": 0.8700221104222086, "nauc_mrr_at_3_max": 0.7320877164020355, "nauc_mrr_at_3_std": -0.16073965722079653, "nauc_mrr_at_5_diff1": 0.869566359696301, "nauc_mrr_at_5_max": 0.7302110743273366, "nauc_mrr_at_5_std": -0.15265026941719037, "nauc_ndcg_at_1000_diff1": 0.8701658703321531, "nauc_ndcg_at_1000_max": 0.7290703499029934, "nauc_ndcg_at_1000_std": -0.14716087659059404, "nauc_ndcg_at_100_diff1": 0.8699994343208388, "nauc_ndcg_at_100_max": 0.7277723702500393, "nauc_ndcg_at_100_std": -0.14525520109240692, "nauc_ndcg_at_10_diff1": 0.8685949260017891, "nauc_ndcg_at_10_max": 0.7366320436014431, "nauc_ndcg_at_10_std": -0.11810262632565337, "nauc_ndcg_at_1_diff1": 0.8785551478415506, "nauc_ndcg_at_1_max": 0.6949085148217262, "nauc_ndcg_at_1_std": -0.21110157923079828, "nauc_ndcg_at_20_diff1": 0.8700391722631855, "nauc_ndcg_at_20_max": 0.7331283394586031, "nauc_ndcg_at_20_std": -0.12834392630963246, "nauc_ndcg_at_3_diff1": 0.8671925487503549, "nauc_ndcg_at_3_max": 0.7471430113550027, "nauc_ndcg_at_3_std": -0.13793944800144223, "nauc_ndcg_at_5_diff1": 0.8661644204364798, "nauc_ndcg_at_5_max": 0.7442138775204387, "nauc_ndcg_at_5_std": -0.1199269593757352, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8552754435107516, "nauc_precision_at_100_max": 0.5719887955182057, "nauc_precision_at_100_std": 0.07301587301587033, "nauc_precision_at_10_diff1": 0.855138133684845, "nauc_precision_at_10_max": 0.8101142417751384, "nauc_precision_at_10_std": 0.21226725984510994, "nauc_precision_at_1_diff1": 0.8785551478415506, "nauc_precision_at_1_max": 0.6949085148217262, "nauc_precision_at_1_std": -0.21110157923079828, "nauc_precision_at_20_diff1": 0.8630563336445668, "nauc_precision_at_20_max": 0.8091347650171139, "nauc_precision_at_20_std": 0.32031331050939155, "nauc_precision_at_3_diff1": 0.8546270359995863, "nauc_precision_at_3_max": 0.8156966490299831, "nauc_precision_at_3_std": -0.03289371304077219, "nauc_precision_at_5_diff1": 0.8471055088702153, "nauc_precision_at_5_max": 0.822604041616646, "nauc_precision_at_5_std": 0.072362278244633, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8552754435107421, "nauc_recall_at_100_max": 0.5719887955182243, "nauc_recall_at_100_std": 0.07301587301587503, "nauc_recall_at_10_diff1": 0.8551381336848447, "nauc_recall_at_10_max": 0.8101142417751407, "nauc_recall_at_10_std": 0.21226725984511266, "nauc_recall_at_1_diff1": 0.8785551478415506, "nauc_recall_at_1_max": 0.6949085148217262, "nauc_recall_at_1_std": -0.21110157923079828, "nauc_recall_at_20_diff1": 0.8630563336445648, "nauc_recall_at_20_max": 0.8091347650171263, "nauc_recall_at_20_std": 0.3203133105093926, "nauc_recall_at_3_diff1": 0.8546270359995847, "nauc_recall_at_3_max": 0.8156966490299821, "nauc_recall_at_3_std": -0.032893713040773036, "nauc_recall_at_5_diff1": 0.8471055088702144, "nauc_recall_at_5_max": 0.8226040416166456, "nauc_recall_at_5_std": 0.07236227824463086, "ndcg_at_1": 0.82444, "ndcg_at_10": 0.89583, "ndcg_at_100": 0.90326, "ndcg_at_1000": 0.90401, "ndcg_at_20": 0.90038, "ndcg_at_3": 0.88066, "ndcg_at_5": 0.88798, "precision_at_1": 0.82444, "precision_at_10": 0.09622, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "precision_at_20": 0.049, "precision_at_3": 0.30667, "precision_at_5": 0.18756, "recall_at_1": 0.82444, "recall_at_10": 0.96222, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "recall_at_20": 0.98, "recall_at_3": 0.92, "recall_at_5": 0.93778 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.83689, "map_at_1": 0.75444, "map_at_10": 0.81117, "map_at_100": 0.81415, "map_at_1000": 0.81427, "map_at_20": 0.81333, "map_at_3": 0.80148, "map_at_5": 0.8072, "mrr_at_1": 0.7544444444444445, "mrr_at_10": 0.8111746031746031, "mrr_at_100": 0.8141532860912081, "mrr_at_1000": 0.8142724960071842, "mrr_at_20": 0.8133258077014527, "mrr_at_3": 0.8014814814814816, "mrr_at_5": 0.8072037037037036, "nauc_map_at_1000_diff1": 0.8083049100518027, "nauc_map_at_1000_max": 0.7307283436029658, "nauc_map_at_1000_std": 0.3433873013726241, "nauc_map_at_100_diff1": 0.8082815175989179, "nauc_map_at_100_max": 0.7307524396818857, "nauc_map_at_100_std": 0.34351917177659325, "nauc_map_at_10_diff1": 0.8076607264797645, "nauc_map_at_10_max": 0.7310244696138717, "nauc_map_at_10_std": 0.3440074274709909, "nauc_map_at_1_diff1": 0.8267667483901106, "nauc_map_at_1_max": 0.7076927796275018, "nauc_map_at_1_std": 0.29862171326633397, "nauc_map_at_20_diff1": 0.8079446901936352, "nauc_map_at_20_max": 0.7307802533053914, "nauc_map_at_20_std": 0.3441420971715125, "nauc_map_at_3_diff1": 0.8057650891455094, "nauc_map_at_3_max": 0.7360422173098745, "nauc_map_at_3_std": 0.3479020108426812, "nauc_map_at_5_diff1": 0.8081779021885703, "nauc_map_at_5_max": 0.7336408216313035, "nauc_map_at_5_std": 0.3462356493217495, "nauc_mrr_at_1000_diff1": 0.8083049100518027, "nauc_mrr_at_1000_max": 0.7307283436029658, "nauc_mrr_at_1000_std": 0.3433873013726241, "nauc_mrr_at_100_diff1": 0.8082815175989179, "nauc_mrr_at_100_max": 0.7307524396818857, "nauc_mrr_at_100_std": 0.34351917177659325, "nauc_mrr_at_10_diff1": 0.8076607264797645, "nauc_mrr_at_10_max": 0.7310244696138717, "nauc_mrr_at_10_std": 0.3440074274709909, "nauc_mrr_at_1_diff1": 0.8267667483901106, "nauc_mrr_at_1_max": 0.7076927796275018, "nauc_mrr_at_1_std": 0.29862171326633397, "nauc_mrr_at_20_diff1": 0.8079446901936352, "nauc_mrr_at_20_max": 0.7307802533053914, "nauc_mrr_at_20_std": 0.3441420971715125, "nauc_mrr_at_3_diff1": 0.8057650891455094, "nauc_mrr_at_3_max": 0.7360422173098745, "nauc_mrr_at_3_std": 0.3479020108426812, "nauc_mrr_at_5_diff1": 0.8081779021885703, "nauc_mrr_at_5_max": 0.7336408216313035, "nauc_mrr_at_5_std": 0.3462356493217495, "nauc_ndcg_at_1000_diff1": 0.8052545003984145, "nauc_ndcg_at_1000_max": 0.7337087713187748, "nauc_ndcg_at_1000_std": 0.3521440097869082, "nauc_ndcg_at_100_diff1": 0.8043710163779854, "nauc_ndcg_at_100_max": 0.73466028469277, "nauc_ndcg_at_100_std": 0.3570677509208818, "nauc_ndcg_at_10_diff1": 0.8011042496275814, "nauc_ndcg_at_10_max": 0.7359503405504093, "nauc_ndcg_at_10_std": 0.3599063932712571, "nauc_ndcg_at_1_diff1": 0.8267667483901106, "nauc_ndcg_at_1_max": 0.7076927796275018, "nauc_ndcg_at_1_std": 0.29862171326633397, "nauc_ndcg_at_20_diff1": 0.8017400377524357, "nauc_ndcg_at_20_max": 0.7349810079603146, "nauc_ndcg_at_20_std": 0.36136200111436173, "nauc_ndcg_at_3_diff1": 0.7979086996889073, "nauc_ndcg_at_3_max": 0.746526925607291, "nauc_ndcg_at_3_std": 0.36613741670310723, "nauc_ndcg_at_5_diff1": 0.8020528589215355, "nauc_ndcg_at_5_max": 0.7423490426830931, "nauc_ndcg_at_5_std": 0.36493608101353514, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.7593493537766106, "nauc_precision_at_100_max": 0.7853948596982715, "nauc_precision_at_100_std": 0.6117008206791629, "nauc_precision_at_10_diff1": 0.7638406473700596, "nauc_precision_at_10_max": 0.759215686274508, "nauc_precision_at_10_std": 0.45134765017117956, "nauc_precision_at_1_diff1": 0.8267667483901106, "nauc_precision_at_1_max": 0.7076927796275018, "nauc_precision_at_1_std": 0.29862171326633397, "nauc_precision_at_20_diff1": 0.749423578955388, "nauc_precision_at_20_max": 0.7604946740600972, "nauc_precision_at_20_std": 0.515272775776977, "nauc_precision_at_3_diff1": 0.7673231246365565, "nauc_precision_at_3_max": 0.7872921108742008, "nauc_precision_at_3_std": 0.4370420624151964, "nauc_precision_at_5_diff1": 0.7750274260426531, "nauc_precision_at_5_max": 0.7808240828545379, "nauc_precision_at_5_std": 0.45057734144028394, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.7593493537765984, "nauc_recall_at_100_max": 0.7853948596982716, "nauc_recall_at_100_std": 0.6117008206791502, "nauc_recall_at_10_diff1": 0.7638406473700595, "nauc_recall_at_10_max": 0.7592156862745094, "nauc_recall_at_10_std": 0.45134765017118056, "nauc_recall_at_1_diff1": 0.8267667483901106, "nauc_recall_at_1_max": 0.7076927796275018, "nauc_recall_at_1_std": 0.29862171326633397, "nauc_recall_at_20_diff1": 0.7494235789553924, "nauc_recall_at_20_max": 0.7604946740601015, "nauc_recall_at_20_std": 0.5152727757769768, "nauc_recall_at_3_diff1": 0.7673231246365568, "nauc_recall_at_3_max": 0.7872921108741997, "nauc_recall_at_3_std": 0.4370420624151967, "nauc_recall_at_5_diff1": 0.7750274260426544, "nauc_recall_at_5_max": 0.7808240828545392, "nauc_recall_at_5_std": 0.4505773414402868, "ndcg_at_1": 0.75444, "ndcg_at_10": 0.83689, "ndcg_at_100": 0.85052, "ndcg_at_1000": 0.85332, "ndcg_at_20": 0.84438, "ndcg_at_3": 0.8168, "ndcg_at_5": 0.82722, "precision_at_1": 0.75444, "precision_at_10": 0.09167, "precision_at_100": 0.00979, "precision_at_1000": 0.001, "precision_at_20": 0.04728, "precision_at_3": 0.28704, "precision_at_5": 0.17733, "recall_at_1": 0.75444, "recall_at_10": 0.91667, "recall_at_100": 0.97889, "recall_at_1000": 1.0, "recall_at_20": 0.94556, "recall_at_3": 0.86111, "recall_at_5": 0.88667 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 26252.303451776505, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.77042, "map_at_1": 0.46592, "map_at_10": 0.72027, "map_at_100": 0.72551, "map_at_1000": 0.72587, "map_at_20": 0.7237, "map_at_3": 0.67501, "map_at_5": 0.70711, "mrr_at_1": 0.7195945945945946, "mrr_at_10": 0.7861949002574001, "mrr_at_100": 0.7875507651031451, "mrr_at_1000": 0.7878467841424145, "mrr_at_20": 0.7870200135825133, "mrr_at_3": 0.777027027027027, "mrr_at_5": 0.7842060810810809, "nauc_map_at_1000_diff1": 0.5798460629285543, "nauc_map_at_1000_max": 0.5709365893012763, "nauc_map_at_1000_std": 0.24963229926789668, "nauc_map_at_100_diff1": 0.579850903420959, "nauc_map_at_100_max": 0.5713400656786721, "nauc_map_at_100_std": 0.24999648567848107, "nauc_map_at_10_diff1": 0.5803471345606962, "nauc_map_at_10_max": 0.5712431756202678, "nauc_map_at_10_std": 0.2421700727422, "nauc_map_at_1_diff1": 0.5512870662759014, "nauc_map_at_1_max": 0.22338378158416522, "nauc_map_at_1_std": 0.00987834782975795, "nauc_map_at_20_diff1": 0.5796823602883358, "nauc_map_at_20_max": 0.5705266832163431, "nauc_map_at_20_std": 0.2468221196093924, "nauc_map_at_3_diff1": 0.5607918178010085, "nauc_map_at_3_max": 0.5245880639557309, "nauc_map_at_3_std": 0.18915077275137712, "nauc_map_at_5_diff1": 0.5721320973412499, "nauc_map_at_5_max": 0.5576511822270637, "nauc_map_at_5_std": 0.21990609077755557, "nauc_mrr_at_1000_diff1": 0.6488061140795381, "nauc_mrr_at_1000_max": 0.6093968304137143, "nauc_mrr_at_1000_std": 0.318541599167972, "nauc_mrr_at_100_diff1": 0.6489704492564633, "nauc_mrr_at_100_max": 0.6098143688186834, "nauc_mrr_at_100_std": 0.31903470358423563, "nauc_mrr_at_10_diff1": 0.6486553682311893, "nauc_mrr_at_10_max": 0.6112651859819409, "nauc_mrr_at_10_std": 0.3203063096386445, "nauc_mrr_at_1_diff1": 0.6206797143265911, "nauc_mrr_at_1_max": 0.4945201476497312, "nauc_mrr_at_1_std": 0.1972028407123734, "nauc_mrr_at_20_diff1": 0.6488366321442393, "nauc_mrr_at_20_max": 0.6098816742420539, "nauc_mrr_at_20_std": 0.3181061242841399, "nauc_mrr_at_3_diff1": 0.6483813289040319, "nauc_mrr_at_3_max": 0.6135880843665676, "nauc_mrr_at_3_std": 0.317010318288184, "nauc_mrr_at_5_diff1": 0.6464749557254158, "nauc_mrr_at_5_max": 0.6146318548989139, "nauc_mrr_at_5_std": 0.3227119161042139, "nauc_ndcg_at_1000_diff1": 0.6113128952769129, "nauc_ndcg_at_1000_max": 0.621181818996922, "nauc_ndcg_at_1000_std": 0.32647263494138723, "nauc_ndcg_at_100_diff1": 0.6098657037764337, "nauc_ndcg_at_100_max": 0.6320186725086027, "nauc_ndcg_at_100_std": 0.33838728012231184, "nauc_ndcg_at_10_diff1": 0.6077329427579963, "nauc_ndcg_at_10_max": 0.6315137058304188, "nauc_ndcg_at_10_std": 0.31395351203019783, "nauc_ndcg_at_1_diff1": 0.6206797143265911, "nauc_ndcg_at_1_max": 0.4945201476497312, "nauc_ndcg_at_1_std": 0.1972028407123734, "nauc_ndcg_at_20_diff1": 0.6070561082184659, "nauc_ndcg_at_20_max": 0.6292145515101722, "nauc_ndcg_at_20_std": 0.3227008210375962, "nauc_ndcg_at_3_diff1": 0.5867661134954778, "nauc_ndcg_at_3_max": 0.5794669490947111, "nauc_ndcg_at_3_std": 0.24828332002512565, "nauc_ndcg_at_5_diff1": 0.5926357588594711, "nauc_ndcg_at_5_max": 0.610982040152071, "nauc_ndcg_at_5_std": 0.28090467113558243, "nauc_precision_at_1000_diff1": -0.1935591294276986, "nauc_precision_at_1000_max": 0.10619685387536802, "nauc_precision_at_1000_std": 0.2468394907699279, "nauc_precision_at_100_diff1": -0.0900605496254096, "nauc_precision_at_100_max": 0.2429950066850094, "nauc_precision_at_100_std": 0.3289252874196544, "nauc_precision_at_10_diff1": 0.0018350440784743298, "nauc_precision_at_10_max": 0.33957770539706905, "nauc_precision_at_10_std": 0.31891611215728227, "nauc_precision_at_1_diff1": 0.6206797143265911, "nauc_precision_at_1_max": 0.4945201476497312, "nauc_precision_at_1_std": 0.1972028407123734, "nauc_precision_at_20_diff1": -0.03365736454035419, "nauc_precision_at_20_max": 0.30114386228196793, "nauc_precision_at_20_std": 0.32274309305272564, "nauc_precision_at_3_diff1": 0.12755097872349536, "nauc_precision_at_3_max": 0.46353058962173, "nauc_precision_at_3_std": 0.3097186037482983, "nauc_precision_at_5_diff1": 0.04223451425936662, "nauc_precision_at_5_max": 0.3925891263112398, "nauc_precision_at_5_std": 0.30983738011566825, "nauc_recall_at_1000_diff1": 0.7872997351004211, "nauc_recall_at_1000_max": 0.9011890957279892, "nauc_recall_at_1000_std": 0.8485183303325131, "nauc_recall_at_100_diff1": 0.6607507229672478, "nauc_recall_at_100_max": 0.8665929700563204, "nauc_recall_at_100_std": 0.6988902010210113, "nauc_recall_at_10_diff1": 0.6153792478846523, "nauc_recall_at_10_max": 0.7718276200957296, "nauc_recall_at_10_std": 0.45811084641605715, "nauc_recall_at_1_diff1": 0.5512870662759014, "nauc_recall_at_1_max": 0.22338378158416522, "nauc_recall_at_1_std": 0.00987834782975795, "nauc_recall_at_20_diff1": 0.6229462319927663, "nauc_recall_at_20_max": 0.7874474748052596, "nauc_recall_at_20_std": 0.5121623906656506, "nauc_recall_at_3_diff1": 0.5522361870538924, "nauc_recall_at_3_max": 0.5990514608940101, "nauc_recall_at_3_std": 0.2546656850956461, "nauc_recall_at_5_diff1": 0.5715206663883843, "nauc_recall_at_5_max": 0.6907112067834904, "nauc_recall_at_5_std": 0.34368037183961025, "ndcg_at_1": 0.71959, "ndcg_at_10": 0.77042, "ndcg_at_100": 0.78684, "ndcg_at_1000": 0.7957, "ndcg_at_20": 0.77812, "ndcg_at_3": 0.7189, "ndcg_at_5": 0.75232, "precision_at_1": 0.71959, "precision_at_10": 0.15625, "precision_at_100": 0.01701, "precision_at_1000": 0.00183, "precision_at_20": 0.08074, "precision_at_3": 0.42849, "precision_at_5": 0.29122, "recall_at_1": 0.46592, "recall_at_10": 0.84045, "recall_at_100": 0.90191, "recall_at_1000": 0.95898, "recall_at_20": 0.86444, "recall_at_3": 0.72924, "recall_at_5": 0.79788 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 1186423.9800457954, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.55798, "map_at_1": 0.29203, "map_at_10": 0.46203, "map_at_100": 0.48745, "map_at_1000": 0.48846, "map_at_20": 0.47661, "map_at_3": 0.40607, "map_at_5": 0.43463, "mrr_at_1": 0.49765258215962443, "mrr_at_10": 0.6230289142260972, "mrr_at_100": 0.6277851950173721, "mrr_at_1000": 0.6279549678842242, "mrr_at_20": 0.6266624066568468, "mrr_at_3": 0.60093896713615, "mrr_at_5": 0.6126760563380279, "nauc_map_at_1000_diff1": 0.491194883591401, "nauc_map_at_1000_max": 0.4069872070975119, "nauc_map_at_1000_std": 0.1503680269848714, "nauc_map_at_100_diff1": 0.4913454233702117, "nauc_map_at_100_max": 0.40729458264289015, "nauc_map_at_100_std": 0.15028625396742795, "nauc_map_at_10_diff1": 0.48358147289855585, "nauc_map_at_10_max": 0.401153758256364, "nauc_map_at_10_std": 0.13214457746948782, "nauc_map_at_1_diff1": 0.5406803589896482, "nauc_map_at_1_max": 0.37956385631507095, "nauc_map_at_1_std": 0.01951489431069034, "nauc_map_at_20_diff1": 0.4881433473837766, "nauc_map_at_20_max": 0.40653494240357857, "nauc_map_at_20_std": 0.13840323230776408, "nauc_map_at_3_diff1": 0.5007445498318888, "nauc_map_at_3_max": 0.39660095984986155, "nauc_map_at_3_std": 0.10303331423281582, "nauc_map_at_5_diff1": 0.4795533699889755, "nauc_map_at_5_max": 0.39376258695768596, "nauc_map_at_5_std": 0.1272747061807712, "nauc_mrr_at_1000_diff1": 0.4864620495031914, "nauc_mrr_at_1000_max": 0.41944207225633723, "nauc_mrr_at_1000_std": 0.2139266544276021, "nauc_mrr_at_100_diff1": 0.48654660037253383, "nauc_mrr_at_100_max": 0.41972927438876206, "nauc_mrr_at_100_std": 0.2143479051340134, "nauc_mrr_at_10_diff1": 0.48504116206277675, "nauc_mrr_at_10_max": 0.42189655584570024, "nauc_mrr_at_10_std": 0.21635367595331065, "nauc_mrr_at_1_diff1": 0.503902511177942, "nauc_mrr_at_1_max": 0.37842900420373343, "nauc_mrr_at_1_std": 0.13358952143755706, "nauc_mrr_at_20_diff1": 0.4870227638485244, "nauc_mrr_at_20_max": 0.42038872245430065, "nauc_mrr_at_20_std": 0.21523868650702208, "nauc_mrr_at_3_diff1": 0.48157473054556565, "nauc_mrr_at_3_max": 0.4240310174815224, "nauc_mrr_at_3_std": 0.2193030086323231, "nauc_mrr_at_5_diff1": 0.4823847363954533, "nauc_mrr_at_5_max": 0.4204479083239435, "nauc_mrr_at_5_std": 0.22258727059077912, "nauc_ndcg_at_1000_diff1": 0.4997349532235543, "nauc_ndcg_at_1000_max": 0.4298387606599779, "nauc_ndcg_at_1000_std": 0.22086264873797648, "nauc_ndcg_at_100_diff1": 0.5044444671929433, "nauc_ndcg_at_100_max": 0.4380204415938954, "nauc_ndcg_at_100_std": 0.23084687735018677, "nauc_ndcg_at_10_diff1": 0.47565872284869415, "nauc_ndcg_at_10_max": 0.41736335785305456, "nauc_ndcg_at_10_std": 0.1817944693451711, "nauc_ndcg_at_1_diff1": 0.503902511177942, "nauc_ndcg_at_1_max": 0.37842900420373343, "nauc_ndcg_at_1_std": 0.13358952143755706, "nauc_ndcg_at_20_diff1": 0.49225862046154123, "nauc_ndcg_at_20_max": 0.42426048013735657, "nauc_ndcg_at_20_std": 0.194972162210241, "nauc_ndcg_at_3_diff1": 0.47687923026335577, "nauc_ndcg_at_3_max": 0.40531035894440326, "nauc_ndcg_at_3_std": 0.19084986432153023, "nauc_ndcg_at_5_diff1": 0.4657060987971936, "nauc_ndcg_at_5_max": 0.39069416975755794, "nauc_ndcg_at_5_std": 0.1940174168936946, "nauc_precision_at_1000_diff1": -0.11961975615453432, "nauc_precision_at_1000_max": -0.05579563542470387, "nauc_precision_at_1000_std": 0.20820953474163356, "nauc_precision_at_100_diff1": -0.07510979845535082, "nauc_precision_at_100_max": -0.014431689714078318, "nauc_precision_at_100_std": 0.24395066530154658, "nauc_precision_at_10_diff1": 0.0033959763225131613, "nauc_precision_at_10_max": 0.09943027216073655, "nauc_precision_at_10_std": 0.26396146250865166, "nauc_precision_at_1_diff1": 0.503902511177942, "nauc_precision_at_1_max": 0.37842900420373343, "nauc_precision_at_1_std": 0.13358952143755706, "nauc_precision_at_20_diff1": -0.03237023973607798, "nauc_precision_at_20_max": 0.051844987113912645, "nauc_precision_at_20_std": 0.24372696384200618, "nauc_precision_at_3_diff1": 0.15365445388164742, "nauc_precision_at_3_max": 0.25053447360765924, "nauc_precision_at_3_std": 0.29623151747718696, "nauc_precision_at_5_diff1": 0.04820706512058263, "nauc_precision_at_5_max": 0.15270339762587773, "nauc_precision_at_5_std": 0.2983220616107835, "nauc_recall_at_1000_diff1": 0.7973836608766782, "nauc_recall_at_1000_max": 0.5872450504603903, "nauc_recall_at_1000_std": 0.6800368078340461, "nauc_recall_at_100_diff1": 0.5701935430871975, "nauc_recall_at_100_max": 0.5487692759263737, "nauc_recall_at_100_std": 0.521275254554224, "nauc_recall_at_10_diff1": 0.3682811335069578, "nauc_recall_at_10_max": 0.3614076934322836, "nauc_recall_at_10_std": 0.1854444416214071, "nauc_recall_at_1_diff1": 0.5406803589896482, "nauc_recall_at_1_max": 0.37956385631507095, "nauc_recall_at_1_std": 0.01951489431069034, "nauc_recall_at_20_diff1": 0.42827597952233176, "nauc_recall_at_20_max": 0.3692076115465993, "nauc_recall_at_20_std": 0.23099687906425917, "nauc_recall_at_3_diff1": 0.42526025206025636, "nauc_recall_at_3_max": 0.3747420478623814, "nauc_recall_at_3_std": 0.14258382259602792, "nauc_recall_at_5_diff1": 0.37441114533135217, "nauc_recall_at_5_max": 0.3510166504479535, "nauc_recall_at_5_std": 0.19577868823310826, "ndcg_at_1": 0.49765, "ndcg_at_10": 0.55798, "ndcg_at_100": 0.62582, "ndcg_at_1000": 0.63931, "ndcg_at_20": 0.58864, "ndcg_at_3": 0.51786, "ndcg_at_5": 0.52451, "precision_at_1": 0.49765, "precision_at_10": 0.14883, "precision_at_100": 0.02282, "precision_at_1000": 0.00252, "precision_at_20": 0.08873, "precision_at_3": 0.32238, "precision_at_5": 0.23286, "recall_at_1": 0.29203, "recall_at_10": 0.66858, "recall_at_100": 0.89209, "recall_at_1000": 0.97269, "recall_at_20": 0.7564, "recall_at_3": 0.49825, "recall_at_5": 0.56757 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 1455631.1653485298, "kg_co2_emissions": null, "mteb_version": "1.21.6", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.40347, "map_at_1": 0.2312, "map_at_10": 0.34018, "map_at_100": 0.35384, "map_at_1000": 0.35433, "map_at_20": 0.34932, "map_at_3": 0.30681, "map_at_5": 0.3253, "mrr_at_1": 0.2494061757719715, "mrr_at_10": 0.35903272631301125, "mrr_at_100": 0.3706073589603738, "mrr_at_1000": 0.3709959642055801, "mrr_at_20": 0.3667203837232443, "mrr_at_3": 0.32818685669041964, "mrr_at_5": 0.34481393507521774, "nauc_map_at_1000_diff1": 0.35314835099740105, "nauc_map_at_1000_max": 0.3184028824344834, "nauc_map_at_1000_std": 0.1432921331223796, "nauc_map_at_100_diff1": 0.35296058643756734, "nauc_map_at_100_max": 0.31841688633705867, "nauc_map_at_100_std": 0.14349656279043632, "nauc_map_at_10_diff1": 0.35112506194836135, "nauc_map_at_10_max": 0.31491953489216773, "nauc_map_at_10_std": 0.13911813914615137, "nauc_map_at_1_diff1": 0.4093468304996029, "nauc_map_at_1_max": 0.327725215421424, "nauc_map_at_1_std": 0.09909470695069918, "nauc_map_at_20_diff1": 0.35100121453234423, "nauc_map_at_20_max": 0.3181574024852935, "nauc_map_at_20_std": 0.14559399484397795, "nauc_map_at_3_diff1": 0.3756745099127106, "nauc_map_at_3_max": 0.32612578483522403, "nauc_map_at_3_std": 0.13379369457944715, "nauc_map_at_5_diff1": 0.3568498924051157, "nauc_map_at_5_max": 0.31074286391034067, "nauc_map_at_5_std": 0.1293251651765584, "nauc_mrr_at_1000_diff1": 0.364977531152561, "nauc_mrr_at_1000_max": 0.33340877689458087, "nauc_mrr_at_1000_std": 0.1452165152480756, "nauc_mrr_at_100_diff1": 0.3647663369033261, "nauc_mrr_at_100_max": 0.33343770667477374, "nauc_mrr_at_100_std": 0.14535532653439698, "nauc_mrr_at_10_diff1": 0.36456902168528393, "nauc_mrr_at_10_max": 0.33150060580676827, "nauc_mrr_at_10_std": 0.1417761017055953, "nauc_mrr_at_1_diff1": 0.41959061576722323, "nauc_mrr_at_1_max": 0.3433839178533463, "nauc_mrr_at_1_std": 0.0974870864589389, "nauc_mrr_at_20_diff1": 0.36332755358638935, "nauc_mrr_at_20_max": 0.33339240206318743, "nauc_mrr_at_20_std": 0.14724678496394428, "nauc_mrr_at_3_diff1": 0.38550657045567355, "nauc_mrr_at_3_max": 0.3415972835297517, "nauc_mrr_at_3_std": 0.14372554311864696, "nauc_mrr_at_5_diff1": 0.36792261359578443, "nauc_mrr_at_5_max": 0.32610972899994883, "nauc_mrr_at_5_std": 0.13251527197477345, "nauc_ndcg_at_1000_diff1": 0.33995483891551825, "nauc_ndcg_at_1000_max": 0.32613427563168335, "nauc_ndcg_at_1000_std": 0.16820157108978356, "nauc_ndcg_at_100_diff1": 0.333763670806927, "nauc_ndcg_at_100_max": 0.3232036273021733, "nauc_ndcg_at_100_std": 0.17143680193565985, "nauc_ndcg_at_10_diff1": 0.32351652052150365, "nauc_ndcg_at_10_max": 0.3092628191110229, "nauc_ndcg_at_10_std": 0.15760843232563637, "nauc_ndcg_at_1_diff1": 0.41959061576722323, "nauc_ndcg_at_1_max": 0.3433839178533463, "nauc_ndcg_at_1_std": 0.0974870864589389, "nauc_ndcg_at_20_diff1": 0.3184009538690653, "nauc_ndcg_at_20_max": 0.31759749112137114, "nauc_ndcg_at_20_std": 0.18107837405928612, "nauc_ndcg_at_3_diff1": 0.3684020079755906, "nauc_ndcg_at_3_max": 0.3289719410412564, "nauc_ndcg_at_3_std": 0.1477630747515411, "nauc_ndcg_at_5_diff1": 0.33512227447065757, "nauc_ndcg_at_5_max": 0.30012607008251885, "nauc_ndcg_at_5_std": 0.13502330718572497, "nauc_precision_at_1000_diff1": 0.06521978170848006, "nauc_precision_at_1000_max": 0.12105515582448478, "nauc_precision_at_1000_std": 0.03867644111859382, "nauc_precision_at_100_diff1": 0.12353944797790105, "nauc_precision_at_100_max": 0.1913192006044018, "nauc_precision_at_100_std": 0.14174840682942003, "nauc_precision_at_10_diff1": 0.1965669422940171, "nauc_precision_at_10_max": 0.25496564994278625, "nauc_precision_at_10_std": 0.18820923954238694, "nauc_precision_at_1_diff1": 0.41959061576722323, "nauc_precision_at_1_max": 0.3433839178533463, "nauc_precision_at_1_std": 0.0974870864589389, "nauc_precision_at_20_diff1": 0.11676195546241633, "nauc_precision_at_20_max": 0.21904929672037599, "nauc_precision_at_20_std": 0.23147430316311335, "nauc_precision_at_3_diff1": 0.34674459790480033, "nauc_precision_at_3_max": 0.33319229819409746, "nauc_precision_at_3_std": 0.1752854772824614, "nauc_precision_at_5_diff1": 0.24789066663662582, "nauc_precision_at_5_max": 0.2485642215680406, "nauc_precision_at_5_std": 0.14710195968857806, "nauc_recall_at_1000_diff1": 0.2548683272740943, "nauc_recall_at_1000_max": 0.5855865374228014, "nauc_recall_at_1000_std": 0.7160706897978473, "nauc_recall_at_100_diff1": 0.21085262386402295, "nauc_recall_at_100_max": 0.3527713843700616, "nauc_recall_at_100_std": 0.38591614385520506, "nauc_recall_at_10_diff1": 0.21019952954619286, "nauc_recall_at_10_max": 0.2622111605771992, "nauc_recall_at_10_std": 0.20840494045508398, "nauc_recall_at_1_diff1": 0.4093468304996029, "nauc_recall_at_1_max": 0.327725215421424, "nauc_recall_at_1_std": 0.09909470695069918, "nauc_recall_at_20_diff1": 0.1573821569814531, "nauc_recall_at_20_max": 0.288872669243251, "nauc_recall_at_20_std": 0.3308644175315028, "nauc_recall_at_3_diff1": 0.3337416059993435, "nauc_recall_at_3_max": 0.3171886527151158, "nauc_recall_at_3_std": 0.1724507303422492, "nauc_recall_at_5_diff1": 0.2552725017557499, "nauc_recall_at_5_max": 0.24659826971970386, "nauc_recall_at_5_std": 0.13957583477761074, "ndcg_at_1": 0.24941, "ndcg_at_10": 0.40347, "ndcg_at_100": 0.46451, "ndcg_at_1000": 0.47738, "ndcg_at_20": 0.43455, "ndcg_at_3": 0.33662, "ndcg_at_5": 0.36865, "precision_at_1": 0.24941, "precision_at_10": 0.06508, "precision_at_100": 0.00981, "precision_at_1000": 0.0011, "precision_at_20": 0.03979, "precision_at_3": 0.14727, "precision_at_5": 0.10594, "recall_at_1": 0.2312, "recall_at_10": 0.58155, "recall_at_100": 0.85234, "recall_at_1000": 0.95051, "recall_at_20": 0.69873, "recall_at_3": 0.40222, "recall_at_5": 0.47783 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 12127.463791370392, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.24346, "map_at_1": 0.16, "map_at_10": 0.21462, "map_at_100": 0.22418, "map_at_1000": 0.22525, "map_at_20": 0.2195, "map_at_3": 0.19833, "map_at_5": 0.20933, "mrr_at_1": 0.16, "mrr_at_10": 0.21461507936507943, "mrr_at_100": 0.22417548602107892, "mrr_at_1000": 0.22525155738712846, "mrr_at_20": 0.21949745576061366, "mrr_at_3": 0.19833333333333336, "mrr_at_5": 0.20933333333333334, "nauc_map_at_1000_diff1": 0.49929863529099233, "nauc_map_at_1000_max": 0.41971415809226786, "nauc_map_at_1000_std": 0.09531890382009271, "nauc_map_at_100_diff1": 0.4990409957299045, "nauc_map_at_100_max": 0.41987897706351596, "nauc_map_at_100_std": 0.0952393361704494, "nauc_map_at_10_diff1": 0.5062956213852428, "nauc_map_at_10_max": 0.4243765133769566, "nauc_map_at_10_std": 0.08795426579167952, "nauc_map_at_1_diff1": 0.6395663696420342, "nauc_map_at_1_max": 0.4531898141035488, "nauc_map_at_1_std": 0.05106391150714374, "nauc_map_at_20_diff1": 0.496346158552349, "nauc_map_at_20_max": 0.4192185873029332, "nauc_map_at_20_std": 0.09161467444411081, "nauc_map_at_3_diff1": 0.5307151722565736, "nauc_map_at_3_max": 0.4392240081280448, "nauc_map_at_3_std": 0.08356857193516706, "nauc_map_at_5_diff1": 0.5156828411866133, "nauc_map_at_5_max": 0.43736539278879716, "nauc_map_at_5_std": 0.08712073248344535, "nauc_mrr_at_1000_diff1": 0.49929863529099233, "nauc_mrr_at_1000_max": 0.41971415809226786, "nauc_mrr_at_1000_std": 0.09531890382009271, "nauc_mrr_at_100_diff1": 0.4990409957299045, "nauc_mrr_at_100_max": 0.41987897706351596, "nauc_mrr_at_100_std": 0.0952393361704494, "nauc_mrr_at_10_diff1": 0.5062956213852428, "nauc_mrr_at_10_max": 0.4243765133769566, "nauc_mrr_at_10_std": 0.08795426579167952, "nauc_mrr_at_1_diff1": 0.6395663696420342, "nauc_mrr_at_1_max": 0.4531898141035488, "nauc_mrr_at_1_std": 0.05106391150714374, "nauc_mrr_at_20_diff1": 0.496346158552349, "nauc_mrr_at_20_max": 0.4192185873029332, "nauc_mrr_at_20_std": 0.09161467444411081, "nauc_mrr_at_3_diff1": 0.5307151722565736, "nauc_mrr_at_3_max": 0.4392240081280448, "nauc_mrr_at_3_std": 0.08356857193516706, "nauc_mrr_at_5_diff1": 0.5156828411866133, "nauc_mrr_at_5_max": 0.43736539278879716, "nauc_mrr_at_5_std": 0.08712073248344535, "nauc_ndcg_at_1000_diff1": 0.45240660942590677, "nauc_ndcg_at_1000_max": 0.40142745355267073, "nauc_ndcg_at_1000_std": 0.14297847584846893, "nauc_ndcg_at_100_diff1": 0.4422257629311109, "nauc_ndcg_at_100_max": 0.39084482306992857, "nauc_ndcg_at_100_std": 0.1396135977087359, "nauc_ndcg_at_10_diff1": 0.4559678485368314, "nauc_ndcg_at_10_max": 0.4016152381044367, "nauc_ndcg_at_10_std": 0.10065001102931954, "nauc_ndcg_at_1_diff1": 0.6395663696420342, "nauc_ndcg_at_1_max": 0.4531898141035488, "nauc_ndcg_at_1_std": 0.05106391150714374, "nauc_ndcg_at_20_diff1": 0.42455881041527677, "nauc_ndcg_at_20_max": 0.3829538123114625, "nauc_ndcg_at_20_std": 0.11126163095011597, "nauc_ndcg_at_3_diff1": 0.5032758392136251, "nauc_ndcg_at_3_max": 0.4361327441336674, "nauc_ndcg_at_3_std": 0.0920914189387461, "nauc_ndcg_at_5_diff1": 0.4768782619701144, "nauc_ndcg_at_5_max": 0.4318323517533922, "nauc_ndcg_at_5_std": 0.09792981024765775, "nauc_precision_at_1000_diff1": 0.29945358643125614, "nauc_precision_at_1000_max": 0.4339723950601696, "nauc_precision_at_1000_std": 0.5704020719497185, "nauc_precision_at_100_diff1": 0.29562207969353155, "nauc_precision_at_100_max": 0.30654874191020737, "nauc_precision_at_100_std": 0.31555271788025674, "nauc_precision_at_10_diff1": 0.32826434865760473, "nauc_precision_at_10_max": 0.33699008559722193, "nauc_precision_at_10_std": 0.1325528425050969, "nauc_precision_at_1_diff1": 0.6395663696420342, "nauc_precision_at_1_max": 0.4531898141035488, "nauc_precision_at_1_std": 0.05106391150714374, "nauc_precision_at_20_diff1": 0.23176493305870235, "nauc_precision_at_20_max": 0.27511585993820825, "nauc_precision_at_20_std": 0.16388388259526274, "nauc_precision_at_3_diff1": 0.43420437425320807, "nauc_precision_at_3_max": 0.4284823701416771, "nauc_precision_at_3_std": 0.11364597789866632, "nauc_precision_at_5_diff1": 0.3807973408349397, "nauc_precision_at_5_max": 0.41757201337756206, "nauc_precision_at_5_std": 0.12434866122209884, "nauc_recall_at_1000_diff1": 0.29945358643125675, "nauc_recall_at_1000_max": 0.4339723950601693, "nauc_recall_at_1000_std": 0.5704020719497167, "nauc_recall_at_100_diff1": 0.29562207969353077, "nauc_recall_at_100_max": 0.3065487419102072, "nauc_recall_at_100_std": 0.3155527178802567, "nauc_recall_at_10_diff1": 0.32826434865760423, "nauc_recall_at_10_max": 0.33699008559722154, "nauc_recall_at_10_std": 0.1325528425050965, "nauc_recall_at_1_diff1": 0.6395663696420342, "nauc_recall_at_1_max": 0.4531898141035488, "nauc_recall_at_1_std": 0.05106391150714374, "nauc_recall_at_20_diff1": 0.23176493305870227, "nauc_recall_at_20_max": 0.27511585993820803, "nauc_recall_at_20_std": 0.16388388259526257, "nauc_recall_at_3_diff1": 0.43420437425320807, "nauc_recall_at_3_max": 0.42848237014167706, "nauc_recall_at_3_std": 0.11364597789866643, "nauc_recall_at_5_diff1": 0.38079734083493955, "nauc_recall_at_5_max": 0.4175720133775618, "nauc_recall_at_5_std": 0.12434866122209884, "ndcg_at_1": 0.16, "ndcg_at_10": 0.24346, "ndcg_at_100": 0.29392, "ndcg_at_1000": 0.3264, "ndcg_at_20": 0.26034, "ndcg_at_3": 0.21036, "ndcg_at_5": 0.23057, "precision_at_1": 0.16, "precision_at_10": 0.0335, "precision_at_100": 0.0058, "precision_at_1000": 0.00085, "precision_at_20": 0.02, "precision_at_3": 0.08167, "precision_at_5": 0.059, "recall_at_1": 0.16, "recall_at_10": 0.335, "recall_at_100": 0.58, "recall_at_1000": 0.845, "recall_at_20": 0.4, "recall_at_3": 0.245, "recall_at_5": 0.295 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.26801, "map_at_1": 0.2, "map_at_10": 0.24508, "map_at_100": 0.24995, "map_at_1000": 0.25111, "map_at_20": 0.24643, "map_at_3": 0.23417, "map_at_5": 0.23917, "mrr_at_1": 0.2, "mrr_at_10": 0.24508333333333332, "mrr_at_100": 0.2499496802510732, "mrr_at_1000": 0.25110701532543067, "mrr_at_20": 0.24642822199730097, "mrr_at_3": 0.2341666666666667, "mrr_at_5": 0.23916666666666667, "nauc_map_at_1000_diff1": 0.6395355878729584, "nauc_map_at_1000_max": 0.365187460643138, "nauc_map_at_1000_std": 0.01970574265860572, "nauc_map_at_100_diff1": 0.6391681401299727, "nauc_map_at_100_max": 0.3650387442985491, "nauc_map_at_100_std": 0.01867822705611146, "nauc_map_at_10_diff1": 0.6434385016870486, "nauc_map_at_10_max": 0.368285928478509, "nauc_map_at_10_std": 0.020151886079335497, "nauc_map_at_1_diff1": 0.71360187278011, "nauc_map_at_1_max": 0.37068937681627384, "nauc_map_at_1_std": -0.027615434291249602, "nauc_map_at_20_diff1": 0.6418435602302738, "nauc_map_at_20_max": 0.3665716207617676, "nauc_map_at_20_std": 0.018595393472855243, "nauc_map_at_3_diff1": 0.6594812851695392, "nauc_map_at_3_max": 0.36501267080996064, "nauc_map_at_3_std": 0.0028860938964413215, "nauc_map_at_5_diff1": 0.6504570573581048, "nauc_map_at_5_max": 0.3660302571596071, "nauc_map_at_5_std": 0.013442348773576428, "nauc_mrr_at_1000_diff1": 0.6395355878729584, "nauc_mrr_at_1000_max": 0.365187460643138, "nauc_mrr_at_1000_std": 0.01970574265860572, "nauc_mrr_at_100_diff1": 0.6391681401299727, "nauc_mrr_at_100_max": 0.3650387442985491, "nauc_mrr_at_100_std": 0.01867822705611146, "nauc_mrr_at_10_diff1": 0.6434385016870486, "nauc_mrr_at_10_max": 0.368285928478509, "nauc_mrr_at_10_std": 0.020151886079335497, "nauc_mrr_at_1_diff1": 0.71360187278011, "nauc_mrr_at_1_max": 0.37068937681627384, "nauc_mrr_at_1_std": -0.027615434291249602, "nauc_mrr_at_20_diff1": 0.6418435602302738, "nauc_mrr_at_20_max": 0.3665716207617676, "nauc_mrr_at_20_std": 0.018595393472855243, "nauc_mrr_at_3_diff1": 0.6594812851695392, "nauc_mrr_at_3_max": 0.36501267080996064, "nauc_mrr_at_3_std": 0.0028860938964413215, "nauc_mrr_at_5_diff1": 0.6504570573581048, "nauc_mrr_at_5_max": 0.3660302571596071, "nauc_mrr_at_5_std": 0.013442348773576428, "nauc_ndcg_at_1000_diff1": 0.596417643300829, "nauc_ndcg_at_1000_max": 0.3518020144512341, "nauc_ndcg_at_1000_std": 0.06391223751601617, "nauc_ndcg_at_100_diff1": 0.5881156440243976, "nauc_ndcg_at_100_max": 0.355639229989986, "nauc_ndcg_at_100_std": 0.04401917229409399, "nauc_ndcg_at_10_diff1": 0.6092903909063339, "nauc_ndcg_at_10_max": 0.36768272278214337, "nauc_ndcg_at_10_std": 0.043181167699574335, "nauc_ndcg_at_1_diff1": 0.71360187278011, "nauc_ndcg_at_1_max": 0.37068937681627384, "nauc_ndcg_at_1_std": -0.027615434291249602, "nauc_ndcg_at_20_diff1": 0.6044725907955486, "nauc_ndcg_at_20_max": 0.3619261402136727, "nauc_ndcg_at_20_std": 0.03685933589523102, "nauc_ndcg_at_3_diff1": 0.6389221482842374, "nauc_ndcg_at_3_max": 0.3596097338870786, "nauc_ndcg_at_3_std": 0.010131471633429949, "nauc_ndcg_at_5_diff1": 0.6241080600798613, "nauc_ndcg_at_5_max": 0.36139735015946556, "nauc_ndcg_at_5_std": 0.02771549327543668, "nauc_precision_at_1000_diff1": 0.40370096225018537, "nauc_precision_at_1000_max": 0.23549507487331303, "nauc_precision_at_1000_std": 0.3946820019358882, "nauc_precision_at_100_diff1": 0.42064133112556024, "nauc_precision_at_100_max": 0.32471090289837545, "nauc_precision_at_100_std": 0.132641384973682, "nauc_precision_at_10_diff1": 0.5151898566469701, "nauc_precision_at_10_max": 0.3685179787487747, "nauc_precision_at_10_std": 0.10883966188823242, "nauc_precision_at_1_diff1": 0.71360187278011, "nauc_precision_at_1_max": 0.37068937681627384, "nauc_precision_at_1_std": -0.027615434291249602, "nauc_precision_at_20_diff1": 0.5002558652180761, "nauc_precision_at_20_max": 0.348403138613342, "nauc_precision_at_20_std": 0.08404188316800501, "nauc_precision_at_3_diff1": 0.5827769707804907, "nauc_precision_at_3_max": 0.3441183865020392, "nauc_precision_at_3_std": 0.028933180624616014, "nauc_precision_at_5_diff1": 0.5527631401162005, "nauc_precision_at_5_max": 0.3484664234562899, "nauc_precision_at_5_std": 0.06611944331847057, "nauc_recall_at_1000_diff1": 0.403700962250185, "nauc_recall_at_1000_max": 0.23549507487331284, "nauc_recall_at_1000_std": 0.3946820019358879, "nauc_recall_at_100_diff1": 0.42064133112556035, "nauc_recall_at_100_max": 0.3247109028983749, "nauc_recall_at_100_std": 0.1326413849736817, "nauc_recall_at_10_diff1": 0.5151898566469698, "nauc_recall_at_10_max": 0.36851797874877423, "nauc_recall_at_10_std": 0.10883966188823224, "nauc_recall_at_1_diff1": 0.71360187278011, "nauc_recall_at_1_max": 0.37068937681627384, "nauc_recall_at_1_std": -0.027615434291249602, "nauc_recall_at_20_diff1": 0.5002558652180759, "nauc_recall_at_20_max": 0.3484031386133417, "nauc_recall_at_20_std": 0.08404188316800514, "nauc_recall_at_3_diff1": 0.5827769707804905, "nauc_recall_at_3_max": 0.3441183865020391, "nauc_recall_at_3_std": 0.028933180624615876, "nauc_recall_at_5_diff1": 0.5527631401162005, "nauc_recall_at_5_max": 0.34846642345628953, "nauc_recall_at_5_std": 0.06611944331847029, "ndcg_at_1": 0.2, "ndcg_at_10": 0.26801, "ndcg_at_100": 0.30033, "ndcg_at_1000": 0.3334, "ndcg_at_20": 0.27302, "ndcg_at_3": 0.24589, "ndcg_at_5": 0.25451, "precision_at_1": 0.2, "precision_at_10": 0.034, "precision_at_100": 0.0051, "precision_at_1000": 0.00078, "precision_at_20": 0.018, "precision_at_3": 0.09333, "precision_at_5": 0.06, "recall_at_1": 0.2, "recall_at_10": 0.34, "recall_at_100": 0.51, "recall_at_1000": 0.775, "recall_at_20": 0.36, "recall_at_3": 0.28, "recall_at_5": 0.3 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 88.03694748878479, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.8605, "map_at_1": 0.72727, "map_at_10": 0.82287, "map_at_100": 0.8246, "map_at_1000": 0.8246, "map_at_20": 0.8246, "map_at_3": 0.80952, "map_at_5": 0.81926, "mrr_at_1": 0.7272727272727273, "mrr_at_10": 0.8228715728715729, "mrr_at_100": 0.8246031746031747, "mrr_at_1000": 0.8246031746031747, "mrr_at_20": 0.8246031746031747, "mrr_at_3": 0.8095238095238095, "mrr_at_5": 0.8192640692640693, "nauc_map_at_1000_diff1": 0.5941123198806364, "nauc_map_at_1000_max": 0.35774172874077775, "nauc_map_at_1000_std": -0.03559328712720151, "nauc_map_at_100_diff1": 0.5941123198806364, "nauc_map_at_100_max": 0.35774172874077775, "nauc_map_at_100_std": -0.03559328712720151, "nauc_map_at_10_diff1": 0.5963003758717501, "nauc_map_at_10_max": 0.36400168489193485, "nauc_map_at_10_std": -0.032820874591175796, "nauc_map_at_1_diff1": 0.6291397163278336, "nauc_map_at_1_max": 0.3512047101861982, "nauc_map_at_1_std": -0.02760603176568628, "nauc_map_at_20_diff1": 0.5941123198806364, "nauc_map_at_20_max": 0.35774172874077775, "nauc_map_at_20_std": -0.03559328712720151, "nauc_map_at_3_diff1": 0.5683248864352117, "nauc_map_at_3_max": 0.3613617400875627, "nauc_map_at_3_std": -0.035344875398458854, "nauc_map_at_5_diff1": 0.5753457266413127, "nauc_map_at_5_max": 0.35922690099588295, "nauc_map_at_5_std": -0.030867869425378183, "nauc_mrr_at_1000_diff1": 0.5941123198806364, "nauc_mrr_at_1000_max": 0.35774172874077775, "nauc_mrr_at_1000_std": -0.03559328712720151, "nauc_mrr_at_100_diff1": 0.5941123198806364, "nauc_mrr_at_100_max": 0.35774172874077775, "nauc_mrr_at_100_std": -0.03559328712720151, "nauc_mrr_at_10_diff1": 0.5963003758717501, "nauc_mrr_at_10_max": 0.36400168489193485, "nauc_mrr_at_10_std": -0.032820874591175796, "nauc_mrr_at_1_diff1": 0.6291397163278336, "nauc_mrr_at_1_max": 0.3512047101861982, "nauc_mrr_at_1_std": -0.02760603176568628, "nauc_mrr_at_20_diff1": 0.5941123198806364, "nauc_mrr_at_20_max": 0.35774172874077775, "nauc_mrr_at_20_std": -0.03559328712720151, "nauc_mrr_at_3_diff1": 0.5683248864352117, "nauc_mrr_at_3_max": 0.3613617400875627, "nauc_mrr_at_3_std": -0.035344875398458854, "nauc_mrr_at_5_diff1": 0.5753457266413127, "nauc_mrr_at_5_max": 0.35922690099588295, "nauc_mrr_at_5_std": -0.030867869425378183, "nauc_ndcg_at_1000_diff1": 0.5923204922760418, "nauc_ndcg_at_1000_max": 0.3617737751423526, "nauc_ndcg_at_1000_std": -0.035546633374422584, "nauc_ndcg_at_100_diff1": 0.5923204922760418, "nauc_ndcg_at_100_max": 0.3617737751423526, "nauc_ndcg_at_100_std": -0.035546633374422584, "nauc_ndcg_at_10_diff1": 0.6027625914595975, "nauc_ndcg_at_10_max": 0.3911779598535934, "nauc_ndcg_at_10_std": -0.02243478615895175, "nauc_ndcg_at_1_diff1": 0.6291397163278336, "nauc_ndcg_at_1_max": 0.3512047101861982, "nauc_ndcg_at_1_std": -0.02760603176568628, "nauc_ndcg_at_20_diff1": 0.5923204922760418, "nauc_ndcg_at_20_max": 0.3617737751423526, "nauc_ndcg_at_20_std": -0.035546633374422584, "nauc_ndcg_at_3_diff1": 0.5389406858000926, "nauc_ndcg_at_3_max": 0.3764518408506499, "nauc_ndcg_at_3_std": -0.034403886138833065, "nauc_ndcg_at_5_diff1": 0.5499029401176652, "nauc_ndcg_at_5_max": 0.3738836785483251, "nauc_ndcg_at_5_std": -0.024967315629729113, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.7956576740092968, "nauc_precision_at_10_max": 0.9343563512361485, "nauc_precision_at_10_std": 0.21977808756952869, "nauc_precision_at_1_diff1": 0.6291397163278336, "nauc_precision_at_1_max": 0.3512047101861982, "nauc_precision_at_1_std": -0.02760603176568628, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_3_diff1": 0.37827560700261104, "nauc_precision_at_3_max": 0.46448289340790067, "nauc_precision_at_3_std": -0.02672054880618261, "nauc_precision_at_5_diff1": 0.3239156642506275, "nauc_precision_at_5_max": 0.5150116195409513, "nauc_precision_at_5_std": 0.03597587103073941, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.7956576740092982, "nauc_recall_at_10_max": 0.9343563512361502, "nauc_recall_at_10_std": 0.21977808756952488, "nauc_recall_at_1_diff1": 0.6291397163278336, "nauc_recall_at_1_max": 0.3512047101861982, "nauc_recall_at_1_std": -0.02760603176568628, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_3_diff1": 0.37827560700261204, "nauc_recall_at_3_max": 0.4644828934079003, "nauc_recall_at_3_std": -0.026720548806184068, "nauc_recall_at_5_diff1": 0.3239156642506218, "nauc_recall_at_5_max": 0.5150116195409485, "nauc_recall_at_5_std": 0.035975871030735844, "ndcg_at_1": 0.72727, "ndcg_at_10": 0.8605, "ndcg_at_100": 0.86699, "ndcg_at_1000": 0.86699, "ndcg_at_20": 0.86699, "ndcg_at_3": 0.83519, "ndcg_at_5": 0.85197, "precision_at_1": 0.72727, "precision_at_10": 0.0974, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.05, "precision_at_3": 0.30303, "precision_at_5": 0.18961, "recall_at_1": 0.72727, "recall_at_10": 0.97403, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 1.0, "recall_at_3": 0.90909, "recall_at_5": 0.94805 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 1152.1036405563354, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.37371, "map_at_1": 0.19952, "map_at_10": 0.32266, "map_at_100": 0.33694, "map_at_1000": 0.33813, "map_at_20": 0.32958, "map_at_3": 0.28349, "map_at_5": 0.30785, "mrr_at_1": 0.2874617737003058, "mrr_at_10": 0.3625163826998689, "mrr_at_100": 0.37359128430577954, "mrr_at_1000": 0.374369843992097, "mrr_at_20": 0.3680337539140476, "mrr_at_3": 0.33970438328236496, "mrr_at_5": 0.35201325178389403, "nauc_map_at_1000_diff1": 0.46444161025916086, "nauc_map_at_1000_max": 0.11479500611039435, "nauc_map_at_1000_std": -0.3216946254853604, "nauc_map_at_100_diff1": 0.463898231197359, "nauc_map_at_100_max": 0.11521867987574255, "nauc_map_at_100_std": -0.3209755901041567, "nauc_map_at_10_diff1": 0.465059961833811, "nauc_map_at_10_max": 0.10964692960708387, "nauc_map_at_10_std": -0.328964243519146, "nauc_map_at_1_diff1": 0.5503972615133271, "nauc_map_at_1_max": 0.12588443767003374, "nauc_map_at_1_std": -0.2873373787737232, "nauc_map_at_20_diff1": 0.4642576023527398, "nauc_map_at_20_max": 0.11061400990262676, "nauc_map_at_20_std": -0.32767134556537214, "nauc_map_at_3_diff1": 0.4709000192756455, "nauc_map_at_3_max": 0.10338449035860794, "nauc_map_at_3_std": -0.3169434722464316, "nauc_map_at_5_diff1": 0.46616412893115433, "nauc_map_at_5_max": 0.1070088513295947, "nauc_map_at_5_std": -0.3277349614494219, "nauc_mrr_at_1000_diff1": 0.49517760407182454, "nauc_mrr_at_1000_max": 0.1153835361655304, "nauc_mrr_at_1000_std": -0.3329689521993201, "nauc_mrr_at_100_diff1": 0.49500566200526813, "nauc_mrr_at_100_max": 0.11591775127846696, "nauc_mrr_at_100_std": -0.332225635828467, "nauc_mrr_at_10_diff1": 0.49450832626240365, "nauc_mrr_at_10_max": 0.11125497198889121, "nauc_mrr_at_10_std": -0.33913630107640047, "nauc_mrr_at_1_diff1": 0.5295829906140828, "nauc_mrr_at_1_max": 0.13620875839895372, "nauc_mrr_at_1_std": -0.33037302783647077, "nauc_mrr_at_20_diff1": 0.4945475348958433, "nauc_mrr_at_20_max": 0.11251481135013906, "nauc_mrr_at_20_std": -0.33668887785018126, "nauc_mrr_at_3_diff1": 0.4938308341804614, "nauc_mrr_at_3_max": 0.10149320444858013, "nauc_mrr_at_3_std": -0.3434660134069083, "nauc_mrr_at_5_diff1": 0.4959445813904095, "nauc_mrr_at_5_max": 0.1084496199527392, "nauc_mrr_at_5_std": -0.34103297689719125, "nauc_ndcg_at_1000_diff1": 0.46716230134313386, "nauc_ndcg_at_1000_max": 0.12359694790565268, "nauc_ndcg_at_1000_std": -0.31023924119027163, "nauc_ndcg_at_100_diff1": 0.4580749898944042, "nauc_ndcg_at_100_max": 0.13924706891695451, "nauc_ndcg_at_100_std": -0.28398539584435967, "nauc_ndcg_at_10_diff1": 0.45886164732099666, "nauc_ndcg_at_10_max": 0.10344883823068486, "nauc_ndcg_at_10_std": -0.3368211415891488, "nauc_ndcg_at_1_diff1": 0.5295829906140828, "nauc_ndcg_at_1_max": 0.13679889247116944, "nauc_ndcg_at_1_std": -0.3311319557153684, "nauc_ndcg_at_20_diff1": 0.4575048476371181, "nauc_ndcg_at_20_max": 0.10615074399619397, "nauc_ndcg_at_20_std": -0.3309719655646826, "nauc_ndcg_at_3_diff1": 0.46054804670971383, "nauc_ndcg_at_3_max": 0.08595210158045091, "nauc_ndcg_at_3_std": -0.33183538771567966, "nauc_ndcg_at_5_diff1": 0.4606539526586548, "nauc_ndcg_at_5_max": 0.09754113254472742, "nauc_ndcg_at_5_std": -0.3363424955563504, "nauc_precision_at_1000_diff1": 0.03322933106078146, "nauc_precision_at_1000_max": 0.09535375289890795, "nauc_precision_at_1000_std": -0.009488197248939465, "nauc_precision_at_100_diff1": 0.11137500357644262, "nauc_precision_at_100_max": 0.17571371775911468, "nauc_precision_at_100_std": 0.003886182907374527, "nauc_precision_at_10_diff1": 0.25578011735014483, "nauc_precision_at_10_max": 0.07875443323051794, "nauc_precision_at_10_std": -0.2900464680421701, "nauc_precision_at_1_diff1": 0.5295829906140828, "nauc_precision_at_1_max": 0.13679889247116944, "nauc_precision_at_1_std": -0.3311319557153684, "nauc_precision_at_20_diff1": 0.21803247757837754, "nauc_precision_at_20_max": 0.08011191417088347, "nauc_precision_at_20_std": -0.24641000603213958, "nauc_precision_at_3_diff1": 0.3125591042971302, "nauc_precision_at_3_max": 0.07252243412420498, "nauc_precision_at_3_std": -0.3164760884209657, "nauc_precision_at_5_diff1": 0.2807649152793834, "nauc_precision_at_5_max": 0.07724142885277911, "nauc_precision_at_5_std": -0.30443002296468685, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.3572248383150459, "nauc_recall_at_100_max": 0.31061429813565594, "nauc_recall_at_100_std": 0.036335209156296955, "nauc_recall_at_10_diff1": 0.3964961266050805, "nauc_recall_at_10_max": 0.08171629769667459, "nauc_recall_at_10_std": -0.3314459840071667, "nauc_recall_at_1_diff1": 0.5503972615133271, "nauc_recall_at_1_max": 0.12588443767003374, "nauc_recall_at_1_std": -0.2873373787737232, "nauc_recall_at_20_diff1": 0.38809851857446975, "nauc_recall_at_20_max": 0.08606656648272232, "nauc_recall_at_20_std": -0.31075764840972503, "nauc_recall_at_3_diff1": 0.40850208475343025, "nauc_recall_at_3_max": 0.04933418986196694, "nauc_recall_at_3_std": -0.3198688670871101, "nauc_recall_at_5_diff1": 0.4034460899480087, "nauc_recall_at_5_max": 0.0674036916753514, "nauc_recall_at_5_std": -0.3329769217802357, "ndcg_at_1": 0.28746, "ndcg_at_10": 0.37371, "ndcg_at_100": 0.44235, "ndcg_at_1000": 0.47117, "ndcg_at_20": 0.39627, "ndcg_at_3": 0.31517, "ndcg_at_5": 0.34389, "precision_at_1": 0.28746, "precision_at_10": 0.07783, "precision_at_100": 0.01263, "precision_at_1000": 0.00156, "precision_at_20": 0.04518, "precision_at_3": 0.17737, "precision_at_5": 0.13089, "recall_at_1": 0.19952, "recall_at_10": 0.48591, "recall_at_100": 0.78682, "recall_at_1000": 1.0, "recall_at_20": 0.56636, "recall_at_3": 0.33629, "recall_at_5": 0.4039 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.33915, "map_at_1": 0.16577, "map_at_10": 0.28601, "map_at_100": 0.30486, "map_at_1000": 0.30666, "map_at_20": 0.29596, "map_at_3": 0.2399, "map_at_5": 0.26884, "mrr_at_1": 0.28287461773700306, "mrr_at_10": 0.3462168098635987, "mrr_at_100": 0.3574002013497723, "mrr_at_1000": 0.3582673706739296, "mrr_at_20": 0.3520306843415528, "mrr_at_3": 0.3182976554536189, "mrr_at_5": 0.3339704383282365, "nauc_map_at_1000_diff1": 0.44862829866771636, "nauc_map_at_1000_max": 0.1868603331777558, "nauc_map_at_1000_std": -0.23781028742308002, "nauc_map_at_100_diff1": 0.44856887266249545, "nauc_map_at_100_max": 0.18565202232669537, "nauc_map_at_100_std": -0.23914927193885566, "nauc_map_at_10_diff1": 0.4483312252840105, "nauc_map_at_10_max": 0.18399006610554597, "nauc_map_at_10_std": -0.2452129696015986, "nauc_map_at_1_diff1": 0.46695496227991584, "nauc_map_at_1_max": 0.16305370574266562, "nauc_map_at_1_std": -0.19783370337577047, "nauc_map_at_20_diff1": 0.4504350704238741, "nauc_map_at_20_max": 0.18555170887364925, "nauc_map_at_20_std": -0.24063964918946817, "nauc_map_at_3_diff1": 0.4637167195369921, "nauc_map_at_3_max": 0.1896462231219421, "nauc_map_at_3_std": -0.22896228336804447, "nauc_map_at_5_diff1": 0.4492227924871043, "nauc_map_at_5_max": 0.19086660193436616, "nauc_map_at_5_std": -0.23324770099359846, "nauc_mrr_at_1000_diff1": 0.4459226568485144, "nauc_mrr_at_1000_max": 0.1765025673789009, "nauc_mrr_at_1000_std": -0.23262559415731116, "nauc_mrr_at_100_diff1": 0.4456985604313047, "nauc_mrr_at_100_max": 0.17637479623488186, "nauc_mrr_at_100_std": -0.23251985996765268, "nauc_mrr_at_10_diff1": 0.44818247963403246, "nauc_mrr_at_10_max": 0.17653252362901384, "nauc_mrr_at_10_std": -0.23664740354352845, "nauc_mrr_at_1_diff1": 0.4636896954638108, "nauc_mrr_at_1_max": 0.1890309030431272, "nauc_mrr_at_1_std": -0.2346242988878434, "nauc_mrr_at_20_diff1": 0.4469219811962643, "nauc_mrr_at_20_max": 0.17637699958853023, "nauc_mrr_at_20_std": -0.234594750073801, "nauc_mrr_at_3_diff1": 0.4528198397450242, "nauc_mrr_at_3_max": 0.17955788910333872, "nauc_mrr_at_3_std": -0.2376516051597019, "nauc_mrr_at_5_diff1": 0.4469627642330273, "nauc_mrr_at_5_max": 0.17643403796964313, "nauc_mrr_at_5_std": -0.23283768470458424, "nauc_ndcg_at_1000_diff1": 0.43900411557115737, "nauc_ndcg_at_1000_max": 0.18754236588293277, "nauc_ndcg_at_1000_std": -0.22172936526380338, "nauc_ndcg_at_100_diff1": 0.43210131283241693, "nauc_ndcg_at_100_max": 0.18109760715434928, "nauc_ndcg_at_100_std": -0.2222934180499701, "nauc_ndcg_at_10_diff1": 0.4390335763848873, "nauc_ndcg_at_10_max": 0.17364612679841726, "nauc_ndcg_at_10_std": -0.251637150473817, "nauc_ndcg_at_1_diff1": 0.4636896954638108, "nauc_ndcg_at_1_max": 0.1890309030431272, "nauc_ndcg_at_1_std": -0.2346242988878434, "nauc_ndcg_at_20_diff1": 0.4422382375961862, "nauc_ndcg_at_20_max": 0.1755348159868798, "nauc_ndcg_at_20_std": -0.24038299245542574, "nauc_ndcg_at_3_diff1": 0.4514476241956719, "nauc_ndcg_at_3_max": 0.18778870526053468, "nauc_ndcg_at_3_std": -0.23870201989358064, "nauc_ndcg_at_5_diff1": 0.4389203557523558, "nauc_ndcg_at_5_max": 0.18209283565176848, "nauc_ndcg_at_5_std": -0.23223058959521303, "nauc_precision_at_1000_diff1": 0.04417038855005799, "nauc_precision_at_1000_max": 0.10874882232618706, "nauc_precision_at_1000_std": 0.02128219191979829, "nauc_precision_at_100_diff1": 0.13318298530387523, "nauc_precision_at_100_max": 0.12181532190768891, "nauc_precision_at_100_std": -0.05883785462110636, "nauc_precision_at_10_diff1": 0.2900814849077367, "nauc_precision_at_10_max": 0.14374772391820556, "nauc_precision_at_10_std": -0.2173607505039916, "nauc_precision_at_1_diff1": 0.4636896954638108, "nauc_precision_at_1_max": 0.1890309030431272, "nauc_precision_at_1_std": -0.2346242988878434, "nauc_precision_at_20_diff1": 0.25999307679332195, "nauc_precision_at_20_max": 0.1320926866538145, "nauc_precision_at_20_std": -0.16959662283088012, "nauc_precision_at_3_diff1": 0.360015785987485, "nauc_precision_at_3_max": 0.18115538783997318, "nauc_precision_at_3_std": -0.23114999628155805, "nauc_precision_at_5_diff1": 0.30225219967341, "nauc_precision_at_5_max": 0.17107969445680038, "nauc_precision_at_5_std": -0.20493189135850728, "nauc_recall_at_1000_diff1": 0.39868921582836264, "nauc_recall_at_1000_max": 0.24677421562564303, "nauc_recall_at_1000_std": 0.17281627161748658, "nauc_recall_at_100_diff1": 0.3148361772259049, "nauc_recall_at_100_max": 0.15649635658167008, "nauc_recall_at_100_std": -0.11398241040093268, "nauc_recall_at_10_diff1": 0.38493841481355845, "nauc_recall_at_10_max": 0.13860452389296737, "nauc_recall_at_10_std": -0.26599323894928106, "nauc_recall_at_1_diff1": 0.46695496227991584, "nauc_recall_at_1_max": 0.16305370574266562, "nauc_recall_at_1_std": -0.19783370337577047, "nauc_recall_at_20_diff1": 0.39277190728666644, "nauc_recall_at_20_max": 0.14008993824278584, "nauc_recall_at_20_std": -0.23145921727069257, "nauc_recall_at_3_diff1": 0.42519191661800404, "nauc_recall_at_3_max": 0.16819962699311578, "nauc_recall_at_3_std": -0.2195755842172038, "nauc_recall_at_5_diff1": 0.39154306459667243, "nauc_recall_at_5_max": 0.15958991890486823, "nauc_recall_at_5_std": -0.21857793250510632, "ndcg_at_1": 0.28287, "ndcg_at_10": 0.33915, "ndcg_at_100": 0.42095, "ndcg_at_1000": 0.45354, "ndcg_at_20": 0.36748, "ndcg_at_3": 0.28262, "ndcg_at_5": 0.30601, "precision_at_1": 0.28287, "precision_at_10": 0.08869, "precision_at_100": 0.0158, "precision_at_1000": 0.00194, "precision_at_20": 0.05466, "precision_at_3": 0.18196, "precision_at_5": 0.1422, "recall_at_1": 0.16577, "recall_at_10": 0.43175, "recall_at_100": 0.77097, "recall_at_1000": 0.99401, "recall_at_20": 0.52212, "recall_at_3": 0.27273, "recall_at_5": 0.34722 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.28069, "map_at_1": 0.14384, "map_at_10": 0.23206, "map_at_100": 0.24851, "map_at_1000": 0.25019, "map_at_20": 0.23928, "map_at_3": 0.20174, "map_at_5": 0.2198, "mrr_at_1": 0.21009771986970685, "mrr_at_10": 0.27868323768160896, "mrr_at_100": 0.2914729717303233, "mrr_at_1000": 0.2923742227125538, "mrr_at_20": 0.28498404529839516, "mrr_at_3": 0.25352877307274707, "mrr_at_5": 0.2673724212812161, "nauc_map_at_1000_diff1": 0.39411401350349234, "nauc_map_at_1000_max": 0.08315202055240034, "nauc_map_at_1000_std": -0.22334318940963396, "nauc_map_at_100_diff1": 0.393133941537155, "nauc_map_at_100_max": 0.08372750234188758, "nauc_map_at_100_std": -0.22244595787765817, "nauc_map_at_10_diff1": 0.3976104145328913, "nauc_map_at_10_max": 0.07667757593928304, "nauc_map_at_10_std": -0.23342235584850604, "nauc_map_at_1_diff1": 0.47888230868159154, "nauc_map_at_1_max": 0.08149092947082825, "nauc_map_at_1_std": -0.22164586926600838, "nauc_map_at_20_diff1": 0.39513065855474855, "nauc_map_at_20_max": 0.07578819036347148, "nauc_map_at_20_std": -0.2318293043634674, "nauc_map_at_3_diff1": 0.42534192375380364, "nauc_map_at_3_max": 0.08278700283177769, "nauc_map_at_3_std": -0.23392898253230987, "nauc_map_at_5_diff1": 0.40496545896359537, "nauc_map_at_5_max": 0.07263646886510684, "nauc_map_at_5_std": -0.23755355346647417, "nauc_mrr_at_1000_diff1": 0.4273326414300659, "nauc_mrr_at_1000_max": 0.08947214016925248, "nauc_mrr_at_1000_std": -0.21867801746554558, "nauc_mrr_at_100_diff1": 0.42672783101627787, "nauc_mrr_at_100_max": 0.0901414497970279, "nauc_mrr_at_100_std": -0.2175503484566645, "nauc_mrr_at_10_diff1": 0.42930279656854725, "nauc_mrr_at_10_max": 0.08580892409809703, "nauc_mrr_at_10_std": -0.22414605867606796, "nauc_mrr_at_1_diff1": 0.49296925922109, "nauc_mrr_at_1_max": 0.11114904317803229, "nauc_mrr_at_1_std": -0.21667452405028526, "nauc_mrr_at_20_diff1": 0.42702370102541154, "nauc_mrr_at_20_max": 0.08583226911350729, "nauc_mrr_at_20_std": -0.22200466434556937, "nauc_mrr_at_3_diff1": 0.44693342638609657, "nauc_mrr_at_3_max": 0.0852618594155734, "nauc_mrr_at_3_std": -0.23280930677054384, "nauc_mrr_at_5_diff1": 0.43273716355120434, "nauc_mrr_at_5_max": 0.07705694781033827, "nauc_mrr_at_5_std": -0.23231080115177496, "nauc_ndcg_at_1000_diff1": 0.3821434066787995, "nauc_ndcg_at_1000_max": 0.09852208182245302, "nauc_ndcg_at_1000_std": -0.19780269601157274, "nauc_ndcg_at_100_diff1": 0.35653592795330313, "nauc_ndcg_at_100_max": 0.12016785115478233, "nauc_ndcg_at_100_std": -0.16381893284621826, "nauc_ndcg_at_10_diff1": 0.3765727975354946, "nauc_ndcg_at_10_max": 0.07103763683123743, "nauc_ndcg_at_10_std": -0.2341567169326087, "nauc_ndcg_at_1_diff1": 0.5000490731839153, "nauc_ndcg_at_1_max": 0.11703668167822963, "nauc_ndcg_at_1_std": -0.21253420128790015, "nauc_ndcg_at_20_diff1": 0.36662533490475036, "nauc_ndcg_at_20_max": 0.0713928514487197, "nauc_ndcg_at_20_std": -0.2252942135515429, "nauc_ndcg_at_3_diff1": 0.42431414758585906, "nauc_ndcg_at_3_max": 0.07704900688736101, "nauc_ndcg_at_3_std": -0.2418866569683114, "nauc_ndcg_at_5_diff1": 0.3894177391749591, "nauc_ndcg_at_5_max": 0.059484874082167453, "nauc_ndcg_at_5_std": -0.24445288527546188, "nauc_precision_at_1000_diff1": 0.06255244128398815, "nauc_precision_at_1000_max": 0.141142616095593, "nauc_precision_at_1000_std": 0.14429878851751324, "nauc_precision_at_100_diff1": 0.08697360897084735, "nauc_precision_at_100_max": 0.25610915785057037, "nauc_precision_at_100_std": 0.20096301840203545, "nauc_precision_at_10_diff1": 0.27123942115127225, "nauc_precision_at_10_max": 0.08630032169377386, "nauc_precision_at_10_std": -0.16295369167027704, "nauc_precision_at_1_diff1": 0.5000490731839153, "nauc_precision_at_1_max": 0.11703668167822963, "nauc_precision_at_1_std": -0.21253420128790015, "nauc_precision_at_20_diff1": 0.23764990738737155, "nauc_precision_at_20_max": 0.08588449008707481, "nauc_precision_at_20_std": -0.12815024827595317, "nauc_precision_at_3_diff1": 0.3768282473477028, "nauc_precision_at_3_max": 0.1103862880909554, "nauc_precision_at_3_std": -0.21218349734884562, "nauc_precision_at_5_diff1": 0.30776817382059346, "nauc_precision_at_5_max": 0.06910657831809393, "nauc_precision_at_5_std": -0.20211493575780667, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.08596297357423918, "nauc_recall_at_100_max": 0.2950883540607703, "nauc_recall_at_100_std": 0.16625199068581592, "nauc_recall_at_10_diff1": 0.270419694339493, "nauc_recall_at_10_max": 0.046831921499422155, "nauc_recall_at_10_std": -0.2291113605930743, "nauc_recall_at_1_diff1": 0.47888230868159154, "nauc_recall_at_1_max": 0.08149092947082825, "nauc_recall_at_1_std": -0.22164586926600838, "nauc_recall_at_20_diff1": 0.22616433288375737, "nauc_recall_at_20_max": 0.04938458080277544, "nauc_recall_at_20_std": -0.19195410173948851, "nauc_recall_at_3_diff1": 0.36684713139195185, "nauc_recall_at_3_max": 0.05142566690039451, "nauc_recall_at_3_std": -0.2508517314624551, "nauc_recall_at_5_diff1": 0.3046746274351698, "nauc_recall_at_5_max": 0.01884075606571528, "nauc_recall_at_5_std": -0.2544658705115468, "ndcg_at_1": 0.20847, "ndcg_at_10": 0.28069, "ndcg_at_100": 0.36561, "ndcg_at_1000": 0.40056, "ndcg_at_20": 0.305, "ndcg_at_3": 0.2281, "ndcg_at_5": 0.25418, "precision_at_1": 0.20847, "precision_at_10": 0.06107, "precision_at_100": 0.01248, "precision_at_1000": 0.00167, "precision_at_20": 0.03738, "precision_at_3": 0.12866, "precision_at_5": 0.09805, "recall_at_1": 0.14384, "recall_at_10": 0.37803, "recall_at_100": 0.75915, "recall_at_1000": 1.0, "recall_at_20": 0.46473, "recall_at_3": 0.24683, "recall_at_5": 0.3057 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/model_meta.json ================================================ {"name": "Salesforce/SFR-Embedding-2_R", "revision": "91762139d94ed4371a9fa31db5551272e0b83818", "release_date": "2024-06-14", "languages": ["eng_Latn"], "n_parameters": 7110000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "cc-by-nc-4.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Salesforce/SFR-Embedding-2_R", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "task_name": "AutoRAGRetrieval", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.68421, "ndcg_at_3": 0.79354, "ndcg_at_5": 0.83356, "ndcg_at_10": 0.83863, "ndcg_at_20": 0.84288, "ndcg_at_100": 0.84288, "ndcg_at_1000": 0.84288, "map_at_1": 0.68421, "map_at_3": 0.76754, "map_at_5": 0.78991, "map_at_10": 0.79167, "map_at_20": 0.79273, "map_at_100": 0.79273, "map_at_1000": 0.79273, "recall_at_1": 0.68421, "recall_at_3": 0.86842, "recall_at_5": 0.96491, "recall_at_10": 0.98246, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.68421, "precision_at_3": 0.28947, "precision_at_5": 0.19298, "precision_at_10": 0.09825, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.684211, "mrr_at_3": 0.767544, "mrr_at_5": 0.789912, "mrr_at_10": 0.791667, "mrr_at_20": 0.792731, "mrr_at_100": 0.792731, "mrr_at_1000": 0.792731, "nauc_ndcg_at_1_max": 0.089373, "nauc_ndcg_at_1_std": -0.409446, "nauc_ndcg_at_1_diff1": 0.731574, "nauc_ndcg_at_3_max": 0.24842, "nauc_ndcg_at_3_std": -0.431082, "nauc_ndcg_at_3_diff1": 0.701366, "nauc_ndcg_at_5_max": 0.215152, "nauc_ndcg_at_5_std": -0.476746, "nauc_ndcg_at_5_diff1": 0.705081, "nauc_ndcg_at_10_max": 0.206529, "nauc_ndcg_at_10_std": -0.441859, "nauc_ndcg_at_10_diff1": 0.717517, "nauc_ndcg_at_20_max": 0.194023, "nauc_ndcg_at_20_std": -0.429968, "nauc_ndcg_at_20_diff1": 0.715736, "nauc_ndcg_at_100_max": 0.194023, "nauc_ndcg_at_100_std": -0.429968, "nauc_ndcg_at_100_diff1": 0.715736, "nauc_ndcg_at_1000_max": 0.194023, "nauc_ndcg_at_1000_std": -0.429968, "nauc_ndcg_at_1000_diff1": 0.715736, "nauc_map_at_1_max": 0.089373, "nauc_map_at_1_std": -0.409446, "nauc_map_at_1_diff1": 0.731574, "nauc_map_at_3_max": 0.207744, "nauc_map_at_3_std": -0.421012, "nauc_map_at_3_diff1": 0.712041, "nauc_map_at_5_max": 0.190054, "nauc_map_at_5_std": -0.437245, "nauc_map_at_5_diff1": 0.714414, "nauc_map_at_10_max": 0.187487, "nauc_map_at_10_std": -0.42746, "nauc_map_at_10_diff1": 0.717856, "nauc_map_at_20_max": 0.185025, "nauc_map_at_20_std": -0.424999, "nauc_map_at_20_diff1": 0.717545, "nauc_map_at_100_max": 0.185025, "nauc_map_at_100_std": -0.424999, "nauc_map_at_100_diff1": 0.717545, "nauc_map_at_1000_max": 0.185025, "nauc_map_at_1000_std": -0.424999, "nauc_map_at_1000_diff1": 0.717545, "nauc_recall_at_1_max": 0.089373, "nauc_recall_at_1_std": -0.409446, "nauc_recall_at_1_diff1": 0.731574, "nauc_recall_at_3_max": 0.431506, "nauc_recall_at_3_std": -0.480129, "nauc_recall_at_3_diff1": 0.6509, "nauc_recall_at_5_max": 0.534901, "nauc_recall_at_5_std": -1.134437, "nauc_recall_at_5_diff1": 0.567454, "nauc_recall_at_10_max": 0.613658, "nauc_recall_at_10_std": -0.817151, "nauc_recall_at_10_diff1": 0.77738, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.089373, "nauc_precision_at_1_std": -0.409446, "nauc_precision_at_1_diff1": 0.731574, "nauc_precision_at_3_max": 0.431506, "nauc_precision_at_3_std": -0.480129, "nauc_precision_at_3_diff1": 0.6509, "nauc_precision_at_5_max": 0.534901, "nauc_precision_at_5_std": -1.134437, "nauc_precision_at_5_diff1": 0.567454, "nauc_precision_at_10_max": 0.613658, "nauc_precision_at_10_std": -0.817151, "nauc_precision_at_10_diff1": 0.77738, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_1000_diff1": 1.0, "nauc_mrr_at_1_max": 0.089373, "nauc_mrr_at_1_std": -0.409446, "nauc_mrr_at_1_diff1": 0.731574, "nauc_mrr_at_3_max": 0.207744, "nauc_mrr_at_3_std": -0.421012, "nauc_mrr_at_3_diff1": 0.712041, "nauc_mrr_at_5_max": 0.190054, "nauc_mrr_at_5_std": -0.437245, "nauc_mrr_at_5_diff1": 0.714414, "nauc_mrr_at_10_max": 0.187487, "nauc_mrr_at_10_std": -0.42746, "nauc_mrr_at_10_diff1": 0.717856, "nauc_mrr_at_20_max": 0.185025, "nauc_mrr_at_20_std": -0.424999, "nauc_mrr_at_20_diff1": 0.717545, "nauc_mrr_at_100_max": 0.185025, "nauc_mrr_at_100_std": -0.424999, "nauc_mrr_at_100_diff1": 0.717545, "nauc_mrr_at_1000_max": 0.185025, "nauc_mrr_at_1000_std": -0.424999, "nauc_mrr_at_1000_diff1": 0.717545, "main_score": 0.83863, "hf_subset": "default", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 23.466877222061157, "kg_co2_emissions": null } ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "task_name": "BelebeleRetrieval", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.88222, "ndcg_at_3": 0.91395, "ndcg_at_5": 0.92314, "ndcg_at_10": 0.9271, "ndcg_at_20": 0.92943, "ndcg_at_100": 0.93212, "ndcg_at_1000": 0.93287, "map_at_1": 0.88222, "map_at_3": 0.90611, "map_at_5": 0.91122, "map_at_10": 0.91286, "map_at_20": 0.91355, "map_at_100": 0.91392, "map_at_1000": 0.91395, "recall_at_1": 0.88222, "recall_at_3": 0.93667, "recall_at_5": 0.95889, "recall_at_10": 0.97111, "recall_at_20": 0.98, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "precision_at_1": 0.88222, "precision_at_3": 0.31222, "precision_at_5": 0.19178, "precision_at_10": 0.09711, "precision_at_20": 0.049, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "mrr_at_1": 0.882222, "mrr_at_3": 0.906111, "mrr_at_5": 0.911222, "mrr_at_10": 0.91286, "mrr_at_20": 0.913546, "mrr_at_100": 0.91392, "mrr_at_1000": 0.913953, "nauc_ndcg_at_1_max": 0.871794, "nauc_ndcg_at_1_std": 0.087549, "nauc_ndcg_at_1_diff1": 0.929623, "nauc_ndcg_at_3_max": 0.894372, "nauc_ndcg_at_3_std": 0.155473, "nauc_ndcg_at_3_diff1": 0.930714, "nauc_ndcg_at_5_max": 0.897519, "nauc_ndcg_at_5_std": 0.187537, "nauc_ndcg_at_5_diff1": 0.931269, "nauc_ndcg_at_10_max": 0.893185, "nauc_ndcg_at_10_std": 0.177046, "nauc_ndcg_at_10_diff1": 0.93079, "nauc_ndcg_at_20_max": 0.891259, "nauc_ndcg_at_20_std": 0.163212, "nauc_ndcg_at_20_diff1": 0.931353, "nauc_ndcg_at_100_max": 0.888519, "nauc_ndcg_at_100_std": 0.153497, "nauc_ndcg_at_100_diff1": 0.929808, "nauc_ndcg_at_1000_max": 0.887597, "nauc_ndcg_at_1000_std": 0.147442, "nauc_ndcg_at_1000_diff1": 0.929651, "nauc_map_at_1_max": 0.871794, "nauc_map_at_1_std": 0.087549, "nauc_map_at_1_diff1": 0.929623, "nauc_map_at_3_max": 0.886618, "nauc_map_at_3_std": 0.132461, "nauc_map_at_3_diff1": 0.929284, "nauc_map_at_5_max": 0.887467, "nauc_map_at_5_std": 0.145346, "nauc_map_at_5_diff1": 0.929489, "nauc_map_at_10_max": 0.885774, "nauc_map_at_10_std": 0.140933, "nauc_map_at_10_diff1": 0.92927, "nauc_map_at_20_max": 0.885161, "nauc_map_at_20_std": 0.137009, "nauc_map_at_20_diff1": 0.929247, "nauc_map_at_100_max": 0.88479, "nauc_map_at_100_std": 0.135555, "nauc_map_at_100_diff1": 0.929069, "nauc_map_at_1000_max": 0.884762, "nauc_map_at_1000_std": 0.135323, "nauc_map_at_1000_diff1": 0.929069, "nauc_recall_at_1_max": 0.871794, "nauc_recall_at_1_std": 0.087549, "nauc_recall_at_1_diff1": 0.929623, "nauc_recall_at_3_max": 0.92772, "nauc_recall_at_3_std": 0.254279, "nauc_recall_at_3_diff1": 0.937425, "nauc_recall_at_5_max": 0.961768, "nauc_recall_at_5_std": 0.462286, "nauc_recall_at_5_diff1": 0.943347, "nauc_recall_at_10_max": 0.955649, "nauc_recall_at_10_std": 0.497684, "nauc_recall_at_10_diff1": 0.945145, "nauc_recall_at_20_max": 0.960707, "nauc_recall_at_20_std": 0.466205, "nauc_recall_at_20_diff1": 0.963689, "nauc_recall_at_100_max": 0.973856, "nauc_recall_at_100_std": 0.679832, "nauc_recall_at_100_diff1": 0.947712, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.871794, "nauc_precision_at_1_std": 0.087549, "nauc_precision_at_1_diff1": 0.929623, "nauc_precision_at_3_max": 0.92772, "nauc_precision_at_3_std": 0.254279, "nauc_precision_at_3_diff1": 0.937425, "nauc_precision_at_5_max": 0.961768, "nauc_precision_at_5_std": 0.462286, "nauc_precision_at_5_diff1": 0.943347, "nauc_precision_at_10_max": 0.955649, "nauc_precision_at_10_std": 0.497684, "nauc_precision_at_10_diff1": 0.945145, "nauc_precision_at_20_max": 0.960707, "nauc_precision_at_20_std": 0.466205, "nauc_precision_at_20_diff1": 0.963689, "nauc_precision_at_100_max": 0.973856, "nauc_precision_at_100_std": 0.679832, "nauc_precision_at_100_diff1": 0.947712, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.871794, "nauc_mrr_at_1_std": 0.087549, "nauc_mrr_at_1_diff1": 0.929623, "nauc_mrr_at_3_max": 0.886618, "nauc_mrr_at_3_std": 0.132461, "nauc_mrr_at_3_diff1": 0.929284, "nauc_mrr_at_5_max": 0.887467, "nauc_mrr_at_5_std": 0.145346, "nauc_mrr_at_5_diff1": 0.929489, "nauc_mrr_at_10_max": 0.885774, "nauc_mrr_at_10_std": 0.140933, "nauc_mrr_at_10_diff1": 0.92927, "nauc_mrr_at_20_max": 0.885161, "nauc_mrr_at_20_std": 0.137009, "nauc_mrr_at_20_diff1": 0.929247, "nauc_mrr_at_100_max": 0.88479, "nauc_mrr_at_100_std": 0.135555, "nauc_mrr_at_100_diff1": 0.929069, "nauc_mrr_at_1000_max": 0.884762, "nauc_mrr_at_1000_std": 0.135323, "nauc_mrr_at_1000_diff1": 0.929069, "main_score": 0.9271, "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ] }, { "ndcg_at_1": 0.82111, "ndcg_at_3": 0.87551, "ndcg_at_5": 0.88427, "ndcg_at_10": 0.89361, "ndcg_at_20": 0.89785, "ndcg_at_100": 0.90122, "ndcg_at_1000": 0.9015, "map_at_1": 0.82111, "map_at_3": 0.86241, "map_at_5": 0.8673, "map_at_10": 0.87116, "map_at_20": 0.87233, "map_at_100": 0.87283, "map_at_1000": 0.87284, "recall_at_1": 0.82111, "recall_at_3": 0.91333, "recall_at_5": 0.93444, "recall_at_10": 0.96333, "recall_at_20": 0.98, "recall_at_100": 0.99778, "recall_at_1000": 1.0, "precision_at_1": 0.82111, "precision_at_3": 0.30444, "precision_at_5": 0.18689, "precision_at_10": 0.09633, "precision_at_20": 0.049, "precision_at_100": 0.00998, "precision_at_1000": 0.001, "mrr_at_1": 0.821111, "mrr_at_3": 0.862407, "mrr_at_5": 0.867296, "mrr_at_10": 0.871156, "mrr_at_20": 0.872332, "mrr_at_100": 0.872827, "mrr_at_1000": 0.872837, "nauc_ndcg_at_1_max": 0.802669, "nauc_ndcg_at_1_std": -0.00893, "nauc_ndcg_at_1_diff1": 0.884766, "nauc_ndcg_at_3_max": 0.827755, "nauc_ndcg_at_3_std": 0.013666, "nauc_ndcg_at_3_diff1": 0.854555, "nauc_ndcg_at_5_max": 0.832537, "nauc_ndcg_at_5_std": 0.045388, "nauc_ndcg_at_5_diff1": 0.858638, "nauc_ndcg_at_10_max": 0.832546, "nauc_ndcg_at_10_std": 0.049355, "nauc_ndcg_at_10_diff1": 0.861864, "nauc_ndcg_at_20_max": 0.82833, "nauc_ndcg_at_20_std": 0.037394, "nauc_ndcg_at_20_diff1": 0.864736, "nauc_ndcg_at_100_max": 0.823982, "nauc_ndcg_at_100_std": 0.026893, "nauc_ndcg_at_100_diff1": 0.866592, "nauc_ndcg_at_1000_max": 0.823846, "nauc_ndcg_at_1000_std": 0.026387, "nauc_ndcg_at_1000_diff1": 0.866377, "nauc_map_at_1_max": 0.802669, "nauc_map_at_1_std": -0.00893, "nauc_map_at_1_diff1": 0.884766, "nauc_map_at_3_max": 0.820357, "nauc_map_at_3_std": 0.007998, "nauc_map_at_3_diff1": 0.863485, "nauc_map_at_5_max": 0.822401, "nauc_map_at_5_std": 0.023728, "nauc_map_at_5_diff1": 0.865723, "nauc_map_at_10_max": 0.822178, "nauc_map_at_10_std": 0.024889, "nauc_map_at_10_diff1": 0.867105, "nauc_map_at_20_max": 0.821079, "nauc_map_at_20_std": 0.021636, "nauc_map_at_20_diff1": 0.867859, "nauc_map_at_100_max": 0.82055, "nauc_map_at_100_std": 0.020074, "nauc_map_at_100_diff1": 0.868173, "nauc_map_at_1000_max": 0.820543, "nauc_map_at_1000_std": 0.020045, "nauc_map_at_1000_diff1": 0.868166, "nauc_recall_at_1_max": 0.802669, "nauc_recall_at_1_std": -0.00893, "nauc_recall_at_1_diff1": 0.884766, "nauc_recall_at_3_max": 0.860094, "nauc_recall_at_3_std": 0.037791, "nauc_recall_at_3_diff1": 0.815503, "nauc_recall_at_5_max": 0.890289, "nauc_recall_at_5_std": 0.172752, "nauc_recall_at_5_diff1": 0.820578, "nauc_recall_at_10_max": 0.934145, "nauc_recall_at_10_std": 0.298333, "nauc_recall_at_10_diff1": 0.817432, "nauc_recall_at_20_max": 0.940087, "nauc_recall_at_20_std": 0.289786, "nauc_recall_at_20_diff1": 0.8331, "nauc_recall_at_100_max": 0.861111, "nauc_recall_at_100_std": 0.152894, "nauc_recall_at_100_diff1": 0.934641, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.802669, "nauc_precision_at_1_std": -0.00893, "nauc_precision_at_1_diff1": 0.884766, "nauc_precision_at_3_max": 0.860094, "nauc_precision_at_3_std": 0.037791, "nauc_precision_at_3_diff1": 0.815503, "nauc_precision_at_5_max": 0.890289, "nauc_precision_at_5_std": 0.172752, "nauc_precision_at_5_diff1": 0.820578, "nauc_precision_at_10_max": 0.934145, "nauc_precision_at_10_std": 0.298333, "nauc_precision_at_10_diff1": 0.817432, "nauc_precision_at_20_max": 0.940087, "nauc_precision_at_20_std": 0.289786, "nauc_precision_at_20_diff1": 0.8331, "nauc_precision_at_100_max": 0.861111, "nauc_precision_at_100_std": 0.152894, "nauc_precision_at_100_diff1": 0.934641, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.802669, "nauc_mrr_at_1_std": -0.00893, "nauc_mrr_at_1_diff1": 0.884766, "nauc_mrr_at_3_max": 0.820357, "nauc_mrr_at_3_std": 0.007998, "nauc_mrr_at_3_diff1": 0.863485, "nauc_mrr_at_5_max": 0.822401, "nauc_mrr_at_5_std": 0.023728, "nauc_mrr_at_5_diff1": 0.865723, "nauc_mrr_at_10_max": 0.822178, "nauc_mrr_at_10_std": 0.024889, "nauc_mrr_at_10_diff1": 0.867105, "nauc_mrr_at_20_max": 0.821079, "nauc_mrr_at_20_std": 0.021636, "nauc_mrr_at_20_diff1": 0.867859, "nauc_mrr_at_100_max": 0.82055, "nauc_mrr_at_100_std": 0.020074, "nauc_mrr_at_100_diff1": 0.868173, "nauc_mrr_at_1000_max": 0.820543, "nauc_mrr_at_1000_std": 0.020045, "nauc_mrr_at_1000_diff1": 0.868166, "main_score": 0.89361, "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ] }, { "ndcg_at_1": 0.79667, "ndcg_at_3": 0.84704, "ndcg_at_5": 0.85765, "ndcg_at_10": 0.86663, "ndcg_at_20": 0.87407, "ndcg_at_100": 0.87837, "ndcg_at_1000": 0.88035, "map_at_1": 0.79667, "map_at_3": 0.83519, "map_at_5": 0.84113, "map_at_10": 0.84483, "map_at_20": 0.84695, "map_at_100": 0.8476, "map_at_1000": 0.8477, "recall_at_1": 0.79667, "recall_at_3": 0.88111, "recall_at_5": 0.90667, "recall_at_10": 0.93444, "recall_at_20": 0.96333, "recall_at_100": 0.98556, "recall_at_1000": 1.0, "precision_at_1": 0.79667, "precision_at_3": 0.2937, "precision_at_5": 0.18133, "precision_at_10": 0.09344, "precision_at_20": 0.04817, "precision_at_100": 0.00986, "precision_at_1000": 0.001, "mrr_at_1": 0.796667, "mrr_at_3": 0.835185, "mrr_at_5": 0.84113, "mrr_at_10": 0.84483, "mrr_at_20": 0.846945, "mrr_at_100": 0.847603, "mrr_at_1000": 0.847697, "nauc_ndcg_at_1_max": 0.840491, "nauc_ndcg_at_1_std": 0.215939, "nauc_ndcg_at_1_diff1": 0.888987, "nauc_ndcg_at_3_max": 0.847283, "nauc_ndcg_at_3_std": 0.237854, "nauc_ndcg_at_3_diff1": 0.870479, "nauc_ndcg_at_5_max": 0.846485, "nauc_ndcg_at_5_std": 0.253122, "nauc_ndcg_at_5_diff1": 0.864964, "nauc_ndcg_at_10_max": 0.847742, "nauc_ndcg_at_10_std": 0.266191, "nauc_ndcg_at_10_diff1": 0.866941, "nauc_ndcg_at_20_max": 0.85193, "nauc_ndcg_at_20_std": 0.266515, "nauc_ndcg_at_20_diff1": 0.872463, "nauc_ndcg_at_100_max": 0.851502, "nauc_ndcg_at_100_std": 0.261466, "nauc_ndcg_at_100_diff1": 0.874104, "nauc_ndcg_at_1000_max": 0.849079, "nauc_ndcg_at_1000_std": 0.252442, "nauc_ndcg_at_1000_diff1": 0.87355, "nauc_map_at_1_max": 0.840491, "nauc_map_at_1_std": 0.215939, "nauc_map_at_1_diff1": 0.888987, "nauc_map_at_3_max": 0.846426, "nauc_map_at_3_std": 0.233667, "nauc_map_at_3_diff1": 0.875654, "nauc_map_at_5_max": 0.846003, "nauc_map_at_5_std": 0.240665, "nauc_map_at_5_diff1": 0.873101, "nauc_map_at_10_max": 0.846476, "nauc_map_at_10_std": 0.244957, "nauc_map_at_10_diff1": 0.873953, "nauc_map_at_20_max": 0.847544, "nauc_map_at_20_std": 0.244568, "nauc_map_at_20_diff1": 0.875416, "nauc_map_at_100_max": 0.847432, "nauc_map_at_100_std": 0.243779, "nauc_map_at_100_diff1": 0.875553, "nauc_map_at_1000_max": 0.847336, "nauc_map_at_1000_std": 0.243415, "nauc_map_at_1000_diff1": 0.875536, "nauc_recall_at_1_max": 0.840491, "nauc_recall_at_1_std": 0.215939, "nauc_recall_at_1_diff1": 0.888987, "nauc_recall_at_3_max": 0.849974, "nauc_recall_at_3_std": 0.252823, "nauc_recall_at_3_diff1": 0.850589, "nauc_recall_at_5_max": 0.847761, "nauc_recall_at_5_std": 0.314092, "nauc_recall_at_5_diff1": 0.825914, "nauc_recall_at_10_max": 0.855378, "nauc_recall_at_10_std": 0.419037, "nauc_recall_at_10_diff1": 0.821899, "nauc_recall_at_20_max": 0.911524, "nauc_recall_at_20_std": 0.547803, "nauc_recall_at_20_diff1": 0.854794, "nauc_recall_at_100_max": 0.989945, "nauc_recall_at_100_std": 0.775946, "nauc_recall_at_100_diff1": 0.906988, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.840491, "nauc_precision_at_1_std": 0.215939, "nauc_precision_at_1_diff1": 0.888987, "nauc_precision_at_3_max": 0.849974, "nauc_precision_at_3_std": 0.252823, "nauc_precision_at_3_diff1": 0.850589, "nauc_precision_at_5_max": 0.847761, "nauc_precision_at_5_std": 0.314092, "nauc_precision_at_5_diff1": 0.825914, "nauc_precision_at_10_max": 0.855378, "nauc_precision_at_10_std": 0.419037, "nauc_precision_at_10_diff1": 0.821899, "nauc_precision_at_20_max": 0.911524, "nauc_precision_at_20_std": 0.547803, "nauc_precision_at_20_diff1": 0.854794, "nauc_precision_at_100_max": 0.989945, "nauc_precision_at_100_std": 0.775946, "nauc_precision_at_100_diff1": 0.906988, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.840491, "nauc_mrr_at_1_std": 0.215939, "nauc_mrr_at_1_diff1": 0.888987, "nauc_mrr_at_3_max": 0.846426, "nauc_mrr_at_3_std": 0.233667, "nauc_mrr_at_3_diff1": 0.875654, "nauc_mrr_at_5_max": 0.846003, "nauc_mrr_at_5_std": 0.240665, "nauc_mrr_at_5_diff1": 0.873101, "nauc_mrr_at_10_max": 0.846476, "nauc_mrr_at_10_std": 0.244957, "nauc_mrr_at_10_diff1": 0.873953, "nauc_mrr_at_20_max": 0.847544, "nauc_mrr_at_20_std": 0.244568, "nauc_mrr_at_20_diff1": 0.875416, "nauc_mrr_at_100_max": 0.847432, "nauc_mrr_at_100_std": 0.243779, "nauc_mrr_at_100_diff1": 0.875553, "nauc_mrr_at_1000_max": 0.847336, "nauc_mrr_at_1000_std": 0.243415, "nauc_mrr_at_1000_diff1": 0.875536, "main_score": 0.86663, "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ] } ] }, "evaluation_time": 19.306873321533203, "kg_co2_emissions": null } ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "task_name": "Ko-StrategyQA", "mteb_version": "1.21.6", "scores": { "dev": [ { "ndcg_at_1": 0.78378, "ndcg_at_3": 0.75718, "ndcg_at_5": 0.78694, "ndcg_at_10": 0.80455, "ndcg_at_20": 0.81148, "ndcg_at_100": 0.81914, "ndcg_at_1000": 0.82687, "map_at_1": 0.50886, "map_at_3": 0.71843, "map_at_5": 0.74839, "map_at_10": 0.76025, "map_at_20": 0.7634, "map_at_100": 0.76489, "map_at_1000": 0.76521, "recall_at_1": 0.50886, "recall_at_3": 0.75575, "recall_at_5": 0.81673, "recall_at_10": 0.85876, "recall_at_20": 0.87945, "recall_at_100": 0.91277, "recall_at_1000": 0.96273, "precision_at_1": 0.78378, "precision_at_3": 0.44369, "precision_at_5": 0.29662, "precision_at_10": 0.15861, "precision_at_20": 0.08193, "precision_at_100": 0.01715, "precision_at_1000": 0.00183, "mrr_at_1": 0.783784, "mrr_at_3": 0.819257, "mrr_at_5": 0.824916, "mrr_at_10": 0.827873, "mrr_at_20": 0.828349, "mrr_at_100": 0.828891, "mrr_at_1000": 0.829072, "nauc_ndcg_at_1_max": 0.66351, "nauc_ndcg_at_1_std": 0.342961, "nauc_ndcg_at_1_diff1": 0.703978, "nauc_ndcg_at_3_max": 0.600697, "nauc_ndcg_at_3_std": 0.246012, "nauc_ndcg_at_3_diff1": 0.555382, "nauc_ndcg_at_5_max": 0.656793, "nauc_ndcg_at_5_std": 0.310234, "nauc_ndcg_at_5_diff1": 0.564383, "nauc_ndcg_at_10_max": 0.68711, "nauc_ndcg_at_10_std": 0.366137, "nauc_ndcg_at_10_diff1": 0.57362, "nauc_ndcg_at_20_max": 0.698608, "nauc_ndcg_at_20_std": 0.388451, "nauc_ndcg_at_20_diff1": 0.580961, "nauc_ndcg_at_100_max": 0.70173, "nauc_ndcg_at_100_std": 0.393134, "nauc_ndcg_at_100_diff1": 0.595129, "nauc_ndcg_at_1000_max": 0.689353, "nauc_ndcg_at_1000_std": 0.379922, "nauc_ndcg_at_1000_diff1": 0.589623, "nauc_map_at_1_max": 0.274554, "nauc_map_at_1_std": 0.049087, "nauc_map_at_1_diff1": 0.603685, "nauc_map_at_3_max": 0.552057, "nauc_map_at_3_std": 0.19298, "nauc_map_at_3_diff1": 0.5385, "nauc_map_at_5_max": 0.599469, "nauc_map_at_5_std": 0.241903, "nauc_map_at_5_diff1": 0.539366, "nauc_map_at_10_max": 0.616915, "nauc_map_at_10_std": 0.274562, "nauc_map_at_10_diff1": 0.543664, "nauc_map_at_20_max": 0.620948, "nauc_map_at_20_std": 0.28246, "nauc_map_at_20_diff1": 0.546535, "nauc_map_at_100_max": 0.621487, "nauc_map_at_100_std": 0.28351, "nauc_map_at_100_diff1": 0.549163, "nauc_map_at_1000_max": 0.621058, "nauc_map_at_1000_std": 0.283215, "nauc_map_at_1000_diff1": 0.548932, "nauc_recall_at_1_max": 0.274554, "nauc_recall_at_1_std": 0.049087, "nauc_recall_at_1_diff1": 0.603685, "nauc_recall_at_3_max": 0.586384, "nauc_recall_at_3_std": 0.220095, "nauc_recall_at_3_diff1": 0.509868, "nauc_recall_at_5_max": 0.695287, "nauc_recall_at_5_std": 0.34578, "nauc_recall_at_5_diff1": 0.514654, "nauc_recall_at_10_max": 0.790076, "nauc_recall_at_10_std": 0.51455, "nauc_recall_at_10_diff1": 0.528608, "nauc_recall_at_20_max": 0.865104, "nauc_recall_at_20_std": 0.640404, "nauc_recall_at_20_diff1": 0.560792, "nauc_recall_at_100_max": 0.953183, "nauc_recall_at_100_std": 0.763603, "nauc_recall_at_100_diff1": 0.671003, "nauc_recall_at_1000_max": 0.947876, "nauc_recall_at_1000_std": 0.861736, "nauc_recall_at_1000_diff1": 0.622166, "nauc_precision_at_1_max": 0.66351, "nauc_precision_at_1_std": 0.342961, "nauc_precision_at_1_diff1": 0.703978, "nauc_precision_at_3_max": 0.39359, "nauc_precision_at_3_std": 0.248765, "nauc_precision_at_3_diff1": -0.007857, "nauc_precision_at_5_max": 0.328642, "nauc_precision_at_5_std": 0.264825, "nauc_precision_at_5_diff1": -0.090444, "nauc_precision_at_10_max": 0.288377, "nauc_precision_at_10_std": 0.305151, "nauc_precision_at_10_diff1": -0.13785, "nauc_precision_at_20_max": 0.261299, "nauc_precision_at_20_std": 0.315143, "nauc_precision_at_20_diff1": -0.156061, "nauc_precision_at_100_max": 0.202035, "nauc_precision_at_100_std": 0.287929, "nauc_precision_at_100_diff1": -0.18364, "nauc_precision_at_1000_max": 0.045862, "nauc_precision_at_1000_std": 0.184977, "nauc_precision_at_1000_diff1": -0.309691, "nauc_mrr_at_1_max": 0.66351, "nauc_mrr_at_1_std": 0.342961, "nauc_mrr_at_1_diff1": 0.703978, "nauc_mrr_at_3_max": 0.737671, "nauc_mrr_at_3_std": 0.42406, "nauc_mrr_at_3_diff1": 0.683418, "nauc_mrr_at_5_max": 0.736757, "nauc_mrr_at_5_std": 0.436965, "nauc_mrr_at_5_diff1": 0.684921, "nauc_mrr_at_10_max": 0.740439, "nauc_mrr_at_10_std": 0.442523, "nauc_mrr_at_10_diff1": 0.686669, "nauc_mrr_at_20_max": 0.740304, "nauc_mrr_at_20_std": 0.442289, "nauc_mrr_at_20_diff1": 0.686576, "nauc_mrr_at_100_max": 0.739776, "nauc_mrr_at_100_std": 0.441713, "nauc_mrr_at_100_diff1": 0.687177, "nauc_mrr_at_1000_max": 0.739504, "nauc_mrr_at_1000_std": 0.441296, "nauc_mrr_at_1000_diff1": 0.687133, "main_score": 0.80455, "hf_subset": "default", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 116.52639317512512, "kg_co2_emissions": null } ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "task_name": "MIRACLRetrieval", "mteb_version": "1.21.6", "scores": { "dev": [ { "ndcg_at_1": 0.6338, "ndcg_at_3": 0.62186, "ndcg_at_5": 0.63188, "ndcg_at_10": 0.66077, "ndcg_at_20": 0.6872, "ndcg_at_100": 0.71572, "ndcg_at_1000": 0.72622, "map_at_1": 0.39586, "map_at_3": 0.50791, "map_at_5": 0.54136, "map_at_10": 0.56835, "map_at_20": 0.58173, "map_at_100": 0.58996, "map_at_1000": 0.59094, "recall_at_1": 0.39586, "recall_at_3": 0.57785, "recall_at_5": 0.65636, "recall_at_10": 0.74806, "recall_at_20": 0.82219, "recall_at_100": 0.92769, "recall_at_1000": 0.98419, "precision_at_1": 0.6338, "precision_at_3": 0.3662, "precision_at_5": 0.26667, "precision_at_10": 0.16479, "precision_at_20": 0.09577, "precision_at_100": 0.02319, "precision_at_1000": 0.00254, "mrr_at_1": 0.633803, "mrr_at_3": 0.720657, "mrr_at_5": 0.732394, "mrr_at_10": 0.737898, "mrr_at_20": 0.739605, "mrr_at_100": 0.740413, "mrr_at_1000": 0.740491, "nauc_ndcg_at_1_max": 0.36433, "nauc_ndcg_at_1_std": 0.057948, "nauc_ndcg_at_1_diff1": 0.317296, "nauc_ndcg_at_3_max": 0.291245, "nauc_ndcg_at_3_std": 0.012819, "nauc_ndcg_at_3_diff1": 0.229384, "nauc_ndcg_at_5_max": 0.301409, "nauc_ndcg_at_5_std": -0.00086, "nauc_ndcg_at_5_diff1": 0.225941, "nauc_ndcg_at_10_max": 0.327252, "nauc_ndcg_at_10_std": 0.018858, "nauc_ndcg_at_10_diff1": 0.222747, "nauc_ndcg_at_20_max": 0.372009, "nauc_ndcg_at_20_std": 0.083221, "nauc_ndcg_at_20_diff1": 0.229619, "nauc_ndcg_at_100_max": 0.364411, "nauc_ndcg_at_100_std": 0.077404, "nauc_ndcg_at_100_diff1": 0.214304, "nauc_ndcg_at_1000_max": 0.357641, "nauc_ndcg_at_1000_std": 0.079982, "nauc_ndcg_at_1000_diff1": 0.219213, "nauc_map_at_1_max": 0.225558, "nauc_map_at_1_std": -0.154759, "nauc_map_at_1_diff1": 0.35403, "nauc_map_at_3_max": 0.253169, "nauc_map_at_3_std": -0.091057, "nauc_map_at_3_diff1": 0.275777, "nauc_map_at_5_max": 0.280381, "nauc_map_at_5_std": -0.048579, "nauc_map_at_5_diff1": 0.256842, "nauc_map_at_10_max": 0.297772, "nauc_map_at_10_std": -0.02757, "nauc_map_at_10_diff1": 0.25329, "nauc_map_at_20_max": 0.315153, "nauc_map_at_20_std": -0.002361, "nauc_map_at_20_diff1": 0.254841, "nauc_map_at_100_max": 0.314783, "nauc_map_at_100_std": -0.002325, "nauc_map_at_100_diff1": 0.25129, "nauc_map_at_1000_max": 0.314298, "nauc_map_at_1000_std": -0.001981, "nauc_map_at_1000_diff1": 0.251598, "nauc_recall_at_1_max": 0.225558, "nauc_recall_at_1_std": -0.154759, "nauc_recall_at_1_diff1": 0.35403, "nauc_recall_at_3_max": 0.200469, "nauc_recall_at_3_std": -0.121722, "nauc_recall_at_3_diff1": 0.212938, "nauc_recall_at_5_max": 0.2192, "nauc_recall_at_5_std": -0.077641, "nauc_recall_at_5_diff1": 0.161298, "nauc_recall_at_10_max": 0.25392, "nauc_recall_at_10_std": -0.032093, "nauc_recall_at_10_diff1": 0.125612, "nauc_recall_at_20_max": 0.398226, "nauc_recall_at_20_std": 0.190764, "nauc_recall_at_20_diff1": 0.140787, "nauc_recall_at_100_max": 0.400267, "nauc_recall_at_100_std": 0.142762, "nauc_recall_at_100_diff1": -0.043031, "nauc_recall_at_1000_max": 0.373517, "nauc_recall_at_1000_std": 0.503357, "nauc_recall_at_1000_diff1": -0.317952, "nauc_precision_at_1_max": 0.36433, "nauc_precision_at_1_std": 0.057948, "nauc_precision_at_1_diff1": 0.317296, "nauc_precision_at_3_max": 0.204832, "nauc_precision_at_3_std": 0.293384, "nauc_precision_at_3_diff1": -0.116665, "nauc_precision_at_5_max": 0.188669, "nauc_precision_at_5_std": 0.332137, "nauc_precision_at_5_diff1": -0.147681, "nauc_precision_at_10_max": 0.161853, "nauc_precision_at_10_std": 0.332841, "nauc_precision_at_10_diff1": -0.173471, "nauc_precision_at_20_max": 0.16721, "nauc_precision_at_20_std": 0.396508, "nauc_precision_at_20_diff1": -0.184459, "nauc_precision_at_100_max": 0.063707, "nauc_precision_at_100_std": 0.323576, "nauc_precision_at_100_diff1": -0.227542, "nauc_precision_at_1000_max": 0.01149, "nauc_precision_at_1000_std": 0.295511, "nauc_precision_at_1000_diff1": -0.213489, "nauc_mrr_at_1_max": 0.36433, "nauc_mrr_at_1_std": 0.057948, "nauc_mrr_at_1_diff1": 0.317296, "nauc_mrr_at_3_max": 0.401771, "nauc_mrr_at_3_std": 0.120315, "nauc_mrr_at_3_diff1": 0.236438, "nauc_mrr_at_5_max": 0.404058, "nauc_mrr_at_5_std": 0.113415, "nauc_mrr_at_5_diff1": 0.24794, "nauc_mrr_at_10_max": 0.402361, "nauc_mrr_at_10_std": 0.120171, "nauc_mrr_at_10_diff1": 0.240421, "nauc_mrr_at_20_max": 0.403472, "nauc_mrr_at_20_std": 0.12475, "nauc_mrr_at_20_diff1": 0.241169, "nauc_mrr_at_100_max": 0.403146, "nauc_mrr_at_100_std": 0.122353, "nauc_mrr_at_100_diff1": 0.243043, "nauc_mrr_at_1000_max": 0.403191, "nauc_mrr_at_1000_std": 0.122326, "nauc_mrr_at_1000_diff1": 0.243267, "main_score": 0.66077, "hf_subset": "ko", "languages": [ "kor-Kore" ] } ] }, "evaluation_time": 10909.090243339539, "kg_co2_emissions": null } ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "task_name": "MrTidyRetrieval", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.46793, "ndcg_at_3": 0.5442, "ndcg_at_5": 0.56769, "ndcg_at_10": 0.59071, "ndcg_at_20": 0.61105, "ndcg_at_100": 0.63034, "ndcg_at_1000": 0.63913, "map_at_1": 0.42953, "map_at_3": 0.51326, "map_at_5": 0.52803, "map_at_10": 0.53806, "map_at_20": 0.54429, "map_at_100": 0.54752, "map_at_1000": 0.5478, "recall_at_1": 0.42953, "recall_at_3": 0.60333, "recall_at_5": 0.65717, "recall_at_10": 0.72328, "recall_at_20": 0.7981, "recall_at_100": 0.89509, "recall_at_1000": 0.96556, "precision_at_1": 0.46793, "precision_at_3": 0.22169, "precision_at_5": 0.14679, "precision_at_10": 0.08242, "precision_at_20": 0.0462, "precision_at_100": 0.01045, "precision_at_1000": 0.00113, "mrr_at_1": 0.467933, "mrr_at_3": 0.545527, "mrr_at_5": 0.556215, "mrr_at_10": 0.565528, "mrr_at_20": 0.570359, "mrr_at_100": 0.572393, "mrr_at_1000": 0.572636, "nauc_ndcg_at_1_max": 0.347778, "nauc_ndcg_at_1_std": -0.058339, "nauc_ndcg_at_1_diff1": 0.501989, "nauc_ndcg_at_3_max": 0.321743, "nauc_ndcg_at_3_std": -0.021783, "nauc_ndcg_at_3_diff1": 0.431511, "nauc_ndcg_at_5_max": 0.332574, "nauc_ndcg_at_5_std": -0.004894, "nauc_ndcg_at_5_diff1": 0.424748, "nauc_ndcg_at_10_max": 0.340703, "nauc_ndcg_at_10_std": -0.012762, "nauc_ndcg_at_10_diff1": 0.427041, "nauc_ndcg_at_20_max": 0.3389, "nauc_ndcg_at_20_std": -0.003515, "nauc_ndcg_at_20_diff1": 0.424289, "nauc_ndcg_at_100_max": 0.343308, "nauc_ndcg_at_100_std": 0.007776, "nauc_ndcg_at_100_diff1": 0.43105, "nauc_ndcg_at_1000_max": 0.342666, "nauc_ndcg_at_1000_std": -2.3e-05, "nauc_ndcg_at_1000_diff1": 0.432686, "nauc_map_at_1_max": 0.322004, "nauc_map_at_1_std": -0.060647, "nauc_map_at_1_diff1": 0.479496, "nauc_map_at_3_max": 0.315617, "nauc_map_at_3_std": -0.032644, "nauc_map_at_3_diff1": 0.440521, "nauc_map_at_5_max": 0.323492, "nauc_map_at_5_std": -0.022745, "nauc_map_at_5_diff1": 0.436075, "nauc_map_at_10_max": 0.326869, "nauc_map_at_10_std": -0.027004, "nauc_map_at_10_diff1": 0.437014, "nauc_map_at_20_max": 0.326, "nauc_map_at_20_std": -0.024601, "nauc_map_at_20_diff1": 0.436045, "nauc_map_at_100_max": 0.326345, "nauc_map_at_100_std": -0.023394, "nauc_map_at_100_diff1": 0.437366, "nauc_map_at_1000_max": 0.326387, "nauc_map_at_1000_std": -0.023538, "nauc_map_at_1000_diff1": 0.437423, "nauc_recall_at_1_max": 0.322004, "nauc_recall_at_1_std": -0.060647, "nauc_recall_at_1_diff1": 0.479496, "nauc_recall_at_3_max": 0.307551, "nauc_recall_at_3_std": 0.011178, "nauc_recall_at_3_diff1": 0.380203, "nauc_recall_at_5_max": 0.330094, "nauc_recall_at_5_std": 0.054829, "nauc_recall_at_5_diff1": 0.359187, "nauc_recall_at_10_max": 0.356746, "nauc_recall_at_10_std": 0.039576, "nauc_recall_at_10_diff1": 0.356908, "nauc_recall_at_20_max": 0.359008, "nauc_recall_at_20_std": 0.103216, "nauc_recall_at_20_diff1": 0.330975, "nauc_recall_at_100_max": 0.448561, "nauc_recall_at_100_std": 0.339303, "nauc_recall_at_100_diff1": 0.34747, "nauc_recall_at_1000_max": 0.604003, "nauc_recall_at_1000_std": 0.505597, "nauc_recall_at_1000_diff1": 0.291852, "nauc_precision_at_1_max": 0.347778, "nauc_precision_at_1_std": -0.058339, "nauc_precision_at_1_diff1": 0.501989, "nauc_precision_at_3_max": 0.293071, "nauc_precision_at_3_std": 0.002078, "nauc_precision_at_3_diff1": 0.346977, "nauc_precision_at_5_max": 0.288524, "nauc_precision_at_5_std": 0.054756, "nauc_precision_at_5_diff1": 0.273477, "nauc_precision_at_10_max": 0.260607, "nauc_precision_at_10_std": 0.032127, "nauc_precision_at_10_diff1": 0.206793, "nauc_precision_at_20_max": 0.202122, "nauc_precision_at_20_std": 0.091149, "nauc_precision_at_20_diff1": 0.111556, "nauc_precision_at_100_max": 0.138927, "nauc_precision_at_100_std": 0.151481, "nauc_precision_at_100_diff1": 0.039547, "nauc_precision_at_1000_max": 0.053952, "nauc_precision_at_1000_std": 0.100944, "nauc_precision_at_1000_diff1": -0.064182, "nauc_mrr_at_1_max": 0.347778, "nauc_mrr_at_1_std": -0.058339, "nauc_mrr_at_1_diff1": 0.501989, "nauc_mrr_at_3_max": 0.343049, "nauc_mrr_at_3_std": -0.030596, "nauc_mrr_at_3_diff1": 0.456467, "nauc_mrr_at_5_max": 0.346555, "nauc_mrr_at_5_std": -0.022801, "nauc_mrr_at_5_diff1": 0.45453, "nauc_mrr_at_10_max": 0.350391, "nauc_mrr_at_10_std": -0.026001, "nauc_mrr_at_10_diff1": 0.456921, "nauc_mrr_at_20_max": 0.349081, "nauc_mrr_at_20_std": -0.024796, "nauc_mrr_at_20_diff1": 0.456687, "nauc_mrr_at_100_max": 0.3492, "nauc_mrr_at_100_std": -0.023745, "nauc_mrr_at_100_diff1": 0.457028, "nauc_mrr_at_1000_max": 0.349218, "nauc_mrr_at_1000_std": -0.023931, "nauc_mrr_at_1000_diff1": 0.45708, "main_score": 0.59071, "hf_subset": "korean", "languages": [ "kor-Kore" ] } ] }, "evaluation_time": 11037.680595874786, "kg_co2_emissions": null } ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "task_name": "MultiLongDocRetrieval", "mteb_version": "1.21.6", "scores": { "dev": [ { "ndcg_at_1": 0.29, "ndcg_at_3": 0.35917, "ndcg_at_5": 0.38176, "ndcg_at_10": 0.40401, "ndcg_at_20": 0.42135, "ndcg_at_100": 0.4465, "ndcg_at_1000": 0.4683, "map_at_1": 0.29, "map_at_3": 0.34167, "map_at_5": 0.35417, "map_at_10": 0.3631, "map_at_20": 0.36767, "map_at_100": 0.37114, "map_at_1000": 0.3719, "recall_at_1": 0.29, "recall_at_3": 0.41, "recall_at_5": 0.465, "recall_at_10": 0.535, "recall_at_20": 0.605, "recall_at_100": 0.74, "recall_at_1000": 0.915, "precision_at_1": 0.29, "precision_at_3": 0.13667, "precision_at_5": 0.093, "precision_at_10": 0.0535, "precision_at_20": 0.03025, "precision_at_100": 0.0074, "precision_at_1000": 0.00092, "mrr_at_1": 0.29, "mrr_at_3": 0.341667, "mrr_at_5": 0.354167, "mrr_at_10": 0.363099, "mrr_at_20": 0.367668, "mrr_at_100": 0.371138, "mrr_at_1000": 0.371897, "nauc_ndcg_at_1_max": 0.39918, "nauc_ndcg_at_1_std": 0.027622, "nauc_ndcg_at_1_diff1": 0.482817, "nauc_ndcg_at_3_max": 0.419841, "nauc_ndcg_at_3_std": 0.065095, "nauc_ndcg_at_3_diff1": 0.454577, "nauc_ndcg_at_5_max": 0.433728, "nauc_ndcg_at_5_std": 0.085569, "nauc_ndcg_at_5_diff1": 0.456413, "nauc_ndcg_at_10_max": 0.43599, "nauc_ndcg_at_10_std": 0.086145, "nauc_ndcg_at_10_diff1": 0.446462, "nauc_ndcg_at_20_max": 0.447888, "nauc_ndcg_at_20_std": 0.111184, "nauc_ndcg_at_20_diff1": 0.436923, "nauc_ndcg_at_100_max": 0.445429, "nauc_ndcg_at_100_std": 0.125808, "nauc_ndcg_at_100_diff1": 0.428585, "nauc_ndcg_at_1000_max": 0.449401, "nauc_ndcg_at_1000_std": 0.123922, "nauc_ndcg_at_1000_diff1": 0.434085, "nauc_map_at_1_max": 0.39918, "nauc_map_at_1_std": 0.027622, "nauc_map_at_1_diff1": 0.482817, "nauc_map_at_3_max": 0.419223, "nauc_map_at_3_std": 0.058823, "nauc_map_at_3_diff1": 0.464053, "nauc_map_at_5_max": 0.426188, "nauc_map_at_5_std": 0.069444, "nauc_map_at_5_diff1": 0.465167, "nauc_map_at_10_max": 0.427027, "nauc_map_at_10_std": 0.069249, "nauc_map_at_10_diff1": 0.461081, "nauc_map_at_20_max": 0.430036, "nauc_map_at_20_std": 0.075798, "nauc_map_at_20_diff1": 0.458386, "nauc_map_at_100_max": 0.429814, "nauc_map_at_100_std": 0.077453, "nauc_map_at_100_diff1": 0.457465, "nauc_map_at_1000_max": 0.429917, "nauc_map_at_1000_std": 0.077293, "nauc_map_at_1000_diff1": 0.457679, "nauc_recall_at_1_max": 0.39918, "nauc_recall_at_1_std": 0.027622, "nauc_recall_at_1_diff1": 0.482817, "nauc_recall_at_3_max": 0.419875, "nauc_recall_at_3_std": 0.081604, "nauc_recall_at_3_diff1": 0.426973, "nauc_recall_at_5_max": 0.456315, "nauc_recall_at_5_std": 0.134105, "nauc_recall_at_5_diff1": 0.430654, "nauc_recall_at_10_max": 0.464821, "nauc_recall_at_10_std": 0.140302, "nauc_recall_at_10_diff1": 0.39827, "nauc_recall_at_20_max": 0.519446, "nauc_recall_at_20_std": 0.253255, "nauc_recall_at_20_diff1": 0.3549, "nauc_recall_at_100_max": 0.521197, "nauc_recall_at_100_std": 0.413396, "nauc_recall_at_100_diff1": 0.261415, "nauc_recall_at_1000_max": 0.766601, "nauc_recall_at_1000_std": 0.843741, "nauc_recall_at_1000_diff1": 0.137502, "nauc_precision_at_1_max": 0.39918, "nauc_precision_at_1_std": 0.027622, "nauc_precision_at_1_diff1": 0.482817, "nauc_precision_at_3_max": 0.419875, "nauc_precision_at_3_std": 0.081604, "nauc_precision_at_3_diff1": 0.426973, "nauc_precision_at_5_max": 0.456315, "nauc_precision_at_5_std": 0.134105, "nauc_precision_at_5_diff1": 0.430654, "nauc_precision_at_10_max": 0.464821, "nauc_precision_at_10_std": 0.140302, "nauc_precision_at_10_diff1": 0.39827, "nauc_precision_at_20_max": 0.519446, "nauc_precision_at_20_std": 0.253255, "nauc_precision_at_20_diff1": 0.3549, "nauc_precision_at_100_max": 0.521197, "nauc_precision_at_100_std": 0.413396, "nauc_precision_at_100_diff1": 0.261415, "nauc_precision_at_1000_max": 0.766601, "nauc_precision_at_1000_std": 0.843741, "nauc_precision_at_1000_diff1": 0.137502, "nauc_mrr_at_1_max": 0.39918, "nauc_mrr_at_1_std": 0.027622, "nauc_mrr_at_1_diff1": 0.482817, "nauc_mrr_at_3_max": 0.419223, "nauc_mrr_at_3_std": 0.058823, "nauc_mrr_at_3_diff1": 0.464053, "nauc_mrr_at_5_max": 0.426188, "nauc_mrr_at_5_std": 0.069444, "nauc_mrr_at_5_diff1": 0.465167, "nauc_mrr_at_10_max": 0.427027, "nauc_mrr_at_10_std": 0.069249, "nauc_mrr_at_10_diff1": 0.461081, "nauc_mrr_at_20_max": 0.430036, "nauc_mrr_at_20_std": 0.075798, "nauc_mrr_at_20_diff1": 0.458386, "nauc_mrr_at_100_max": 0.429814, "nauc_mrr_at_100_std": 0.077453, "nauc_mrr_at_100_diff1": 0.457465, "nauc_mrr_at_1000_max": 0.429917, "nauc_mrr_at_1000_std": 0.077293, "nauc_mrr_at_1000_diff1": 0.457679, "main_score": 0.40401, "hf_subset": "ko", "languages": [ "kor-Hang" ] } ], "test": [ { "ndcg_at_1": 0.275, "ndcg_at_3": 0.31958, "ndcg_at_5": 0.34024, "ndcg_at_10": 0.36877, "ndcg_at_20": 0.38634, "ndcg_at_100": 0.41038, "ndcg_at_1000": 0.43478, "map_at_1": 0.275, "map_at_3": 0.3075, "map_at_5": 0.319, "map_at_10": 0.33044, "map_at_20": 0.33519, "map_at_100": 0.33817, "map_at_1000": 0.33902, "recall_at_1": 0.275, "recall_at_3": 0.355, "recall_at_5": 0.405, "recall_at_10": 0.495, "recall_at_20": 0.565, "recall_at_100": 0.7, "recall_at_1000": 0.895, "precision_at_1": 0.275, "precision_at_3": 0.11833, "precision_at_5": 0.081, "precision_at_10": 0.0495, "precision_at_20": 0.02825, "precision_at_100": 0.007, "precision_at_1000": 0.0009, "mrr_at_1": 0.275, "mrr_at_3": 0.3075, "mrr_at_5": 0.319, "mrr_at_10": 0.330438, "mrr_at_20": 0.335192, "mrr_at_100": 0.338168, "mrr_at_1000": 0.339015, "nauc_ndcg_at_1_max": 0.59183, "nauc_ndcg_at_1_std": 0.147642, "nauc_ndcg_at_1_diff1": 0.599303, "nauc_ndcg_at_3_max": 0.598075, "nauc_ndcg_at_3_std": 0.151217, "nauc_ndcg_at_3_diff1": 0.555373, "nauc_ndcg_at_5_max": 0.60373, "nauc_ndcg_at_5_std": 0.156529, "nauc_ndcg_at_5_diff1": 0.575234, "nauc_ndcg_at_10_max": 0.610056, "nauc_ndcg_at_10_std": 0.179175, "nauc_ndcg_at_10_diff1": 0.542187, "nauc_ndcg_at_20_max": 0.61707, "nauc_ndcg_at_20_std": 0.201235, "nauc_ndcg_at_20_diff1": 0.542073, "nauc_ndcg_at_100_max": 0.622019, "nauc_ndcg_at_100_std": 0.204358, "nauc_ndcg_at_100_diff1": 0.542194, "nauc_ndcg_at_1000_max": 0.619199, "nauc_ndcg_at_1000_std": 0.201103, "nauc_ndcg_at_1000_diff1": 0.547707, "nauc_map_at_1_max": 0.59183, "nauc_map_at_1_std": 0.147642, "nauc_map_at_1_diff1": 0.599303, "nauc_map_at_3_max": 0.595925, "nauc_map_at_3_std": 0.150037, "nauc_map_at_3_diff1": 0.566575, "nauc_map_at_5_max": 0.599379, "nauc_map_at_5_std": 0.152998, "nauc_map_at_5_diff1": 0.577861, "nauc_map_at_10_max": 0.602238, "nauc_map_at_10_std": 0.161671, "nauc_map_at_10_diff1": 0.56457, "nauc_map_at_20_max": 0.603882, "nauc_map_at_20_std": 0.166928, "nauc_map_at_20_diff1": 0.564489, "nauc_map_at_100_max": 0.603692, "nauc_map_at_100_std": 0.167095, "nauc_map_at_100_diff1": 0.564556, "nauc_map_at_1000_max": 0.603511, "nauc_map_at_1000_std": 0.16704, "nauc_map_at_1000_diff1": 0.564661, "nauc_recall_at_1_max": 0.59183, "nauc_recall_at_1_std": 0.147642, "nauc_recall_at_1_diff1": 0.599303, "nauc_recall_at_3_max": 0.604326, "nauc_recall_at_3_std": 0.154633, "nauc_recall_at_3_diff1": 0.523863, "nauc_recall_at_5_max": 0.616376, "nauc_recall_at_5_std": 0.167102, "nauc_recall_at_5_diff1": 0.570192, "nauc_recall_at_10_max": 0.634644, "nauc_recall_at_10_std": 0.23984, "nauc_recall_at_10_diff1": 0.467283, "nauc_recall_at_20_max": 0.66675, "nauc_recall_at_20_std": 0.339937, "nauc_recall_at_20_diff1": 0.462338, "nauc_recall_at_100_max": 0.730166, "nauc_recall_at_100_std": 0.401171, "nauc_recall_at_100_diff1": 0.444388, "nauc_recall_at_1000_max": 0.834146, "nauc_recall_at_1000_std": 0.591383, "nauc_recall_at_1000_diff1": 0.44743, "nauc_precision_at_1_max": 0.59183, "nauc_precision_at_1_std": 0.147642, "nauc_precision_at_1_diff1": 0.599303, "nauc_precision_at_3_max": 0.604326, "nauc_precision_at_3_std": 0.154633, "nauc_precision_at_3_diff1": 0.523863, "nauc_precision_at_5_max": 0.616376, "nauc_precision_at_5_std": 0.167102, "nauc_precision_at_5_diff1": 0.570192, "nauc_precision_at_10_max": 0.634644, "nauc_precision_at_10_std": 0.23984, "nauc_precision_at_10_diff1": 0.467283, "nauc_precision_at_20_max": 0.66675, "nauc_precision_at_20_std": 0.339937, "nauc_precision_at_20_diff1": 0.462338, "nauc_precision_at_100_max": 0.730166, "nauc_precision_at_100_std": 0.401171, "nauc_precision_at_100_diff1": 0.444388, "nauc_precision_at_1000_max": 0.834146, "nauc_precision_at_1000_std": 0.591383, "nauc_precision_at_1000_diff1": 0.44743, "nauc_mrr_at_1_max": 0.59183, "nauc_mrr_at_1_std": 0.147642, "nauc_mrr_at_1_diff1": 0.599303, "nauc_mrr_at_3_max": 0.595925, "nauc_mrr_at_3_std": 0.150037, "nauc_mrr_at_3_diff1": 0.566575, "nauc_mrr_at_5_max": 0.599379, "nauc_mrr_at_5_std": 0.152998, "nauc_mrr_at_5_diff1": 0.577861, "nauc_mrr_at_10_max": 0.602238, "nauc_mrr_at_10_std": 0.161671, "nauc_mrr_at_10_diff1": 0.56457, "nauc_mrr_at_20_max": 0.603882, "nauc_mrr_at_20_std": 0.166928, "nauc_mrr_at_20_diff1": 0.564489, "nauc_mrr_at_100_max": 0.603692, "nauc_mrr_at_100_std": 0.167095, "nauc_mrr_at_100_diff1": 0.564556, "nauc_mrr_at_1000_max": 0.603511, "nauc_mrr_at_1000_std": 0.16704, "nauc_mrr_at_1000_diff1": 0.564661, "main_score": 0.36877, "hf_subset": "ko", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 11602.840583086014, "kg_co2_emissions": null } ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "task_name": "PublicHealthQA", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.67532, "ndcg_at_3": 0.78324, "ndcg_at_5": 0.79888, "ndcg_at_10": 0.81679, "ndcg_at_20": 0.82679, "ndcg_at_100": 0.82963, "ndcg_at_1000": 0.82963, "map_at_1": 0.67532, "map_at_3": 0.75758, "map_at_5": 0.76602, "map_at_10": 0.77406, "map_at_20": 0.77689, "map_at_100": 0.77745, "map_at_1000": 0.77745, "recall_at_1": 0.67532, "recall_at_3": 0.85714, "recall_at_5": 0.8961, "recall_at_10": 0.94805, "recall_at_20": 0.98701, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.67532, "precision_at_3": 0.28571, "precision_at_5": 0.17922, "precision_at_10": 0.09481, "precision_at_20": 0.04935, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.675325, "mrr_at_3": 0.757576, "mrr_at_5": 0.766017, "mrr_at_10": 0.774057, "mrr_at_20": 0.776887, "mrr_at_100": 0.777451, "mrr_at_1000": 0.777451, "nauc_ndcg_at_1_max": 0.275234, "nauc_ndcg_at_1_std": -0.024039, "nauc_ndcg_at_1_diff1": 0.7852, "nauc_ndcg_at_3_max": 0.238945, "nauc_ndcg_at_3_std": -0.129105, "nauc_ndcg_at_3_diff1": 0.725551, "nauc_ndcg_at_5_max": 0.260064, "nauc_ndcg_at_5_std": -0.14163, "nauc_ndcg_at_5_diff1": 0.764278, "nauc_ndcg_at_10_max": 0.229219, "nauc_ndcg_at_10_std": -0.138331, "nauc_ndcg_at_10_diff1": 0.766539, "nauc_ndcg_at_20_max": 0.221126, "nauc_ndcg_at_20_std": -0.109715, "nauc_ndcg_at_20_diff1": 0.756675, "nauc_ndcg_at_100_max": 0.247464, "nauc_ndcg_at_100_std": -0.101146, "nauc_ndcg_at_100_diff1": 0.757323, "nauc_ndcg_at_1000_max": 0.247464, "nauc_ndcg_at_1000_std": -0.101146, "nauc_ndcg_at_1000_diff1": 0.757323, "nauc_map_at_1_max": 0.275234, "nauc_map_at_1_std": -0.024039, "nauc_map_at_1_diff1": 0.7852, "nauc_map_at_3_max": 0.253713, "nauc_map_at_3_std": -0.099555, "nauc_map_at_3_diff1": 0.742159, "nauc_map_at_5_max": 0.263202, "nauc_map_at_5_std": -0.104581, "nauc_map_at_5_diff1": 0.760417, "nauc_map_at_10_max": 0.251371, "nauc_map_at_10_std": -0.102456, "nauc_map_at_10_diff1": 0.760742, "nauc_map_at_20_max": 0.249733, "nauc_map_at_20_std": -0.093969, "nauc_map_at_20_diff1": 0.758372, "nauc_map_at_100_max": 0.253853, "nauc_map_at_100_std": -0.09261, "nauc_map_at_100_diff1": 0.758476, "nauc_map_at_1000_max": 0.253853, "nauc_map_at_1000_std": -0.09261, "nauc_map_at_1000_diff1": 0.758476, "nauc_recall_at_1_max": 0.275234, "nauc_recall_at_1_std": -0.024039, "nauc_recall_at_1_diff1": 0.7852, "nauc_recall_at_3_max": 0.170913, "nauc_recall_at_3_std": -0.258852, "nauc_recall_at_3_diff1": 0.652751, "nauc_recall_at_5_max": 0.248113, "nauc_recall_at_5_std": -0.353764, "nauc_recall_at_5_diff1": 0.801157, "nauc_recall_at_10_max": -0.031491, "nauc_recall_at_10_std": -0.524913, "nauc_recall_at_10_diff1": 0.865007, "nauc_recall_at_20_max": -1.164365, "nauc_recall_at_20_std": -0.560444, "nauc_recall_at_20_diff1": 0.722603, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.275234, "nauc_precision_at_1_std": -0.024039, "nauc_precision_at_1_diff1": 0.7852, "nauc_precision_at_3_max": 0.170913, "nauc_precision_at_3_std": -0.258852, "nauc_precision_at_3_diff1": 0.652751, "nauc_precision_at_5_max": 0.248113, "nauc_precision_at_5_std": -0.353764, "nauc_precision_at_5_diff1": 0.801157, "nauc_precision_at_10_max": -0.031491, "nauc_precision_at_10_std": -0.524913, "nauc_precision_at_10_diff1": 0.865007, "nauc_precision_at_20_max": -1.164365, "nauc_precision_at_20_std": -0.560444, "nauc_precision_at_20_diff1": 0.722603, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.275234, "nauc_mrr_at_1_std": -0.024039, "nauc_mrr_at_1_diff1": 0.7852, "nauc_mrr_at_3_max": 0.253713, "nauc_mrr_at_3_std": -0.099555, "nauc_mrr_at_3_diff1": 0.742159, "nauc_mrr_at_5_max": 0.263202, "nauc_mrr_at_5_std": -0.104581, "nauc_mrr_at_5_diff1": 0.760417, "nauc_mrr_at_10_max": 0.251371, "nauc_mrr_at_10_std": -0.102456, "nauc_mrr_at_10_diff1": 0.760742, "nauc_mrr_at_20_max": 0.249733, "nauc_mrr_at_20_std": -0.093969, "nauc_mrr_at_20_diff1": 0.758372, "nauc_mrr_at_100_max": 0.253853, "nauc_mrr_at_100_std": -0.09261, "nauc_mrr_at_100_diff1": 0.758476, "nauc_mrr_at_1000_max": 0.253853, "nauc_mrr_at_1000_std": -0.09261, "nauc_mrr_at_1000_diff1": 0.758476, "main_score": 0.81679, "hf_subset": "korean", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 2.0190324783325195, "kg_co2_emissions": null } ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "task_name": "XPQARetrieval", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.34098, "ndcg_at_3": 0.37001, "ndcg_at_5": 0.39582, "ndcg_at_10": 0.43018, "ndcg_at_20": 0.45717, "ndcg_at_100": 0.49835, "ndcg_at_1000": 0.5198, "map_at_1": 0.24118, "map_at_3": 0.3399, "map_at_5": 0.36301, "map_at_10": 0.3793, "map_at_20": 0.38769, "map_at_100": 0.39471, "map_at_1000": 0.39563, "recall_at_1": 0.24118, "recall_at_3": 0.38856, "recall_at_5": 0.45178, "recall_at_10": 0.54684, "recall_at_20": 0.64126, "recall_at_100": 0.83856, "recall_at_1000": 1.0, "precision_at_1": 0.34098, "precision_at_3": 0.20897, "precision_at_5": 0.14893, "precision_at_10": 0.08945, "precision_at_20": 0.05252, "precision_at_100": 0.01373, "precision_at_1000": 0.00156, "mrr_at_1": 0.340979, "mrr_at_3": 0.390163, "mrr_at_5": 0.401784, "mrr_at_10": 0.414107, "mrr_at_20": 0.420741, "mrr_at_100": 0.424817, "mrr_at_1000": 0.425472, "nauc_ndcg_at_1_max": 0.371938, "nauc_ndcg_at_1_std": -0.223132, "nauc_ndcg_at_1_diff1": 0.498818, "nauc_ndcg_at_3_max": 0.342361, "nauc_ndcg_at_3_std": -0.223941, "nauc_ndcg_at_3_diff1": 0.440807, "nauc_ndcg_at_5_max": 0.323188, "nauc_ndcg_at_5_std": -0.236331, "nauc_ndcg_at_5_diff1": 0.431617, "nauc_ndcg_at_10_max": 0.31163, "nauc_ndcg_at_10_std": -0.234533, "nauc_ndcg_at_10_diff1": 0.422442, "nauc_ndcg_at_20_max": 0.307969, "nauc_ndcg_at_20_std": -0.23559, "nauc_ndcg_at_20_diff1": 0.413812, "nauc_ndcg_at_100_max": 0.321761, "nauc_ndcg_at_100_std": -0.213297, "nauc_ndcg_at_100_diff1": 0.412137, "nauc_ndcg_at_1000_max": 0.327573, "nauc_ndcg_at_1000_std": -0.21945, "nauc_ndcg_at_1000_diff1": 0.42314, "nauc_map_at_1_max": 0.285472, "nauc_map_at_1_std": -0.215826, "nauc_map_at_1_diff1": 0.551963, "nauc_map_at_3_max": 0.34034, "nauc_map_at_3_std": -0.21525, "nauc_map_at_3_diff1": 0.449593, "nauc_map_at_5_max": 0.332048, "nauc_map_at_5_std": -0.228033, "nauc_map_at_5_diff1": 0.441525, "nauc_map_at_10_max": 0.326302, "nauc_map_at_10_std": -0.229127, "nauc_map_at_10_diff1": 0.43694, "nauc_map_at_20_max": 0.325678, "nauc_map_at_20_std": -0.229578, "nauc_map_at_20_diff1": 0.434603, "nauc_map_at_100_max": 0.327011, "nauc_map_at_100_std": -0.226667, "nauc_map_at_100_diff1": 0.433772, "nauc_map_at_1000_max": 0.327509, "nauc_map_at_1000_std": -0.226583, "nauc_map_at_1000_diff1": 0.434253, "nauc_recall_at_1_max": 0.285472, "nauc_recall_at_1_std": -0.215826, "nauc_recall_at_1_diff1": 0.551963, "nauc_recall_at_3_max": 0.32072, "nauc_recall_at_3_std": -0.221916, "nauc_recall_at_3_diff1": 0.412067, "nauc_recall_at_5_max": 0.280654, "nauc_recall_at_5_std": -0.246186, "nauc_recall_at_5_diff1": 0.381538, "nauc_recall_at_10_max": 0.244628, "nauc_recall_at_10_std": -0.233764, "nauc_recall_at_10_diff1": 0.352702, "nauc_recall_at_20_max": 0.214972, "nauc_recall_at_20_std": -0.240123, "nauc_recall_at_20_diff1": 0.30565, "nauc_recall_at_100_max": 0.260948, "nauc_recall_at_100_std": -0.082979, "nauc_recall_at_100_diff1": 0.243887, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.371938, "nauc_precision_at_1_std": -0.223132, "nauc_precision_at_1_diff1": 0.498818, "nauc_precision_at_3_max": 0.337407, "nauc_precision_at_3_std": -0.162082, "nauc_precision_at_3_diff1": 0.218464, "nauc_precision_at_5_max": 0.280191, "nauc_precision_at_5_std": -0.174784, "nauc_precision_at_5_diff1": 0.178532, "nauc_precision_at_10_max": 0.225539, "nauc_precision_at_10_std": -0.159273, "nauc_precision_at_10_diff1": 0.125716, "nauc_precision_at_20_max": 0.193822, "nauc_precision_at_20_std": -0.139024, "nauc_precision_at_20_diff1": 0.068406, "nauc_precision_at_100_max": 0.194409, "nauc_precision_at_100_std": 0.026692, "nauc_precision_at_100_diff1": -0.049554, "nauc_precision_at_1000_max": 0.183445, "nauc_precision_at_1000_std": 0.077361, "nauc_precision_at_1000_diff1": -0.123717, "nauc_mrr_at_1_max": 0.371938, "nauc_mrr_at_1_std": -0.223132, "nauc_mrr_at_1_diff1": 0.498818, "nauc_mrr_at_3_max": 0.353543, "nauc_mrr_at_3_std": -0.233109, "nauc_mrr_at_3_diff1": 0.467199, "nauc_mrr_at_5_max": 0.343539, "nauc_mrr_at_5_std": -0.232805, "nauc_mrr_at_5_diff1": 0.45681, "nauc_mrr_at_10_max": 0.339967, "nauc_mrr_at_10_std": -0.231545, "nauc_mrr_at_10_diff1": 0.453967, "nauc_mrr_at_20_max": 0.339934, "nauc_mrr_at_20_std": -0.230488, "nauc_mrr_at_20_diff1": 0.452631, "nauc_mrr_at_100_max": 0.341445, "nauc_mrr_at_100_std": -0.228595, "nauc_mrr_at_100_diff1": 0.45271, "nauc_mrr_at_1000_max": 0.341512, "nauc_mrr_at_1000_std": -0.228865, "nauc_mrr_at_1000_diff1": 0.453134, "main_score": 0.43018, "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ] }, { "ndcg_at_1": 0.3211, "ndcg_at_3": 0.32729, "ndcg_at_5": 0.34032, "ndcg_at_10": 0.37783, "ndcg_at_20": 0.40662, "ndcg_at_100": 0.45834, "ndcg_at_1000": 0.48616, "map_at_1": 0.18524, "map_at_3": 0.27607, "map_at_5": 0.30003, "map_at_10": 0.32097, "map_at_20": 0.33112, "map_at_100": 0.33984, "map_at_1000": 0.34145, "recall_at_1": 0.18524, "recall_at_3": 0.32011, "recall_at_5": 0.37513, "recall_at_10": 0.47146, "recall_at_20": 0.56577, "recall_at_100": 0.80892, "recall_at_1000": 0.99796, "precision_at_1": 0.3211, "precision_at_3": 0.21458, "precision_at_5": 0.15841, "precision_at_10": 0.09924, "precision_at_20": 0.05917, "precision_at_100": 0.01633, "precision_at_1000": 0.00195, "mrr_at_1": 0.321101, "mrr_at_3": 0.364934, "mrr_at_5": 0.378313, "mrr_at_10": 0.389471, "mrr_at_20": 0.395791, "mrr_at_100": 0.400832, "mrr_at_1000": 0.401506, "nauc_ndcg_at_1_max": 0.274415, "nauc_ndcg_at_1_std": -0.192429, "nauc_ndcg_at_1_diff1": 0.498666, "nauc_ndcg_at_3_max": 0.2414, "nauc_ndcg_at_3_std": -0.216653, "nauc_ndcg_at_3_diff1": 0.459131, "nauc_ndcg_at_5_max": 0.228722, "nauc_ndcg_at_5_std": -0.223779, "nauc_ndcg_at_5_diff1": 0.447031, "nauc_ndcg_at_10_max": 0.214371, "nauc_ndcg_at_10_std": -0.238081, "nauc_ndcg_at_10_diff1": 0.437757, "nauc_ndcg_at_20_max": 0.200516, "nauc_ndcg_at_20_std": -0.243786, "nauc_ndcg_at_20_diff1": 0.433649, "nauc_ndcg_at_100_max": 0.219774, "nauc_ndcg_at_100_std": -0.211152, "nauc_ndcg_at_100_diff1": 0.440395, "nauc_ndcg_at_1000_max": 0.231401, "nauc_ndcg_at_1000_std": -0.207592, "nauc_ndcg_at_1000_diff1": 0.439766, "nauc_map_at_1_max": 0.194908, "nauc_map_at_1_std": -0.201311, "nauc_map_at_1_diff1": 0.526747, "nauc_map_at_3_max": 0.234864, "nauc_map_at_3_std": -0.214969, "nauc_map_at_3_diff1": 0.480137, "nauc_map_at_5_max": 0.238417, "nauc_map_at_5_std": -0.218501, "nauc_map_at_5_diff1": 0.459737, "nauc_map_at_10_max": 0.234651, "nauc_map_at_10_std": -0.226327, "nauc_map_at_10_diff1": 0.451325, "nauc_map_at_20_max": 0.229985, "nauc_map_at_20_std": -0.228496, "nauc_map_at_20_diff1": 0.450564, "nauc_map_at_100_max": 0.231539, "nauc_map_at_100_std": -0.224153, "nauc_map_at_100_diff1": 0.450247, "nauc_map_at_1000_max": 0.232752, "nauc_map_at_1000_std": -0.223034, "nauc_map_at_1000_diff1": 0.45016, "nauc_recall_at_1_max": 0.194908, "nauc_recall_at_1_std": -0.201311, "nauc_recall_at_1_diff1": 0.526747, "nauc_recall_at_3_max": 0.190339, "nauc_recall_at_3_std": -0.23048, "nauc_recall_at_3_diff1": 0.440644, "nauc_recall_at_5_max": 0.185054, "nauc_recall_at_5_std": -0.235978, "nauc_recall_at_5_diff1": 0.397687, "nauc_recall_at_10_max": 0.14123, "nauc_recall_at_10_std": -0.271337, "nauc_recall_at_10_diff1": 0.376031, "nauc_recall_at_20_max": 0.085163, "nauc_recall_at_20_std": -0.296573, "nauc_recall_at_20_diff1": 0.351952, "nauc_recall_at_100_max": 0.139824, "nauc_recall_at_100_std": -0.129667, "nauc_recall_at_100_diff1": 0.384461, "nauc_recall_at_1000_max": 0.707255, "nauc_recall_at_1000_std": 0.400807, "nauc_recall_at_1000_diff1": -0.343666, "nauc_precision_at_1_max": 0.274415, "nauc_precision_at_1_std": -0.192429, "nauc_precision_at_1_diff1": 0.498666, "nauc_precision_at_3_max": 0.257978, "nauc_precision_at_3_std": -0.170719, "nauc_precision_at_3_diff1": 0.310497, "nauc_precision_at_5_max": 0.232798, "nauc_precision_at_5_std": -0.157916, "nauc_precision_at_5_diff1": 0.241854, "nauc_precision_at_10_max": 0.188433, "nauc_precision_at_10_std": -0.163574, "nauc_precision_at_10_diff1": 0.191728, "nauc_precision_at_20_max": 0.142616, "nauc_precision_at_20_std": -0.150251, "nauc_precision_at_20_diff1": 0.158454, "nauc_precision_at_100_max": 0.159918, "nauc_precision_at_100_std": 0.018437, "nauc_precision_at_100_diff1": 0.066529, "nauc_precision_at_1000_max": 0.170039, "nauc_precision_at_1000_std": 0.101311, "nauc_precision_at_1000_diff1": -0.027352, "nauc_mrr_at_1_max": 0.274415, "nauc_mrr_at_1_std": -0.192429, "nauc_mrr_at_1_diff1": 0.498666, "nauc_mrr_at_3_max": 0.246087, "nauc_mrr_at_3_std": -0.212935, "nauc_mrr_at_3_diff1": 0.471176, "nauc_mrr_at_5_max": 0.243808, "nauc_mrr_at_5_std": -0.210646, "nauc_mrr_at_5_diff1": 0.46394, "nauc_mrr_at_10_max": 0.238075, "nauc_mrr_at_10_std": -0.212904, "nauc_mrr_at_10_diff1": 0.461785, "nauc_mrr_at_20_max": 0.235441, "nauc_mrr_at_20_std": -0.213807, "nauc_mrr_at_20_diff1": 0.460976, "nauc_mrr_at_100_max": 0.23882, "nauc_mrr_at_100_std": -0.209904, "nauc_mrr_at_100_diff1": 0.46242, "nauc_mrr_at_1000_max": 0.239092, "nauc_mrr_at_1000_std": -0.209955, "nauc_mrr_at_1000_diff1": 0.462475, "main_score": 0.37783, "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ] }, { "ndcg_at_1": 0.31107, "ndcg_at_3": 0.33633, "ndcg_at_5": 0.36534, "ndcg_at_10": 0.39146, "ndcg_at_20": 0.42104, "ndcg_at_100": 0.46851, "ndcg_at_1000": 0.49445, "map_at_1": 0.21633, "map_at_3": 0.30453, "map_at_5": 0.3291, "map_at_10": 0.3436, "map_at_20": 0.35342, "map_at_100": 0.36168, "map_at_1000": 0.36292, "recall_at_1": 0.21633, "recall_at_3": 0.34737, "recall_at_5": 0.41921, "recall_at_10": 0.48758, "recall_at_20": 0.59012, "recall_at_100": 0.8113, "recall_at_1000": 1.0, "precision_at_1": 0.31107, "precision_at_3": 0.19598, "precision_at_5": 0.14332, "precision_at_10": 0.0855, "precision_at_20": 0.05147, "precision_at_100": 0.01417, "precision_at_1000": 0.00167, "mrr_at_1": 0.311075, "mrr_at_3": 0.360206, "mrr_at_5": 0.376004, "mrr_at_10": 0.384515, "mrr_at_20": 0.391031, "mrr_at_100": 0.396106, "mrr_at_1000": 0.396896, "nauc_ndcg_at_1_max": 0.235083, "nauc_ndcg_at_1_std": -0.247901, "nauc_ndcg_at_1_diff1": 0.521614, "nauc_ndcg_at_3_max": 0.208812, "nauc_ndcg_at_3_std": -0.238682, "nauc_ndcg_at_3_diff1": 0.466576, "nauc_ndcg_at_5_max": 0.198242, "nauc_ndcg_at_5_std": -0.244, "nauc_ndcg_at_5_diff1": 0.452791, "nauc_ndcg_at_10_max": 0.198586, "nauc_ndcg_at_10_std": -0.246105, "nauc_ndcg_at_10_diff1": 0.434085, "nauc_ndcg_at_20_max": 0.180033, "nauc_ndcg_at_20_std": -0.25126, "nauc_ndcg_at_20_diff1": 0.424635, "nauc_ndcg_at_100_max": 0.202984, "nauc_ndcg_at_100_std": -0.217858, "nauc_ndcg_at_100_diff1": 0.433032, "nauc_ndcg_at_1000_max": 0.207098, "nauc_ndcg_at_1000_std": -0.230045, "nauc_ndcg_at_1000_diff1": 0.443934, "nauc_map_at_1_max": 0.180573, "nauc_map_at_1_std": -0.20972, "nauc_map_at_1_diff1": 0.574387, "nauc_map_at_3_max": 0.212718, "nauc_map_at_3_std": -0.225802, "nauc_map_at_3_diff1": 0.481218, "nauc_map_at_5_max": 0.203508, "nauc_map_at_5_std": -0.237369, "nauc_map_at_5_diff1": 0.466685, "nauc_map_at_10_max": 0.199659, "nauc_map_at_10_std": -0.244784, "nauc_map_at_10_diff1": 0.454792, "nauc_map_at_20_max": 0.194065, "nauc_map_at_20_std": -0.2454, "nauc_map_at_20_diff1": 0.450977, "nauc_map_at_100_max": 0.197705, "nauc_map_at_100_std": -0.240541, "nauc_map_at_100_diff1": 0.452898, "nauc_map_at_1000_max": 0.198274, "nauc_map_at_1000_std": -0.240633, "nauc_map_at_1000_diff1": 0.45334, "nauc_recall_at_1_max": 0.180573, "nauc_recall_at_1_std": -0.20972, "nauc_recall_at_1_diff1": 0.574387, "nauc_recall_at_3_max": 0.200321, "nauc_recall_at_3_std": -0.217471, "nauc_recall_at_3_diff1": 0.440816, "nauc_recall_at_5_max": 0.172545, "nauc_recall_at_5_std": -0.23075, "nauc_recall_at_5_diff1": 0.394698, "nauc_recall_at_10_max": 0.173802, "nauc_recall_at_10_std": -0.230833, "nauc_recall_at_10_diff1": 0.342878, "nauc_recall_at_20_max": 0.093338, "nauc_recall_at_20_std": -0.255665, "nauc_recall_at_20_diff1": 0.29516, "nauc_recall_at_100_max": 0.179841, "nauc_recall_at_100_std": -0.023651, "nauc_recall_at_100_diff1": 0.273318, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.235083, "nauc_precision_at_1_std": -0.247901, "nauc_precision_at_1_diff1": 0.521614, "nauc_precision_at_3_max": 0.19542, "nauc_precision_at_3_std": -0.211587, "nauc_precision_at_3_diff1": 0.248417, "nauc_precision_at_5_max": 0.144452, "nauc_precision_at_5_std": -0.22976, "nauc_precision_at_5_diff1": 0.192698, "nauc_precision_at_10_max": 0.122354, "nauc_precision_at_10_std": -0.217942, "nauc_precision_at_10_diff1": 0.134457, "nauc_precision_at_20_max": 0.07614, "nauc_precision_at_20_std": -0.19295, "nauc_precision_at_20_diff1": 0.084722, "nauc_precision_at_100_max": 0.146839, "nauc_precision_at_100_std": 0.009803, "nauc_precision_at_100_diff1": 0.006768, "nauc_precision_at_1000_max": 0.15342, "nauc_precision_at_1000_std": 0.044927, "nauc_precision_at_1000_diff1": -0.068738, "nauc_mrr_at_1_max": 0.235083, "nauc_mrr_at_1_std": -0.247901, "nauc_mrr_at_1_diff1": 0.521614, "nauc_mrr_at_3_max": 0.223592, "nauc_mrr_at_3_std": -0.250996, "nauc_mrr_at_3_diff1": 0.48789, "nauc_mrr_at_5_max": 0.218129, "nauc_mrr_at_5_std": -0.251186, "nauc_mrr_at_5_diff1": 0.481346, "nauc_mrr_at_10_max": 0.221502, "nauc_mrr_at_10_std": -0.246732, "nauc_mrr_at_10_diff1": 0.476944, "nauc_mrr_at_20_max": 0.217876, "nauc_mrr_at_20_std": -0.249077, "nauc_mrr_at_20_diff1": 0.476331, "nauc_mrr_at_100_max": 0.219187, "nauc_mrr_at_100_std": -0.246605, "nauc_mrr_at_100_diff1": 0.476969, "nauc_mrr_at_1000_max": 0.219149, "nauc_mrr_at_1000_std": -0.247012, "nauc_mrr_at_1000_diff1": 0.477279, "main_score": 0.39146, "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ] } ] }, "evaluation_time": 14.93372392654419, "kg_co2_emissions": null } ================================================ FILE: eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/model_meta.json ================================================ {"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "adapted_from": null, "superseded_by": null, "loader": null} ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 22.999385833740234, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.87379, "map_at_1": 0.75439, "map_at_10": 0.83841, "map_at_100": 0.83957, "map_at_1000": 0.83957, "map_at_20": 0.83957, "map_at_3": 0.82602, "map_at_5": 0.83216, "mrr_at_1": 0.7543859649122807, "mrr_at_10": 0.8384120022277919, "mrr_at_100": 0.8395740954951483, "mrr_at_1000": 0.8395740954951483, "mrr_at_20": 0.8395740954951483, "mrr_at_3": 0.8260233918128655, "mrr_at_5": 0.8321637426900585, "nauc_map_at_1000_diff1": 0.7553580826697276, "nauc_map_at_1000_max": 0.15569810026809622, "nauc_map_at_1000_std": -0.691306901301535, "nauc_map_at_100_diff1": 0.7553580826697276, "nauc_map_at_100_max": 0.15569810026809622, "nauc_map_at_100_std": -0.691306901301535, "nauc_map_at_10_diff1": 0.7543342191950719, "nauc_map_at_10_max": 0.15938639419130274, "nauc_map_at_10_std": -0.6934863935974837, "nauc_map_at_1_diff1": 0.7493933183304148, "nauc_map_at_1_max": 0.06264615667212535, "nauc_map_at_1_std": -0.6420183946779819, "nauc_map_at_20_diff1": 0.7553580826697276, "nauc_map_at_20_max": 0.15569810026809622, "nauc_map_at_20_std": -0.691306901301535, "nauc_map_at_3_diff1": 0.7616179634532826, "nauc_map_at_3_max": 0.1681637034361113, "nauc_map_at_3_std": -0.7226590901715979, "nauc_map_at_5_diff1": 0.7610831361890733, "nauc_map_at_5_max": 0.1768332483014387, "nauc_map_at_5_std": -0.6970842647826092, "nauc_mrr_at_1000_diff1": 0.7553580826697276, "nauc_mrr_at_1000_max": 0.15569810026809622, "nauc_mrr_at_1000_std": -0.691306901301535, "nauc_mrr_at_100_diff1": 0.7553580826697276, "nauc_mrr_at_100_max": 0.15569810026809622, "nauc_mrr_at_100_std": -0.691306901301535, "nauc_mrr_at_10_diff1": 0.7543342191950719, "nauc_mrr_at_10_max": 0.15938639419130274, "nauc_mrr_at_10_std": -0.6934863935974837, "nauc_mrr_at_1_diff1": 0.7493933183304148, "nauc_mrr_at_1_max": 0.06264615667212535, "nauc_mrr_at_1_std": -0.6420183946779819, "nauc_mrr_at_20_diff1": 0.7553580826697276, "nauc_mrr_at_20_max": 0.15569810026809622, "nauc_mrr_at_20_std": -0.691306901301535, "nauc_mrr_at_3_diff1": 0.7616179634532826, "nauc_mrr_at_3_max": 0.1681637034361113, "nauc_mrr_at_3_std": -0.7226590901715979, "nauc_mrr_at_5_diff1": 0.7610831361890733, "nauc_mrr_at_5_max": 0.1768332483014387, "nauc_mrr_at_5_std": -0.6970842647826092, "nauc_ndcg_at_1000_diff1": 0.7560582117239595, "nauc_ndcg_at_1000_max": 0.165803773315689, "nauc_ndcg_at_1000_std": -0.696811115693176, "nauc_ndcg_at_100_diff1": 0.7560582117239595, "nauc_ndcg_at_100_max": 0.165803773315689, "nauc_ndcg_at_100_std": -0.696811115693176, "nauc_ndcg_at_10_diff1": 0.751463249238206, "nauc_ndcg_at_10_max": 0.18342906202141002, "nauc_ndcg_at_10_std": -0.703275491144023, "nauc_ndcg_at_1_diff1": 0.7493933183304148, "nauc_ndcg_at_1_max": 0.06264615667212535, "nauc_ndcg_at_1_std": -0.6420183946779819, "nauc_ndcg_at_20_diff1": 0.7560582117239595, "nauc_ndcg_at_20_max": 0.165803773315689, "nauc_ndcg_at_20_std": -0.696811115693176, "nauc_ndcg_at_3_diff1": 0.7704450250564626, "nauc_ndcg_at_3_max": 0.20710284523661493, "nauc_ndcg_at_3_std": -0.7680242628597699, "nauc_ndcg_at_5_diff1": 0.7702091882328883, "nauc_ndcg_at_5_max": 0.2274499137326895, "nauc_ndcg_at_5_std": -0.7186386105526826, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.638583133029159, "nauc_precision_at_10_max": 0.638583133029159, "nauc_precision_at_10_std": -0.8171511695270405, "nauc_precision_at_1_diff1": 0.7493933183304148, "nauc_precision_at_1_max": 0.06264615667212535, "nauc_precision_at_1_std": -0.6420183946779819, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_3_diff1": 0.8183963637732921, "nauc_precision_at_3_max": 0.40576075757101193, "nauc_precision_at_3_std": -1.010017397627182, "nauc_precision_at_5_diff1": 0.8384802315250225, "nauc_precision_at_5_max": 0.5892262094609966, "nauc_precision_at_5_std": -0.864259619975316, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.6385831330291614, "nauc_recall_at_10_max": 0.6385831330291614, "nauc_recall_at_10_std": -0.8171511695270118, "nauc_recall_at_1_diff1": 0.7493933183304148, "nauc_recall_at_1_max": 0.06264615667212535, "nauc_recall_at_1_std": -0.6420183946779819, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_3_diff1": 0.8183963637732907, "nauc_recall_at_3_max": 0.4057607575710131, "nauc_recall_at_3_std": -1.0100173976271816, "nauc_recall_at_5_diff1": 0.8384802315250196, "nauc_recall_at_5_max": 0.589226209460994, "nauc_recall_at_5_std": -0.8642596199753173, "ndcg_at_1": 0.75439, "ndcg_at_10": 0.87379, "ndcg_at_100": 0.87816, "ndcg_at_1000": 0.87816, "ndcg_at_20": 0.87816, "ndcg_at_3": 0.84826, "ndcg_at_5": 0.85921, "precision_at_1": 0.75439, "precision_at_10": 0.09825, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.05, "precision_at_3": 0.30409, "precision_at_5": 0.18772, "recall_at_1": 0.75439, "recall_at_10": 0.98246, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 1.0, "recall_at_3": 0.91228, "recall_at_5": 0.9386 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 20.49944758415222, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.95027, "map_at_1": 0.90889, "map_at_10": 0.93864, "map_at_100": 0.93898, "map_at_1000": 0.939, "map_at_20": 0.93884, "map_at_3": 0.935, "map_at_5": 0.93744, "mrr_at_1": 0.9088888888888889, "mrr_at_10": 0.9386366843033511, "mrr_at_100": 0.9389811252026834, "mrr_at_1000": 0.938999955232359, "mrr_at_20": 0.9388439733593242, "mrr_at_3": 0.9350000000000003, "mrr_at_5": 0.9374444444444445, "nauc_map_at_1000_diff1": 0.926260018196481, "nauc_map_at_1000_max": 0.8410999356826026, "nauc_map_at_1000_std": -0.09167429373268775, "nauc_map_at_100_diff1": 0.9262622706953337, "nauc_map_at_100_max": 0.8411394879430902, "nauc_map_at_100_std": -0.09142238699159651, "nauc_map_at_10_diff1": 0.9260798552935571, "nauc_map_at_10_max": 0.8405924657296209, "nauc_map_at_10_std": -0.09365617696316625, "nauc_map_at_1_diff1": 0.9282924551934596, "nauc_map_at_1_max": 0.8302418528386962, "nauc_map_at_1_std": -0.09564232196943509, "nauc_map_at_20_diff1": 0.9263382226923206, "nauc_map_at_20_max": 0.8414087434785231, "nauc_map_at_20_std": -0.09255316406161156, "nauc_map_at_3_diff1": 0.9257955793903508, "nauc_map_at_3_max": 0.841332354404249, "nauc_map_at_3_std": -0.10062486533074633, "nauc_map_at_5_diff1": 0.9262936870031215, "nauc_map_at_5_max": 0.8423775470322344, "nauc_map_at_5_std": -0.09012841371006793, "nauc_mrr_at_1000_diff1": 0.926260018196481, "nauc_mrr_at_1000_max": 0.8410999356826026, "nauc_mrr_at_1000_std": -0.09167429373268775, "nauc_mrr_at_100_diff1": 0.9262622706953337, "nauc_mrr_at_100_max": 0.8411394879430902, "nauc_mrr_at_100_std": -0.09142238699159651, "nauc_mrr_at_10_diff1": 0.9260798552935571, "nauc_mrr_at_10_max": 0.8405924657296209, "nauc_mrr_at_10_std": -0.09365617696316625, "nauc_mrr_at_1_diff1": 0.9282924551934596, "nauc_mrr_at_1_max": 0.8302418528386962, "nauc_mrr_at_1_std": -0.09564232196943509, "nauc_mrr_at_20_diff1": 0.9263382226923206, "nauc_mrr_at_20_max": 0.8414087434785231, "nauc_mrr_at_20_std": -0.09255316406161156, "nauc_mrr_at_3_diff1": 0.9257955793903508, "nauc_mrr_at_3_max": 0.841332354404249, "nauc_mrr_at_3_std": -0.10062486533074633, "nauc_mrr_at_5_diff1": 0.9262936870031215, "nauc_mrr_at_5_max": 0.8423775470322344, "nauc_mrr_at_5_std": -0.09012841371006793, "nauc_ndcg_at_1000_diff1": 0.926427485475153, "nauc_ndcg_at_1000_max": 0.8437950581403517, "nauc_ndcg_at_1000_std": -0.08569476135678136, "nauc_ndcg_at_100_diff1": 0.9265205593685252, "nauc_ndcg_at_100_max": 0.845252988548431, "nauc_ndcg_at_100_std": -0.0763280082712212, "nauc_ndcg_at_10_diff1": 0.9261317588839924, "nauc_ndcg_at_10_max": 0.8439812832617384, "nauc_ndcg_at_10_std": -0.08993729428532617, "nauc_ndcg_at_1_diff1": 0.9282924551934596, "nauc_ndcg_at_1_max": 0.8302418528386962, "nauc_ndcg_at_1_std": -0.09564232196943509, "nauc_ndcg_at_20_diff1": 0.9272449406946948, "nauc_ndcg_at_20_max": 0.8478768490075218, "nauc_ndcg_at_20_std": -0.08408505632860358, "nauc_ndcg_at_3_diff1": 0.9255871319261063, "nauc_ndcg_at_3_max": 0.8458986599961826, "nauc_ndcg_at_3_std": -0.10218990873613223, "nauc_ndcg_at_5_diff1": 0.9265901724509867, "nauc_ndcg_at_5_max": 0.8487990745945639, "nauc_ndcg_at_5_std": -0.07950641622331456, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9346405228758011, "nauc_precision_at_100_max": 0.9673202614378776, "nauc_precision_at_100_std": 0.7071661998132774, "nauc_precision_at_10_diff1": 0.9283559577677162, "nauc_precision_at_10_max": 0.8797313797313696, "nauc_precision_at_10_std": -0.04787043022338166, "nauc_precision_at_1_diff1": 0.9282924551934596, "nauc_precision_at_1_max": 0.8302418528386962, "nauc_precision_at_1_std": -0.09564232196943509, "nauc_precision_at_20_diff1": 0.9477124183006497, "nauc_precision_at_20_max": 0.959150326797383, "nauc_precision_at_20_std": 0.07348272642390268, "nauc_precision_at_3_diff1": 0.9250022589681047, "nauc_precision_at_3_max": 0.8701996927803407, "nauc_precision_at_3_std": -0.11020752386976498, "nauc_precision_at_5_diff1": 0.9299719887955139, "nauc_precision_at_5_max": 0.8990929705215371, "nauc_precision_at_5_std": 0.01182695300341017, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9346405228758294, "nauc_recall_at_100_max": 0.9673202614379001, "nauc_recall_at_100_std": 0.7071661998132666, "nauc_recall_at_10_diff1": 0.928355957767723, "nauc_recall_at_10_max": 0.8797313797313832, "nauc_recall_at_10_std": -0.04787043022337262, "nauc_recall_at_1_diff1": 0.9282924551934596, "nauc_recall_at_1_max": 0.8302418528386962, "nauc_recall_at_1_std": -0.09564232196943509, "nauc_recall_at_20_diff1": 0.9477124183006521, "nauc_recall_at_20_max": 0.9591503267973885, "nauc_recall_at_20_std": 0.07348272642390039, "nauc_recall_at_3_diff1": 0.9250022589680997, "nauc_recall_at_3_max": 0.8701996927803363, "nauc_recall_at_3_std": -0.11020752386975824, "nauc_recall_at_5_diff1": 0.9299719887955186, "nauc_recall_at_5_max": 0.8990929705215392, "nauc_recall_at_5_std": 0.011826953003422682, "ndcg_at_1": 0.90889, "ndcg_at_10": 0.95027, "ndcg_at_100": 0.95225, "ndcg_at_1000": 0.95281, "ndcg_at_20": 0.95108, "ndcg_at_3": 0.9429, "ndcg_at_5": 0.94739, "precision_at_1": 0.90889, "precision_at_10": 0.09856, "precision_at_100": 0.00996, "precision_at_1000": 0.001, "precision_at_20": 0.04944, "precision_at_3": 0.32185, "precision_at_5": 0.19533, "recall_at_1": 0.90889, "recall_at_10": 0.98556, "recall_at_100": 0.99556, "recall_at_1000": 1.0, "recall_at_20": 0.98889, "recall_at_3": 0.96556, "recall_at_5": 0.97667 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.9243, "map_at_1": 0.86778, "map_at_10": 0.90697, "map_at_100": 0.90797, "map_at_1000": 0.90799, "map_at_20": 0.90784, "map_at_3": 0.89981, "map_at_5": 0.90387, "mrr_at_1": 0.8677777777777778, "mrr_at_10": 0.9069664902998237, "mrr_at_100": 0.9079666626771419, "mrr_at_1000": 0.9079885704290849, "mrr_at_20": 0.9078379264163577, "mrr_at_3": 0.8998148148148151, "mrr_at_5": 0.9038703703703705, "nauc_map_at_1000_diff1": 0.8850402796626184, "nauc_map_at_1000_max": 0.7599982828341915, "nauc_map_at_1000_std": 0.00892263162801917, "nauc_map_at_100_diff1": 0.8850459164348188, "nauc_map_at_100_max": 0.7600567511471158, "nauc_map_at_100_std": 0.009088482022350434, "nauc_map_at_10_diff1": 0.8844167646901695, "nauc_map_at_10_max": 0.7605216684007551, "nauc_map_at_10_std": 0.008632526869306515, "nauc_map_at_1_diff1": 0.8992049534399333, "nauc_map_at_1_max": 0.7400578657740957, "nauc_map_at_1_std": -0.01392344536385056, "nauc_map_at_20_diff1": 0.8847442992928001, "nauc_map_at_20_max": 0.7601359790995998, "nauc_map_at_20_std": 0.009396726928626828, "nauc_map_at_3_diff1": 0.8836902425995753, "nauc_map_at_3_max": 0.7634041510425207, "nauc_map_at_3_std": 0.002016815018924026, "nauc_map_at_5_diff1": 0.8838014395998648, "nauc_map_at_5_max": 0.7580897726104552, "nauc_map_at_5_std": 0.0029415599545449143, "nauc_mrr_at_1000_diff1": 0.8850402796626184, "nauc_mrr_at_1000_max": 0.7599982828341915, "nauc_mrr_at_1000_std": 0.00892263162801917, "nauc_mrr_at_100_diff1": 0.8850459164348188, "nauc_mrr_at_100_max": 0.7600567511471158, "nauc_mrr_at_100_std": 0.009088482022350434, "nauc_mrr_at_10_diff1": 0.8844167646901695, "nauc_mrr_at_10_max": 0.7605216684007551, "nauc_mrr_at_10_std": 0.008632526869306515, "nauc_mrr_at_1_diff1": 0.8992049534399333, "nauc_mrr_at_1_max": 0.7400578657740957, "nauc_mrr_at_1_std": -0.01392344536385056, "nauc_mrr_at_20_diff1": 0.8847442992928001, "nauc_mrr_at_20_max": 0.7601359790995998, "nauc_mrr_at_20_std": 0.009396726928626828, "nauc_mrr_at_3_diff1": 0.8836902425995753, "nauc_mrr_at_3_max": 0.7634041510425207, "nauc_mrr_at_3_std": 0.002016815018924026, "nauc_mrr_at_5_diff1": 0.8838014395998648, "nauc_mrr_at_5_max": 0.7580897726104552, "nauc_mrr_at_5_std": 0.0029415599545449143, "nauc_ndcg_at_1000_diff1": 0.8825641009246987, "nauc_ndcg_at_1000_max": 0.7631111439660291, "nauc_ndcg_at_1000_std": 0.01749127399569384, "nauc_ndcg_at_100_diff1": 0.8826975512935871, "nauc_ndcg_at_100_max": 0.7650497757016197, "nauc_ndcg_at_100_std": 0.02378241842482664, "nauc_ndcg_at_10_diff1": 0.8782614858008976, "nauc_ndcg_at_10_max": 0.7674941037073223, "nauc_ndcg_at_10_std": 0.025064113529180507, "nauc_ndcg_at_1_diff1": 0.8992049534399333, "nauc_ndcg_at_1_max": 0.7400578657740957, "nauc_ndcg_at_1_std": -0.01392344536385056, "nauc_ndcg_at_20_diff1": 0.8792986972419587, "nauc_ndcg_at_20_max": 0.765631594679748, "nauc_ndcg_at_20_std": 0.027100791792958764, "nauc_ndcg_at_3_diff1": 0.8772492993225067, "nauc_ndcg_at_3_max": 0.7706921507590612, "nauc_ndcg_at_3_std": 0.008060679840721787, "nauc_ndcg_at_5_diff1": 0.8769899243312133, "nauc_ndcg_at_5_max": 0.760375068676849, "nauc_ndcg_at_5_std": 0.009614546520145218, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8978758169934363, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.8068394024276223, "nauc_precision_at_10_diff1": 0.8088468720821603, "nauc_precision_at_10_max": 0.8489729225023332, "nauc_precision_at_10_std": 0.2315826330532213, "nauc_precision_at_1_diff1": 0.8992049534399333, "nauc_precision_at_1_max": 0.7400578657740957, "nauc_precision_at_1_std": -0.01392344536385056, "nauc_precision_at_20_diff1": 0.7500259363004358, "nauc_precision_at_20_max": 0.8905488121174372, "nauc_precision_at_20_std": 0.5106338831828816, "nauc_precision_at_3_diff1": 0.8477293947882198, "nauc_precision_at_3_max": 0.8028860028860081, "nauc_precision_at_3_std": 0.03547237076648694, "nauc_precision_at_5_diff1": 0.8348176877588617, "nauc_precision_at_5_max": 0.76960185783715, "nauc_precision_at_5_std": 0.04983360865712764, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8978758169934707, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.8068394024276523, "nauc_recall_at_10_diff1": 0.8088468720821673, "nauc_recall_at_10_max": 0.8489729225023311, "nauc_recall_at_10_std": 0.23158263305322288, "nauc_recall_at_1_diff1": 0.8992049534399333, "nauc_recall_at_1_max": 0.7400578657740957, "nauc_recall_at_1_std": -0.01392344536385056, "nauc_recall_at_20_diff1": 0.7500259363004375, "nauc_recall_at_20_max": 0.8905488121174361, "nauc_recall_at_20_std": 0.5106338831829043, "nauc_recall_at_3_diff1": 0.8477293947882195, "nauc_recall_at_3_max": 0.8028860028860038, "nauc_recall_at_3_std": 0.03547237076648996, "nauc_recall_at_5_diff1": 0.8348176877588636, "nauc_recall_at_5_max": 0.7696018578371483, "nauc_recall_at_5_std": 0.04983360865714109, "ndcg_at_1": 0.86778, "ndcg_at_10": 0.9243, "ndcg_at_100": 0.92843, "ndcg_at_1000": 0.929, "ndcg_at_20": 0.92742, "ndcg_at_3": 0.90988, "ndcg_at_5": 0.9172, "precision_at_1": 0.86778, "precision_at_10": 0.09778, "precision_at_100": 0.00996, "precision_at_1000": 0.001, "precision_at_20": 0.0495, "precision_at_3": 0.31296, "precision_at_5": 0.19133, "recall_at_1": 0.86778, "recall_at_10": 0.97778, "recall_at_100": 0.99556, "recall_at_1000": 1.0, "recall_at_20": 0.99, "recall_at_3": 0.93889, "recall_at_5": 0.95667 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.91243, "map_at_1": 0.85111, "map_at_10": 0.89436, "map_at_100": 0.89557, "map_at_1000": 0.89561, "map_at_20": 0.89523, "map_at_3": 0.88796, "map_at_5": 0.89191, "mrr_at_1": 0.8511111111111112, "mrr_at_10": 0.8943633156966495, "mrr_at_100": 0.8955659825522202, "mrr_at_1000": 0.8956053837277665, "mrr_at_20": 0.8952312013512996, "mrr_at_3": 0.8879629629629634, "mrr_at_5": 0.8919074074074078, "nauc_map_at_1000_diff1": 0.8718693922157789, "nauc_map_at_1000_max": 0.715312248960243, "nauc_map_at_1000_std": -0.02182153633698573, "nauc_map_at_100_diff1": 0.8718442887486781, "nauc_map_at_100_max": 0.715407001301985, "nauc_map_at_100_std": -0.021514166210986174, "nauc_map_at_10_diff1": 0.8707102953249783, "nauc_map_at_10_max": 0.715404021150519, "nauc_map_at_10_std": -0.022836282188191188, "nauc_map_at_1_diff1": 0.8839859050337086, "nauc_map_at_1_max": 0.6831071537778296, "nauc_map_at_1_std": -0.07323048473288021, "nauc_map_at_20_diff1": 0.8715081801438168, "nauc_map_at_20_max": 0.7159718638485788, "nauc_map_at_20_std": -0.01951756920340846, "nauc_map_at_3_diff1": 0.8703959648913775, "nauc_map_at_3_max": 0.7207982842845231, "nauc_map_at_3_std": -0.02022796775090404, "nauc_map_at_5_diff1": 0.8702499758102804, "nauc_map_at_5_max": 0.715969021995253, "nauc_map_at_5_std": -0.030122174974660958, "nauc_mrr_at_1000_diff1": 0.8718693922157789, "nauc_mrr_at_1000_max": 0.715312248960243, "nauc_mrr_at_1000_std": -0.02182153633698573, "nauc_mrr_at_100_diff1": 0.8718442887486781, "nauc_mrr_at_100_max": 0.715407001301985, "nauc_mrr_at_100_std": -0.021514166210986174, "nauc_mrr_at_10_diff1": 0.8707102953249783, "nauc_mrr_at_10_max": 0.715404021150519, "nauc_mrr_at_10_std": -0.022836282188191188, "nauc_mrr_at_1_diff1": 0.8839859050337086, "nauc_mrr_at_1_max": 0.6831071537778296, "nauc_mrr_at_1_std": -0.07323048473288021, "nauc_mrr_at_20_diff1": 0.8715081801438168, "nauc_mrr_at_20_max": 0.7159718638485788, "nauc_mrr_at_20_std": -0.01951756920340846, "nauc_mrr_at_3_diff1": 0.8703959648913775, "nauc_mrr_at_3_max": 0.7207982842845231, "nauc_mrr_at_3_std": -0.02022796775090404, "nauc_mrr_at_5_diff1": 0.8702499758102804, "nauc_mrr_at_5_max": 0.715969021995253, "nauc_mrr_at_5_std": -0.030122174974660958, "nauc_ndcg_at_1000_diff1": 0.8704215704436838, "nauc_ndcg_at_1000_max": 0.7206784091883098, "nauc_ndcg_at_1000_std": -0.01043167083553248, "nauc_ndcg_at_100_diff1": 0.870036775850903, "nauc_ndcg_at_100_max": 0.7228799515433424, "nauc_ndcg_at_100_std": -0.003498233838454148, "nauc_ndcg_at_10_diff1": 0.8653318379092054, "nauc_ndcg_at_10_max": 0.7248937716084193, "nauc_ndcg_at_10_std": -0.0020540513430598534, "nauc_ndcg_at_1_diff1": 0.8839859050337086, "nauc_ndcg_at_1_max": 0.6831071537778296, "nauc_ndcg_at_1_std": -0.07323048473288021, "nauc_ndcg_at_20_diff1": 0.8684097990221203, "nauc_ndcg_at_20_max": 0.7283072515601547, "nauc_ndcg_at_20_std": 0.013062095849363017, "nauc_ndcg_at_3_diff1": 0.8651129208552935, "nauc_ndcg_at_3_max": 0.7351914072551375, "nauc_ndcg_at_3_std": -0.0023841775056271893, "nauc_ndcg_at_5_diff1": 0.8646168164453103, "nauc_ndcg_at_5_max": 0.7261357664944987, "nauc_ndcg_at_5_std": -0.022761294127049272, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8366013071895334, "nauc_precision_at_100_max": 0.9444444444444465, "nauc_precision_at_100_std": 0.6873015873015987, "nauc_precision_at_10_diff1": 0.8173637271000318, "nauc_precision_at_10_max": 0.80266589394378, "nauc_precision_at_10_std": 0.19208602981421555, "nauc_precision_at_1_diff1": 0.8839859050337086, "nauc_precision_at_1_max": 0.6831071537778296, "nauc_precision_at_1_std": -0.07323048473288021, "nauc_precision_at_20_diff1": 0.8354341736694668, "nauc_precision_at_20_max": 0.9101307189542478, "nauc_precision_at_20_std": 0.5516391741881902, "nauc_precision_at_3_diff1": 0.8410785079058934, "nauc_precision_at_3_max": 0.8005311414183193, "nauc_precision_at_3_std": 0.07730633236901657, "nauc_precision_at_5_diff1": 0.8312584292976438, "nauc_precision_at_5_max": 0.7830895321091386, "nauc_precision_at_5_std": 0.01041601825915237, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8366013071895522, "nauc_recall_at_100_max": 0.9444444444444562, "nauc_recall_at_100_std": 0.6873015873015951, "nauc_recall_at_10_diff1": 0.817363727100036, "nauc_recall_at_10_max": 0.8026658939437843, "nauc_recall_at_10_std": 0.1920860298142285, "nauc_recall_at_1_diff1": 0.8839859050337086, "nauc_recall_at_1_max": 0.6831071537778296, "nauc_recall_at_1_std": -0.07323048473288021, "nauc_recall_at_20_diff1": 0.8354341736694711, "nauc_recall_at_20_max": 0.9101307189542475, "nauc_recall_at_20_std": 0.551639174188192, "nauc_recall_at_3_diff1": 0.8410785079058953, "nauc_recall_at_3_max": 0.8005311414183154, "nauc_recall_at_3_std": 0.07730633236901466, "nauc_recall_at_5_diff1": 0.831258429297646, "nauc_recall_at_5_max": 0.7830895321091382, "nauc_recall_at_5_std": 0.01041601825915693, "ndcg_at_1": 0.85111, "ndcg_at_10": 0.91243, "ndcg_at_100": 0.91815, "ndcg_at_1000": 0.91892, "ndcg_at_20": 0.91555, "ndcg_at_3": 0.89938, "ndcg_at_5": 0.9066, "precision_at_1": 0.85111, "precision_at_10": 0.09678, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "precision_at_20": 0.049, "precision_at_3": 0.31074, "precision_at_5": 0.19, "recall_at_1": 0.85111, "recall_at_10": 0.96778, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "recall_at_20": 0.98, "recall_at_3": 0.93222, "recall_at_5": 0.95 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 115.5916862487793, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.7959, "map_at_1": 0.49056, "map_at_10": 0.75131, "map_at_100": 0.75657, "map_at_1000": 0.75691, "map_at_20": 0.75501, "map_at_3": 0.71088, "map_at_5": 0.73983, "mrr_at_1": 0.7584459459459459, "mrr_at_10": 0.8134411196911195, "mrr_at_100": 0.8147137843388809, "mrr_at_1000": 0.8148351718852996, "mrr_at_20": 0.8141698656404537, "mrr_at_3": 0.8054617117117114, "mrr_at_5": 0.810529279279279, "nauc_map_at_1000_diff1": 0.6077723078794142, "nauc_map_at_1000_max": 0.6573089583997457, "nauc_map_at_1000_std": 0.2554489899927413, "nauc_map_at_100_diff1": 0.6076594934702981, "nauc_map_at_100_max": 0.6575393737587643, "nauc_map_at_100_std": 0.25591233440846073, "nauc_map_at_10_diff1": 0.6060453073908345, "nauc_map_at_10_max": 0.656696375458025, "nauc_map_at_10_std": 0.24695748909037316, "nauc_map_at_1_diff1": 0.6458252442838546, "nauc_map_at_1_max": 0.3121693863445959, "nauc_map_at_1_std": 0.01652697084396131, "nauc_map_at_20_diff1": 0.6069872318028429, "nauc_map_at_20_max": 0.656927115861969, "nauc_map_at_20_std": 0.25433092620557457, "nauc_map_at_3_diff1": 0.5925124364325525, "nauc_map_at_3_max": 0.6081099885122733, "nauc_map_at_3_std": 0.1852920843231918, "nauc_map_at_5_diff1": 0.6036824581960382, "nauc_map_at_5_max": 0.6503765291038188, "nauc_map_at_5_std": 0.21835916030257338, "nauc_mrr_at_1000_diff1": 0.6832517415820536, "nauc_mrr_at_1000_max": 0.7211637033411988, "nauc_mrr_at_1000_std": 0.3641206608277797, "nauc_mrr_at_100_diff1": 0.683308766051111, "nauc_mrr_at_100_max": 0.7213415575516359, "nauc_mrr_at_100_std": 0.3644459960058063, "nauc_mrr_at_10_diff1": 0.6832094931397921, "nauc_mrr_at_10_max": 0.7227122005456524, "nauc_mrr_at_10_std": 0.3653443095501317, "nauc_mrr_at_1_diff1": 0.6939000921334538, "nauc_mrr_at_1_max": 0.6351097031562031, "nauc_mrr_at_1_std": 0.26217789737674524, "nauc_mrr_at_20_diff1": 0.6831394705607008, "nauc_mrr_at_20_max": 0.7219536740322078, "nauc_mrr_at_20_std": 0.3649367382073439, "nauc_mrr_at_3_diff1": 0.6793523696127164, "nauc_mrr_at_3_max": 0.7248716464145419, "nauc_mrr_at_3_std": 0.35751312232773125, "nauc_mrr_at_5_diff1": 0.6821188934286015, "nauc_mrr_at_5_max": 0.7245259039980231, "nauc_mrr_at_5_std": 0.36255712853232797, "nauc_ndcg_at_1000_diff1": 0.6285352838789675, "nauc_ndcg_at_1000_max": 0.7091198441855272, "nauc_ndcg_at_1000_std": 0.3444313941136184, "nauc_ndcg_at_100_diff1": 0.6284775238416308, "nauc_ndcg_at_100_max": 0.7166963735380042, "nauc_ndcg_at_100_std": 0.357603340604324, "nauc_ndcg_at_10_diff1": 0.6224918160953309, "nauc_ndcg_at_10_max": 0.7144185507442697, "nauc_ndcg_at_10_std": 0.33006553182834125, "nauc_ndcg_at_1_diff1": 0.6939000921334538, "nauc_ndcg_at_1_max": 0.6351097031562031, "nauc_ndcg_at_1_std": 0.26217789737674524, "nauc_ndcg_at_20_diff1": 0.6253116873776855, "nauc_ndcg_at_20_max": 0.7161801994053858, "nauc_ndcg_at_20_std": 0.35005964923241917, "nauc_ndcg_at_3_diff1": 0.5996129518498234, "nauc_ndcg_at_3_max": 0.6570283339171115, "nauc_ndcg_at_3_std": 0.2354207606263176, "nauc_ndcg_at_5_diff1": 0.6183812518364293, "nauc_ndcg_at_5_max": 0.704540189540427, "nauc_ndcg_at_5_std": 0.2791596614003792, "nauc_precision_at_1000_diff1": -0.3068570126990965, "nauc_precision_at_1000_max": 0.03384203635866813, "nauc_precision_at_1000_std": 0.21656754445137796, "nauc_precision_at_100_diff1": -0.20359255842444202, "nauc_precision_at_100_max": 0.1624705732683665, "nauc_precision_at_100_std": 0.30479175635537603, "nauc_precision_at_10_diff1": -0.12392988970999819, "nauc_precision_at_10_max": 0.2721968635468893, "nauc_precision_at_10_std": 0.3113424136318071, "nauc_precision_at_1_diff1": 0.6939000921334538, "nauc_precision_at_1_max": 0.6351097031562031, "nauc_precision_at_1_std": 0.26217789737674524, "nauc_precision_at_20_diff1": -0.15781732437023308, "nauc_precision_at_20_max": 0.22767847657175486, "nauc_precision_at_20_std": 0.32515083661935607, "nauc_precision_at_3_diff1": 0.008293364102137213, "nauc_precision_at_3_max": 0.41108421339521484, "nauc_precision_at_3_std": 0.2756698186482374, "nauc_precision_at_5_diff1": -0.06855671223726188, "nauc_precision_at_5_max": 0.3447541035044828, "nauc_precision_at_5_std": 0.27670493704200283, "nauc_recall_at_1000_diff1": 0.6184491717695515, "nauc_recall_at_1000_max": 0.9373322639715344, "nauc_recall_at_1000_std": 0.8707432685857276, "nauc_recall_at_100_diff1": 0.6262762168447928, "nauc_recall_at_100_max": 0.9130875453909402, "nauc_recall_at_100_std": 0.7664218156346466, "nauc_recall_at_10_diff1": 0.5851607028848523, "nauc_recall_at_10_max": 0.8231971304893786, "nauc_recall_at_10_std": 0.49114599357555316, "nauc_recall_at_1_diff1": 0.6458252442838546, "nauc_recall_at_1_max": 0.3121693863445959, "nauc_recall_at_1_std": 0.01652697084396131, "nauc_recall_at_20_diff1": 0.6031863187385538, "nauc_recall_at_20_max": 0.8633164767407354, "nauc_recall_at_20_std": 0.6171635453840554, "nauc_recall_at_3_diff1": 0.5518966024258635, "nauc_recall_at_3_max": 0.6726580995760291, "nauc_recall_at_3_std": 0.23325305684585876, "nauc_recall_at_5_diff1": 0.5825866310075252, "nauc_recall_at_5_max": 0.777951156236192, "nauc_recall_at_5_std": 0.3204334395785277, "ndcg_at_1": 0.75845, "ndcg_at_10": 0.7959, "ndcg_at_100": 0.81128, "ndcg_at_1000": 0.81853, "ndcg_at_20": 0.80402, "ndcg_at_3": 0.75049, "ndcg_at_5": 0.77913, "precision_at_1": 0.75845, "precision_at_10": 0.15828, "precision_at_100": 0.01711, "precision_at_1000": 0.00182, "precision_at_20": 0.08193, "precision_at_3": 0.44538, "precision_at_5": 0.29662, "recall_at_1": 0.49056, "recall_at_10": 0.85471, "recall_at_100": 0.91048, "recall_at_1000": 0.95746, "recall_at_20": 0.87968, "recall_at_3": 0.75651, "recall_at_5": 0.81552 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 11312.0097489357, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.68331, "map_at_1": 0.3923, "map_at_10": 0.59239, "map_at_100": 0.6184, "map_at_1000": 0.61894, "map_at_20": 0.60938, "map_at_3": 0.51673, "map_at_5": 0.56081, "mrr_at_1": 0.6431924882629108, "mrr_at_10": 0.745903196959535, "mrr_at_100": 0.748772757610797, "mrr_at_1000": 0.7488523347263736, "mrr_at_20": 0.7485247970051528, "mrr_at_3": 0.7261345852895147, "mrr_at_5": 0.7392801251956181, "nauc_map_at_1000_diff1": 0.3774946180359577, "nauc_map_at_1000_max": 0.3535314711403068, "nauc_map_at_1000_std": -0.019909988048151767, "nauc_map_at_100_diff1": 0.3768667446156779, "nauc_map_at_100_max": 0.3528725827953486, "nauc_map_at_100_std": -0.019930418545331507, "nauc_map_at_10_diff1": 0.3850131229245043, "nauc_map_at_10_max": 0.32331258334429996, "nauc_map_at_10_std": -0.0671951047858982, "nauc_map_at_1_diff1": 0.5196816318543573, "nauc_map_at_1_max": 0.26411793746035594, "nauc_map_at_1_std": -0.1503999839349953, "nauc_map_at_20_diff1": 0.37669876948229386, "nauc_map_at_20_max": 0.34069178684781326, "nauc_map_at_20_std": -0.034742643713636716, "nauc_map_at_3_diff1": 0.4182627612388757, "nauc_map_at_3_max": 0.2739352819666986, "nauc_map_at_3_std": -0.12101619469325169, "nauc_map_at_5_diff1": 0.39687144596343005, "nauc_map_at_5_max": 0.2971376440937249, "nauc_map_at_5_std": -0.11478928058095475, "nauc_mrr_at_1000_diff1": 0.3739869856099168, "nauc_mrr_at_1000_max": 0.4806436351696716, "nauc_mrr_at_1000_std": 0.12498978009344504, "nauc_mrr_at_100_diff1": 0.37374067751519524, "nauc_mrr_at_100_max": 0.4806897016682821, "nauc_mrr_at_100_std": 0.12521604708949863, "nauc_mrr_at_10_diff1": 0.37390275527505346, "nauc_mrr_at_10_max": 0.48134636338291814, "nauc_mrr_at_10_std": 0.12252512689369086, "nauc_mrr_at_1_diff1": 0.40798053381505517, "nauc_mrr_at_1_max": 0.43355324394619194, "nauc_mrr_at_1_std": 0.10194903601481653, "nauc_mrr_at_20_diff1": 0.3733532620658847, "nauc_mrr_at_20_max": 0.481188843537414, "nauc_mrr_at_20_std": 0.12577842594499059, "nauc_mrr_at_3_diff1": 0.3702505623508384, "nauc_mrr_at_3_max": 0.48860309634358867, "nauc_mrr_at_3_std": 0.12959244782825213, "nauc_mrr_at_5_diff1": 0.37440575876060483, "nauc_mrr_at_5_max": 0.48190539569289936, "nauc_mrr_at_5_std": 0.10821144432670865, "nauc_ndcg_at_1000_diff1": 0.36468099422761685, "nauc_ndcg_at_1000_max": 0.4283410599592818, "nauc_ndcg_at_1000_std": 0.0777771522512293, "nauc_ndcg_at_100_diff1": 0.3568350316982012, "nauc_ndcg_at_100_max": 0.4267523913568, "nauc_ndcg_at_100_std": 0.08664324619901745, "nauc_ndcg_at_10_diff1": 0.3709148638740794, "nauc_ndcg_at_10_max": 0.370843620720174, "nauc_ndcg_at_10_std": -0.011817563463895356, "nauc_ndcg_at_1_diff1": 0.40798053381505517, "nauc_ndcg_at_1_max": 0.43355324394619194, "nauc_ndcg_at_1_std": 0.10194903601481653, "nauc_ndcg_at_20_diff1": 0.3545292942937683, "nauc_ndcg_at_20_max": 0.40307170590881086, "nauc_ndcg_at_20_std": 0.053364236078871974, "nauc_ndcg_at_3_diff1": 0.3634317603574458, "nauc_ndcg_at_3_max": 0.3702287982936343, "nauc_ndcg_at_3_std": 0.006161518715522704, "nauc_ndcg_at_5_diff1": 0.37810234789919145, "nauc_ndcg_at_5_max": 0.36167845453393555, "nauc_ndcg_at_5_std": -0.054103875622263674, "nauc_precision_at_1000_diff1": -0.23764683402393874, "nauc_precision_at_1000_max": 0.17062992955525072, "nauc_precision_at_1000_std": 0.41900460356843805, "nauc_precision_at_100_diff1": -0.24135047779614466, "nauc_precision_at_100_max": 0.18576687324926303, "nauc_precision_at_100_std": 0.4442303287310702, "nauc_precision_at_10_diff1": -0.22766796771967845, "nauc_precision_at_10_max": 0.21497148440048625, "nauc_precision_at_10_std": 0.37218123721668905, "nauc_precision_at_1_diff1": 0.40798053381505517, "nauc_precision_at_1_max": 0.43355324394619194, "nauc_precision_at_1_std": 0.10194903601481653, "nauc_precision_at_20_diff1": -0.26615333990630646, "nauc_precision_at_20_max": 0.20236992458242156, "nauc_precision_at_20_std": 0.4430085750960338, "nauc_precision_at_3_diff1": -0.11383669894778764, "nauc_precision_at_3_max": 0.2717879122260088, "nauc_precision_at_3_std": 0.2709135728694691, "nauc_precision_at_5_diff1": -0.19463523232683233, "nauc_precision_at_5_max": 0.2160201242456816, "nauc_precision_at_5_std": 0.2789409714060988, "nauc_recall_at_1000_diff1": 1.0, "nauc_recall_at_1000_max": 1.0, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_100_diff1": 0.18908040393134162, "nauc_recall_at_100_max": 0.47390319498448813, "nauc_recall_at_100_std": 0.49521202474372344, "nauc_recall_at_10_diff1": 0.29897134897446154, "nauc_recall_at_10_max": 0.25670493721041504, "nauc_recall_at_10_std": -0.082099196511308, "nauc_recall_at_1_diff1": 0.5196816318543573, "nauc_recall_at_1_max": 0.26411793746035594, "nauc_recall_at_1_std": -0.1503999839349953, "nauc_recall_at_20_diff1": 0.21242467085037842, "nauc_recall_at_20_max": 0.31989790929720013, "nauc_recall_at_20_std": 0.12476081609645708, "nauc_recall_at_3_diff1": 0.35945142823550263, "nauc_recall_at_3_max": 0.22427651751677, "nauc_recall_at_3_std": -0.166837960755657, "nauc_recall_at_5_diff1": 0.3154176983700154, "nauc_recall_at_5_max": 0.22034669463342174, "nauc_recall_at_5_std": -0.2377572626036249, "ndcg_at_1": 0.64319, "ndcg_at_10": 0.68331, "ndcg_at_100": 0.74053, "ndcg_at_1000": 0.74724, "ndcg_at_20": 0.71542, "ndcg_at_3": 0.6319, "ndcg_at_5": 0.65301, "precision_at_1": 0.64319, "precision_at_10": 0.17606, "precision_at_100": 0.02451, "precision_at_1000": 0.00256, "precision_at_20": 0.10399, "precision_at_3": 0.37559, "precision_at_5": 0.28263, "recall_at_1": 0.3923, "recall_at_10": 0.77817, "recall_at_100": 0.95483, "recall_at_1000": 0.99531, "recall_at_20": 0.86781, "recall_at_3": 0.5853, "recall_at_5": 0.68579 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 11472.845931529999, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.60992, "map_at_1": 0.43587, "map_at_10": 0.55105, "map_at_100": 0.55937, "map_at_1000": 0.55967, "map_at_20": 0.55623, "map_at_3": 0.52375, "map_at_5": 0.53682, "mrr_at_1": 0.47268408551068886, "mrr_at_10": 0.5790040719375636, "mrr_at_100": 0.5846102667713428, "mrr_at_1000": 0.5848739682713796, "mrr_at_20": 0.5824617319866726, "mrr_at_3": 0.5570071258907365, "mrr_at_5": 0.5674584323040381, "nauc_map_at_1000_diff1": 0.3753050258625946, "nauc_map_at_1000_max": 0.40673073430037027, "nauc_map_at_1000_std": 0.028514895004117826, "nauc_map_at_100_diff1": 0.37509599591180637, "nauc_map_at_100_max": 0.4068533874045987, "nauc_map_at_100_std": 0.02853250905845099, "nauc_map_at_10_diff1": 0.372372520798537, "nauc_map_at_10_max": 0.4038802418315354, "nauc_map_at_10_std": 0.028654103214843855, "nauc_map_at_1_diff1": 0.42515981554619875, "nauc_map_at_1_max": 0.37700543732750713, "nauc_map_at_1_std": -0.016879277061596795, "nauc_map_at_20_diff1": 0.3722103022931264, "nauc_map_at_20_max": 0.4052825801949023, "nauc_map_at_20_std": 0.03006486477801866, "nauc_map_at_3_diff1": 0.37971485207970246, "nauc_map_at_3_max": 0.40590668886879533, "nauc_map_at_3_std": 0.02401990421735226, "nauc_map_at_5_diff1": 0.37185612411403407, "nauc_map_at_5_max": 0.40773810091462803, "nauc_map_at_5_std": 0.02940435777573965, "nauc_mrr_at_1000_diff1": 0.4078523594341167, "nauc_mrr_at_1000_max": 0.4394076714687208, "nauc_mrr_at_1000_std": 0.04120542083949322, "nauc_mrr_at_100_diff1": 0.40758829173709177, "nauc_mrr_at_100_max": 0.4395221029740597, "nauc_mrr_at_100_std": 0.04133303089568957, "nauc_mrr_at_10_diff1": 0.4057055441388549, "nauc_mrr_at_10_max": 0.43793081284299035, "nauc_mrr_at_10_std": 0.04359601366998576, "nauc_mrr_at_1_diff1": 0.4543862358578472, "nauc_mrr_at_1_max": 0.4147346811251485, "nauc_mrr_at_1_std": 0.000748455478851234, "nauc_mrr_at_20_diff1": 0.40601610889868633, "nauc_mrr_at_20_max": 0.43892856816163567, "nauc_mrr_at_20_std": 0.04265928289658641, "nauc_mrr_at_3_diff1": 0.40922453974926726, "nauc_mrr_at_3_max": 0.43797844418516424, "nauc_mrr_at_3_std": 0.04160958218861414, "nauc_mrr_at_5_diff1": 0.40357752382017725, "nauc_mrr_at_5_max": 0.4404496442108461, "nauc_mrr_at_5_std": 0.04311527223227418, "nauc_ndcg_at_1000_diff1": 0.3757771157677983, "nauc_ndcg_at_1000_max": 0.4283923481165144, "nauc_ndcg_at_1000_std": 0.04388913877302194, "nauc_ndcg_at_100_diff1": 0.3697015687305564, "nauc_ndcg_at_100_max": 0.43299779530852794, "nauc_ndcg_at_100_std": 0.049515406975730564, "nauc_ndcg_at_10_diff1": 0.3568643008644076, "nauc_ndcg_at_10_max": 0.4191783096040415, "nauc_ndcg_at_10_std": 0.05327069324253914, "nauc_ndcg_at_1_diff1": 0.4543862358578472, "nauc_ndcg_at_1_max": 0.4147346811251485, "nauc_ndcg_at_1_std": 0.000748455478851234, "nauc_ndcg_at_20_diff1": 0.35642410549907505, "nauc_ndcg_at_20_max": 0.42428646497726574, "nauc_ndcg_at_20_std": 0.0571135852810691, "nauc_ndcg_at_3_diff1": 0.36932447470250523, "nauc_ndcg_at_3_max": 0.4216462868741249, "nauc_ndcg_at_3_std": 0.04413272834181027, "nauc_ndcg_at_5_diff1": 0.35585160636930985, "nauc_ndcg_at_5_max": 0.42614120371530917, "nauc_ndcg_at_5_std": 0.05331191865956327, "nauc_precision_at_1000_diff1": -0.015998553156530166, "nauc_precision_at_1000_max": 0.06158313258393215, "nauc_precision_at_1000_std": 0.0858583970040919, "nauc_precision_at_100_diff1": 0.02788668125934561, "nauc_precision_at_100_max": 0.18002237908297197, "nauc_precision_at_100_std": 0.13411651596172475, "nauc_precision_at_10_diff1": 0.1203763746848603, "nauc_precision_at_10_max": 0.28187948547700625, "nauc_precision_at_10_std": 0.15123759848899393, "nauc_precision_at_1_diff1": 0.4543862358578472, "nauc_precision_at_1_max": 0.4147346811251485, "nauc_precision_at_1_std": 0.000748455478851234, "nauc_precision_at_20_diff1": 0.06066286887665023, "nauc_precision_at_20_max": 0.23305425439373617, "nauc_precision_at_20_std": 0.1663912099113602, "nauc_precision_at_3_diff1": 0.27540446711958216, "nauc_precision_at_3_max": 0.4246280669061289, "nauc_precision_at_3_std": 0.13011858770823792, "nauc_precision_at_5_diff1": 0.18631257416980498, "nauc_precision_at_5_max": 0.3771850902827074, "nauc_precision_at_5_std": 0.1485475132080138, "nauc_recall_at_1000_diff1": 0.42933168115881765, "nauc_recall_at_1000_max": 0.6590218806072644, "nauc_recall_at_1000_std": 0.008830912115977791, "nauc_recall_at_100_diff1": 0.24903970913072537, "nauc_recall_at_100_max": 0.6180656734448082, "nauc_recall_at_100_std": 0.18147764743985456, "nauc_recall_at_10_diff1": 0.23934696991783594, "nauc_recall_at_10_max": 0.4244251936633175, "nauc_recall_at_10_std": 0.1290442757956174, "nauc_recall_at_1_diff1": 0.42515981554619875, "nauc_recall_at_1_max": 0.37700543732750713, "nauc_recall_at_1_std": -0.016879277061596795, "nauc_recall_at_20_diff1": 0.2061196808707721, "nauc_recall_at_20_max": 0.4563681973865881, "nauc_recall_at_20_std": 0.1729491007398988, "nauc_recall_at_3_diff1": 0.2985328976269163, "nauc_recall_at_3_max": 0.42777057232006016, "nauc_recall_at_3_std": 0.08664950021076616, "nauc_recall_at_5_diff1": 0.26064350809617987, "nauc_recall_at_5_max": 0.4435986352499763, "nauc_recall_at_5_std": 0.11666456046771807, "ndcg_at_1": 0.47268, "ndcg_at_10": 0.60992, "ndcg_at_100": 0.64446, "ndcg_at_1000": 0.65166, "ndcg_at_20": 0.62619, "ndcg_at_3": 0.55724, "ndcg_at_5": 0.57855, "precision_at_1": 0.47268, "precision_at_10": 0.08741, "precision_at_100": 0.01064, "precision_at_1000": 0.00113, "precision_at_20": 0.04751, "precision_at_3": 0.2304, "precision_at_5": 0.15154, "recall_at_1": 0.43587, "recall_at_10": 0.76128, "recall_at_100": 0.91251, "recall_at_1000": 0.96675, "recall_at_20": 0.82225, "recall_at_3": 0.62233, "recall_at_5": 0.67023 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 11403.819789886475, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.40135, "map_at_1": 0.305, "map_at_10": 0.36971, "map_at_100": 0.38027, "map_at_1000": 0.38101, "map_at_20": 0.37548, "map_at_3": 0.35333, "map_at_5": 0.36383, "mrr_at_1": 0.305, "mrr_at_10": 0.3697142857142858, "mrr_at_100": 0.38027095017778445, "mrr_at_1000": 0.3810058122522981, "mrr_at_20": 0.375484421710041, "mrr_at_3": 0.35333333333333344, "mrr_at_5": 0.3638333333333334, "nauc_map_at_1000_diff1": 0.620492315177477, "nauc_map_at_1000_max": 0.5405636248210866, "nauc_map_at_1000_std": 0.04283524136714581, "nauc_map_at_100_diff1": 0.6202335672884487, "nauc_map_at_100_max": 0.5403203845586158, "nauc_map_at_100_std": 0.04307170013811605, "nauc_map_at_10_diff1": 0.6222606937807431, "nauc_map_at_10_max": 0.5410835318264443, "nauc_map_at_10_std": 0.033766238863526594, "nauc_map_at_1_diff1": 0.6205732308086288, "nauc_map_at_1_max": 0.5077466354869362, "nauc_map_at_1_std": 0.016915228422975038, "nauc_map_at_20_diff1": 0.6223990320710757, "nauc_map_at_20_max": 0.5405664933566876, "nauc_map_at_20_std": 0.03971339133638137, "nauc_map_at_3_diff1": 0.6234319716950789, "nauc_map_at_3_max": 0.5516904083152316, "nauc_map_at_3_std": 0.0369078724163609, "nauc_map_at_5_diff1": 0.6273667124297218, "nauc_map_at_5_max": 0.5489856322874752, "nauc_map_at_5_std": 0.03448790522028278, "nauc_mrr_at_1000_diff1": 0.620492315177477, "nauc_mrr_at_1000_max": 0.5405636248210866, "nauc_mrr_at_1000_std": 0.04283524136714581, "nauc_mrr_at_100_diff1": 0.6202335672884487, "nauc_mrr_at_100_max": 0.5403203845586158, "nauc_mrr_at_100_std": 0.04307170013811605, "nauc_mrr_at_10_diff1": 0.6222606937807431, "nauc_mrr_at_10_max": 0.5410835318264443, "nauc_mrr_at_10_std": 0.033766238863526594, "nauc_mrr_at_1_diff1": 0.6205732308086288, "nauc_mrr_at_1_max": 0.5077466354869362, "nauc_mrr_at_1_std": 0.016915228422975038, "nauc_mrr_at_20_diff1": 0.6223990320710757, "nauc_mrr_at_20_max": 0.5405664933566876, "nauc_mrr_at_20_std": 0.03971339133638137, "nauc_mrr_at_3_diff1": 0.6234319716950789, "nauc_mrr_at_3_max": 0.5516904083152316, "nauc_mrr_at_3_std": 0.0369078724163609, "nauc_mrr_at_5_diff1": 0.6273667124297218, "nauc_mrr_at_5_max": 0.5489856322874752, "nauc_mrr_at_5_std": 0.03448790522028278, "nauc_ndcg_at_1000_diff1": 0.6144687969628315, "nauc_ndcg_at_1000_max": 0.5451156152792646, "nauc_ndcg_at_1000_std": 0.07430926447896827, "nauc_ndcg_at_100_diff1": 0.6061270612245266, "nauc_ndcg_at_100_max": 0.5394659921296081, "nauc_ndcg_at_100_std": 0.0868759587339719, "nauc_ndcg_at_10_diff1": 0.6171285085868562, "nauc_ndcg_at_10_max": 0.5396922542610059, "nauc_ndcg_at_10_std": 0.03410463633600367, "nauc_ndcg_at_1_diff1": 0.6205732308086288, "nauc_ndcg_at_1_max": 0.5077466354869362, "nauc_ndcg_at_1_std": 0.016915228422975038, "nauc_ndcg_at_20_diff1": 0.6179654605898276, "nauc_ndcg_at_20_max": 0.5385710587988575, "nauc_ndcg_at_20_std": 0.05852404464151673, "nauc_ndcg_at_3_diff1": 0.6234456627258951, "nauc_ndcg_at_3_max": 0.5633149892929821, "nauc_ndcg_at_3_std": 0.040746553975076705, "nauc_ndcg_at_5_diff1": 0.6298276712412417, "nauc_ndcg_at_5_max": 0.5576675497793011, "nauc_ndcg_at_5_std": 0.03545339125528916, "nauc_precision_at_1000_diff1": 0.615857454092746, "nauc_precision_at_1000_max": 0.7201213818860871, "nauc_precision_at_1000_std": 0.698996265172736, "nauc_precision_at_100_diff1": 0.5077437837111977, "nauc_precision_at_100_max": 0.5325977484836956, "nauc_precision_at_100_std": 0.4298023768492202, "nauc_precision_at_10_diff1": 0.5969353764157226, "nauc_precision_at_10_max": 0.5273417721518988, "nauc_precision_at_10_std": 0.03258494337108581, "nauc_precision_at_1_diff1": 0.6205732308086288, "nauc_precision_at_1_max": 0.5077466354869362, "nauc_precision_at_1_std": 0.016915228422975038, "nauc_precision_at_20_diff1": 0.5994981437187257, "nauc_precision_at_20_max": 0.5234419710332074, "nauc_precision_at_20_std": 0.1423980512715159, "nauc_precision_at_3_diff1": 0.623215221483978, "nauc_precision_at_3_max": 0.5955386197928461, "nauc_precision_at_3_std": 0.050863357292118244, "nauc_precision_at_5_diff1": 0.6370266221649893, "nauc_precision_at_5_max": 0.580451236498384, "nauc_precision_at_5_std": 0.03604373078237124, "nauc_recall_at_1000_diff1": 0.6158574540927494, "nauc_recall_at_1000_max": 0.7201213818860859, "nauc_recall_at_1000_std": 0.6989962651727336, "nauc_recall_at_100_diff1": 0.5077437837111974, "nauc_recall_at_100_max": 0.5325977484836953, "nauc_recall_at_100_std": 0.42980237684921996, "nauc_recall_at_10_diff1": 0.5969353764157229, "nauc_recall_at_10_max": 0.5273417721518984, "nauc_recall_at_10_std": 0.032584943371085875, "nauc_recall_at_1_diff1": 0.6205732308086288, "nauc_recall_at_1_max": 0.5077466354869362, "nauc_recall_at_1_std": 0.016915228422975038, "nauc_recall_at_20_diff1": 0.5994981437187257, "nauc_recall_at_20_max": 0.5234419710332074, "nauc_recall_at_20_std": 0.14239805127151595, "nauc_recall_at_3_diff1": 0.6232152214839783, "nauc_recall_at_3_max": 0.5955386197928457, "nauc_recall_at_3_std": 0.050863357292118404, "nauc_recall_at_5_diff1": 0.6370266221649892, "nauc_recall_at_5_max": 0.5804512364983837, "nauc_recall_at_5_std": 0.03604373078237104, "ndcg_at_1": 0.305, "ndcg_at_10": 0.40135, "ndcg_at_100": 0.45729, "ndcg_at_1000": 0.47851, "ndcg_at_20": 0.42351, "ndcg_at_3": 0.36917, "ndcg_at_5": 0.38789, "precision_at_1": 0.305, "precision_at_10": 0.05, "precision_at_100": 0.0077, "precision_at_1000": 0.00094, "precision_at_20": 0.0295, "precision_at_3": 0.13833, "precision_at_5": 0.092, "recall_at_1": 0.305, "recall_at_10": 0.5, "recall_at_100": 0.77, "recall_at_1000": 0.94, "recall_at_20": 0.59, "recall_at_3": 0.415, "recall_at_5": 0.46 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.3784, "map_at_1": 0.29, "map_at_10": 0.35064, "map_at_100": 0.36037, "map_at_1000": 0.36121, "map_at_20": 0.35632, "map_at_3": 0.33833, "map_at_5": 0.34508, "mrr_at_1": 0.29, "mrr_at_10": 0.35064285714285715, "mrr_at_100": 0.36037241577362655, "mrr_at_1000": 0.36121480369607695, "mrr_at_20": 0.3563192501942504, "mrr_at_3": 0.3383333333333334, "mrr_at_5": 0.3450833333333334, "nauc_map_at_1000_diff1": 0.5555260535712806, "nauc_map_at_1000_max": 0.511946413470973, "nauc_map_at_1000_std": 0.022591047932817226, "nauc_map_at_100_diff1": 0.5553419096812181, "nauc_map_at_100_max": 0.5117643587928126, "nauc_map_at_100_std": 0.022828388659646884, "nauc_map_at_10_diff1": 0.5572509137694774, "nauc_map_at_10_max": 0.5085916101319876, "nauc_map_at_10_std": 0.012676658329875811, "nauc_map_at_1_diff1": 0.6181956787495021, "nauc_map_at_1_max": 0.5593656854799941, "nauc_map_at_1_std": 0.005727785942886888, "nauc_map_at_20_diff1": 0.5550542388638184, "nauc_map_at_20_max": 0.5106556823005847, "nauc_map_at_20_std": 0.01847058609115171, "nauc_map_at_3_diff1": 0.5666017009607073, "nauc_map_at_3_max": 0.5049535424829046, "nauc_map_at_3_std": -0.009695964940741828, "nauc_map_at_5_diff1": 0.5647817750504657, "nauc_map_at_5_max": 0.5100247925875246, "nauc_map_at_5_std": 0.00415943136932681, "nauc_mrr_at_1000_diff1": 0.5555260535712806, "nauc_mrr_at_1000_max": 0.511946413470973, "nauc_mrr_at_1000_std": 0.022591047932817226, "nauc_mrr_at_100_diff1": 0.5553419096812181, "nauc_mrr_at_100_max": 0.5117643587928126, "nauc_mrr_at_100_std": 0.022828388659646884, "nauc_mrr_at_10_diff1": 0.5572509137694774, "nauc_mrr_at_10_max": 0.5085916101319876, "nauc_mrr_at_10_std": 0.012676658329875811, "nauc_mrr_at_1_diff1": 0.6181956787495021, "nauc_mrr_at_1_max": 0.5593656854799941, "nauc_mrr_at_1_std": 0.005727785942886888, "nauc_mrr_at_20_diff1": 0.5550542388638184, "nauc_mrr_at_20_max": 0.5106556823005847, "nauc_mrr_at_20_std": 0.01847058609115171, "nauc_mrr_at_3_diff1": 0.5666017009607073, "nauc_mrr_at_3_max": 0.5049535424829046, "nauc_mrr_at_3_std": -0.009695964940741828, "nauc_mrr_at_5_diff1": 0.5647817750504657, "nauc_mrr_at_5_max": 0.5100247925875246, "nauc_mrr_at_5_std": 0.00415943136932681, "nauc_ndcg_at_1000_diff1": 0.5337453799092685, "nauc_ndcg_at_1000_max": 0.5143767757163529, "nauc_ndcg_at_1000_std": 0.0723995373076684, "nauc_ndcg_at_100_diff1": 0.530109793159288, "nauc_ndcg_at_100_max": 0.5082470366543096, "nauc_ndcg_at_100_std": 0.07922652454856884, "nauc_ndcg_at_10_diff1": 0.5330177561108366, "nauc_ndcg_at_10_max": 0.4955767787741621, "nauc_ndcg_at_10_std": 0.030057528041445056, "nauc_ndcg_at_1_diff1": 0.6181956787495021, "nauc_ndcg_at_1_max": 0.5593656854799941, "nauc_ndcg_at_1_std": 0.005727785942886888, "nauc_ndcg_at_20_diff1": 0.5242176447056347, "nauc_ndcg_at_20_max": 0.5020210419233608, "nauc_ndcg_at_20_std": 0.05067602714199018, "nauc_ndcg_at_3_diff1": 0.5526245593820885, "nauc_ndcg_at_3_max": 0.4884707148210188, "nauc_ndcg_at_3_std": -0.016061366465278247, "nauc_ndcg_at_5_diff1": 0.550736289740471, "nauc_ndcg_at_5_max": 0.49873468260312426, "nauc_ndcg_at_5_std": 0.008663144671605812, "nauc_precision_at_1000_diff1": 0.3775124084721601, "nauc_precision_at_1000_max": 0.6907219027962068, "nauc_precision_at_1000_std": 0.7452208953756946, "nauc_precision_at_100_diff1": 0.4407570816548374, "nauc_precision_at_100_max": 0.5148118714203502, "nauc_precision_at_100_std": 0.40133183524704724, "nauc_precision_at_10_diff1": 0.45892813657933, "nauc_precision_at_10_max": 0.46018520346713526, "nauc_precision_at_10_std": 0.09171323079353164, "nauc_precision_at_1_diff1": 0.6181956787495021, "nauc_precision_at_1_max": 0.5593656854799941, "nauc_precision_at_1_std": 0.005727785942886888, "nauc_precision_at_20_diff1": 0.41805590286460453, "nauc_precision_at_20_max": 0.48255119335389396, "nauc_precision_at_20_std": 0.17746103192464793, "nauc_precision_at_3_diff1": 0.5141393482286294, "nauc_precision_at_3_max": 0.4424844992999681, "nauc_precision_at_3_std": -0.03434348648003473, "nauc_precision_at_5_diff1": 0.5125645103703462, "nauc_precision_at_5_max": 0.4693888149566688, "nauc_precision_at_5_std": 0.02425265344542158, "nauc_recall_at_1000_diff1": 0.3775124084721618, "nauc_recall_at_1000_max": 0.6907219027962078, "nauc_recall_at_1000_std": 0.7452208953756937, "nauc_recall_at_100_diff1": 0.4407570816548371, "nauc_recall_at_100_max": 0.5148118714203502, "nauc_recall_at_100_std": 0.4013318352470472, "nauc_recall_at_10_diff1": 0.4589281365793298, "nauc_recall_at_10_max": 0.46018520346713554, "nauc_recall_at_10_std": 0.09171323079353179, "nauc_recall_at_1_diff1": 0.6181956787495021, "nauc_recall_at_1_max": 0.5593656854799941, "nauc_recall_at_1_std": 0.005727785942886888, "nauc_recall_at_20_diff1": 0.4180559028646046, "nauc_recall_at_20_max": 0.48255119335389335, "nauc_recall_at_20_std": 0.17746103192464796, "nauc_recall_at_3_diff1": 0.5141393482286297, "nauc_recall_at_3_max": 0.44248449929996847, "nauc_recall_at_3_std": -0.034343486480034824, "nauc_recall_at_5_diff1": 0.5125645103703461, "nauc_recall_at_5_max": 0.46938881495666834, "nauc_recall_at_5_std": 0.024252653445421834, "ndcg_at_1": 0.29, "ndcg_at_10": 0.3784, "ndcg_at_100": 0.42817, "ndcg_at_1000": 0.45293, "ndcg_at_20": 0.39795, "ndcg_at_3": 0.35297, "ndcg_at_5": 0.36524, "precision_at_1": 0.29, "precision_at_10": 0.0465, "precision_at_100": 0.00705, "precision_at_1000": 0.00091, "precision_at_20": 0.027, "precision_at_3": 0.13167, "precision_at_5": 0.085, "recall_at_1": 0.29, "recall_at_10": 0.465, "recall_at_100": 0.705, "recall_at_1000": 0.905, "recall_at_20": 0.54, "recall_at_3": 0.395, "recall_at_5": 0.425 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 1.4347784519195557, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.81545, "map_at_1": 0.7013, "map_at_10": 0.77759, "map_at_100": 0.78062, "map_at_1000": 0.78062, "map_at_20": 0.77954, "map_at_3": 0.75974, "map_at_5": 0.76883, "mrr_at_1": 0.7012987012987013, "mrr_at_10": 0.7775922490208205, "mrr_at_100": 0.7806158886754763, "mrr_at_1000": 0.7806158886754763, "mrr_at_20": 0.779536828505507, "mrr_at_3": 0.7597402597402597, "mrr_at_5": 0.7688311688311689, "nauc_map_at_1000_diff1": 0.6585307223652647, "nauc_map_at_1000_max": 0.37680650139474897, "nauc_map_at_1000_std": -0.1509877249691887, "nauc_map_at_100_diff1": 0.6585307223652647, "nauc_map_at_100_max": 0.37680650139474897, "nauc_map_at_100_std": -0.1509877249691887, "nauc_map_at_10_diff1": 0.6568847829190118, "nauc_map_at_10_max": 0.3767377885847465, "nauc_map_at_10_std": -0.14852493078538043, "nauc_map_at_1_diff1": 0.6668096724205888, "nauc_map_at_1_max": 0.36184014288741295, "nauc_map_at_1_std": -0.15777912686633552, "nauc_map_at_20_diff1": 0.6552188727740201, "nauc_map_at_20_max": 0.37306386412717907, "nauc_map_at_20_std": -0.15147819783345062, "nauc_map_at_3_diff1": 0.6738011334604562, "nauc_map_at_3_max": 0.39203434874434695, "nauc_map_at_3_std": -0.1433564775107005, "nauc_map_at_5_diff1": 0.663166522294746, "nauc_map_at_5_max": 0.38276613935527093, "nauc_map_at_5_std": -0.15743918578933352, "nauc_mrr_at_1000_diff1": 0.6585307223652647, "nauc_mrr_at_1000_max": 0.37680650139474897, "nauc_mrr_at_1000_std": -0.1509877249691887, "nauc_mrr_at_100_diff1": 0.6585307223652647, "nauc_mrr_at_100_max": 0.37680650139474897, "nauc_mrr_at_100_std": -0.1509877249691887, "nauc_mrr_at_10_diff1": 0.6568847829190118, "nauc_mrr_at_10_max": 0.3767377885847465, "nauc_mrr_at_10_std": -0.14852493078538043, "nauc_mrr_at_1_diff1": 0.6668096724205888, "nauc_mrr_at_1_max": 0.36184014288741295, "nauc_mrr_at_1_std": -0.15777912686633552, "nauc_mrr_at_20_diff1": 0.6552188727740201, "nauc_mrr_at_20_max": 0.37306386412717907, "nauc_mrr_at_20_std": -0.15147819783345062, "nauc_mrr_at_3_diff1": 0.6738011334604562, "nauc_mrr_at_3_max": 0.39203434874434695, "nauc_mrr_at_3_std": -0.1433564775107005, "nauc_mrr_at_5_diff1": 0.663166522294746, "nauc_mrr_at_5_max": 0.38276613935527093, "nauc_mrr_at_5_std": -0.15743918578933352, "nauc_ndcg_at_1000_diff1": 0.6529694109214854, "nauc_ndcg_at_1000_max": 0.3733407917688893, "nauc_ndcg_at_1000_std": -0.1508903313817761, "nauc_ndcg_at_100_diff1": 0.6529694109214854, "nauc_ndcg_at_100_max": 0.3733407917688893, "nauc_ndcg_at_100_std": -0.1508903313817761, "nauc_ndcg_at_10_diff1": 0.6362204198354469, "nauc_ndcg_at_10_max": 0.3622349150702435, "nauc_ndcg_at_10_std": -0.14284302622133724, "nauc_ndcg_at_1_diff1": 0.6668096724205888, "nauc_ndcg_at_1_max": 0.36184014288741295, "nauc_ndcg_at_1_std": -0.15777912686633552, "nauc_ndcg_at_20_diff1": 0.6295586442414385, "nauc_ndcg_at_20_max": 0.3478591208886981, "nauc_ndcg_at_20_std": -0.14884275561618454, "nauc_ndcg_at_3_diff1": 0.6758508656326555, "nauc_ndcg_at_3_max": 0.39908738607341165, "nauc_ndcg_at_3_std": -0.14344324960656918, "nauc_ndcg_at_5_diff1": 0.6553047110252233, "nauc_ndcg_at_5_max": 0.38037489847847916, "nauc_ndcg_at_5_std": -0.17020307882711103, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.42261678126376323, "nauc_precision_at_10_max": 0.19705791037241055, "nauc_precision_at_10_std": -0.06801212066082346, "nauc_precision_at_1_diff1": 0.6668096724205888, "nauc_precision_at_1_max": 0.36184014288741295, "nauc_precision_at_1_std": -0.15777912686633552, "nauc_precision_at_20_diff1": 0.1860790840193073, "nauc_precision_at_20_max": -0.12420871870242532, "nauc_precision_at_20_std": -0.058565069938573355, "nauc_precision_at_3_diff1": 0.6834473314679363, "nauc_precision_at_3_max": 0.42424295062548845, "nauc_precision_at_3_std": -0.14612372971147158, "nauc_precision_at_5_diff1": 0.6137297328925272, "nauc_precision_at_5_max": 0.36146272035494137, "nauc_precision_at_5_std": -0.23800854389467807, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.4226167812637596, "nauc_recall_at_10_max": 0.1970579103724084, "nauc_recall_at_10_std": -0.06801212066082829, "nauc_recall_at_1_diff1": 0.6668096724205888, "nauc_recall_at_1_max": 0.36184014288741295, "nauc_recall_at_1_std": -0.15777912686633552, "nauc_recall_at_20_diff1": 0.18607908401930062, "nauc_recall_at_20_max": -0.12420871870243454, "nauc_recall_at_20_std": -0.058565069938578386, "nauc_recall_at_3_diff1": 0.6834473314679358, "nauc_recall_at_3_max": 0.42424295062548667, "nauc_recall_at_3_std": -0.14612372971147305, "nauc_recall_at_5_diff1": 0.6137297328925254, "nauc_recall_at_5_max": 0.36146272035494126, "nauc_recall_at_5_std": -0.23800854389468104, "ndcg_at_1": 0.7013, "ndcg_at_10": 0.81545, "ndcg_at_100": 0.82965, "ndcg_at_1000": 0.82965, "ndcg_at_20": 0.82219, "ndcg_at_3": 0.77814, "ndcg_at_5": 0.79435, "precision_at_1": 0.7013, "precision_at_10": 0.09351, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04805, "precision_at_3": 0.27706, "precision_at_5": 0.17403, "recall_at_1": 0.7013, "recall_at_10": 0.93506, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.96104, "recall_at_3": 0.83117, "recall_at_5": 0.87013 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 15.096574306488037, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.38131, "map_at_1": 0.20805, "map_at_10": 0.32937, "map_at_100": 0.3439, "map_at_1000": 0.34498, "map_at_20": 0.33584, "map_at_3": 0.29493, "map_at_5": 0.3129, "mrr_at_1": 0.29510703363914376, "mrr_at_10": 0.37063795446822967, "mrr_at_100": 0.3814246776969569, "mrr_at_1000": 0.3821234798000114, "mrr_at_20": 0.37556796485400895, "mrr_at_3": 0.34836901121304803, "mrr_at_5": 0.358231396534149, "nauc_map_at_1000_diff1": 0.47194617185309445, "nauc_map_at_1000_max": 0.20063304959237294, "nauc_map_at_1000_std": -0.27080467316354245, "nauc_map_at_100_diff1": 0.47167258973917525, "nauc_map_at_100_max": 0.20103909814762727, "nauc_map_at_100_std": -0.2699996587163077, "nauc_map_at_10_diff1": 0.47494322451299137, "nauc_map_at_10_max": 0.19755348750394167, "nauc_map_at_10_std": -0.277782394156506, "nauc_map_at_1_diff1": 0.5243004316160466, "nauc_map_at_1_max": 0.16858988412450363, "nauc_map_at_1_std": -0.23630036197807808, "nauc_map_at_20_diff1": 0.4731181060816184, "nauc_map_at_20_max": 0.19910885900001027, "nauc_map_at_20_std": -0.27552019101544084, "nauc_map_at_3_diff1": 0.4660117998053003, "nauc_map_at_3_max": 0.19183641669365997, "nauc_map_at_3_std": -0.2739927144150334, "nauc_map_at_5_diff1": 0.47215462722285606, "nauc_map_at_5_max": 0.19356885465948628, "nauc_map_at_5_std": -0.2743860928780198, "nauc_mrr_at_1000_diff1": 0.49532131721462486, "nauc_mrr_at_1000_max": 0.20403872185607005, "nauc_mrr_at_1000_std": -0.2714557158295466, "nauc_mrr_at_100_diff1": 0.49510161178221995, "nauc_mrr_at_100_max": 0.20455710839900487, "nauc_mrr_at_100_std": -0.2706221095946801, "nauc_mrr_at_10_diff1": 0.4978837020078874, "nauc_mrr_at_10_max": 0.20061088996099663, "nauc_mrr_at_10_std": -0.278041836280318, "nauc_mrr_at_1_diff1": 0.5221472978688461, "nauc_mrr_at_1_max": 0.21672555112727923, "nauc_mrr_at_1_std": -0.27530302599967527, "nauc_mrr_at_20_diff1": 0.4953208420700562, "nauc_mrr_at_20_max": 0.2027147445718628, "nauc_mrr_at_20_std": -0.273790593913149, "nauc_mrr_at_3_diff1": 0.49391292145586174, "nauc_mrr_at_3_max": 0.19579114213137086, "nauc_mrr_at_3_std": -0.2853501027400802, "nauc_mrr_at_5_diff1": 0.49648266111657696, "nauc_mrr_at_5_max": 0.19622910821499567, "nauc_mrr_at_5_std": -0.27956239907708147, "nauc_ndcg_at_1000_diff1": 0.47207814235419326, "nauc_ndcg_at_1000_max": 0.21188037356947, "nauc_ndcg_at_1000_std": -0.25352243997667206, "nauc_ndcg_at_100_diff1": 0.4633453263763678, "nauc_ndcg_at_100_max": 0.2243824116388276, "nauc_ndcg_at_100_std": -0.22895084512717154, "nauc_ndcg_at_10_diff1": 0.4788916780647435, "nauc_ndcg_at_10_max": 0.19978376922980615, "nauc_ndcg_at_10_std": -0.2785919189291289, "nauc_ndcg_at_1_diff1": 0.5221472978688461, "nauc_ndcg_at_1_max": 0.21865071515849718, "nauc_ndcg_at_1_std": -0.2685722998447304, "nauc_ndcg_at_20_diff1": 0.4711012521645191, "nauc_ndcg_at_20_max": 0.2056753067718257, "nauc_ndcg_at_20_std": -0.26741589614765826, "nauc_ndcg_at_3_diff1": 0.46474190308719693, "nauc_ndcg_at_3_max": 0.1847644418071173, "nauc_ndcg_at_3_std": -0.2867448273254548, "nauc_ndcg_at_5_diff1": 0.4735161679168353, "nauc_ndcg_at_5_max": 0.18954784747065154, "nauc_ndcg_at_5_std": -0.27643406781430935, "nauc_precision_at_1000_diff1": -0.007073399531117126, "nauc_precision_at_1000_max": 0.1339207045233831, "nauc_precision_at_1000_std": 0.043721130389564745, "nauc_precision_at_100_diff1": 0.07948052573255646, "nauc_precision_at_100_max": 0.20810014729214976, "nauc_precision_at_100_std": 0.056528611535616245, "nauc_precision_at_10_diff1": 0.29732012581042677, "nauc_precision_at_10_max": 0.1823552920291509, "nauc_precision_at_10_std": -0.23160492342624037, "nauc_precision_at_1_diff1": 0.5221472978688461, "nauc_precision_at_1_max": 0.21865071515849718, "nauc_precision_at_1_std": -0.2685722998447304, "nauc_precision_at_20_diff1": 0.24428953207068782, "nauc_precision_at_20_max": 0.1801858221280134, "nauc_precision_at_20_std": -0.18286248889452628, "nauc_precision_at_3_diff1": 0.32329540590167366, "nauc_precision_at_3_max": 0.19220633029286657, "nauc_precision_at_3_std": -0.2677700359244656, "nauc_precision_at_5_diff1": 0.3184896312745215, "nauc_precision_at_5_max": 0.18225716187240096, "nauc_precision_at_5_std": -0.24637777822360407, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.3073481689515628, "nauc_recall_at_100_max": 0.3795704344039231, "nauc_recall_at_100_std": 0.12193816628411744, "nauc_recall_at_10_diff1": 0.44821276251784015, "nauc_recall_at_10_max": 0.19824435141573585, "nauc_recall_at_10_std": -0.264794242965765, "nauc_recall_at_1_diff1": 0.5243004316160466, "nauc_recall_at_1_max": 0.16858988412450363, "nauc_recall_at_1_std": -0.23630036197807808, "nauc_recall_at_20_diff1": 0.4150155740384653, "nauc_recall_at_20_max": 0.2175971000293212, "nauc_recall_at_20_std": -0.22208607862333246, "nauc_recall_at_3_diff1": 0.42772977991353756, "nauc_recall_at_3_max": 0.16622301552591512, "nauc_recall_at_3_std": -0.28349676394966694, "nauc_recall_at_5_diff1": 0.4383484768687609, "nauc_recall_at_5_max": 0.17066050173979333, "nauc_recall_at_5_std": -0.2648561618543115, "ndcg_at_1": 0.29511, "ndcg_at_10": 0.38131, "ndcg_at_100": 0.45497, "ndcg_at_1000": 0.47873, "ndcg_at_20": 0.40202, "ndcg_at_3": 0.32597, "ndcg_at_5": 0.34719, "precision_at_1": 0.29511, "precision_at_10": 0.07905, "precision_at_100": 0.0132, "precision_at_1000": 0.00156, "precision_at_20": 0.04534, "precision_at_3": 0.18145, "precision_at_5": 0.12752, "recall_at_1": 0.20805, "recall_at_10": 0.4963, "recall_at_100": 0.83022, "recall_at_1000": 1.0, "recall_at_20": 0.56929, "recall_at_3": 0.3486, "recall_at_5": 0.40194 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.36783, "map_at_1": 0.17171, "map_at_10": 0.30648, "map_at_100": 0.3248, "map_at_1000": 0.3265, "map_at_20": 0.31589, "map_at_3": 0.25825, "map_at_5": 0.28643, "mrr_at_1": 0.2966360856269113, "mrr_at_10": 0.37645927382165906, "mrr_at_100": 0.3879992762175121, "mrr_at_1000": 0.38875263914647545, "mrr_at_20": 0.3828819704394874, "mrr_at_3": 0.3509174311926606, "mrr_at_5": 0.36590214067278287, "nauc_map_at_1000_diff1": 0.39522495538335056, "nauc_map_at_1000_max": 0.04174976457405107, "nauc_map_at_1000_std": -0.31885078685550683, "nauc_map_at_100_diff1": 0.3946345133323068, "nauc_map_at_100_max": 0.04078098781556888, "nauc_map_at_100_std": -0.3195025671933243, "nauc_map_at_10_diff1": 0.3954986642658384, "nauc_map_at_10_max": 0.041271257279853146, "nauc_map_at_10_std": -0.3201810221507347, "nauc_map_at_1_diff1": 0.46018940673767755, "nauc_map_at_1_max": 0.026857409991088595, "nauc_map_at_1_std": -0.26072768511708483, "nauc_map_at_20_diff1": 0.39526425341235644, "nauc_map_at_20_max": 0.03852146604102061, "nauc_map_at_20_std": -0.3229992529439229, "nauc_map_at_3_diff1": 0.40751986930045464, "nauc_map_at_3_max": 0.033755935440305126, "nauc_map_at_3_std": -0.31104300522643485, "nauc_map_at_5_diff1": 0.39815137542719947, "nauc_map_at_5_max": 0.036140431875306425, "nauc_map_at_5_std": -0.320757402972474, "nauc_mrr_at_1000_diff1": 0.41138892539207805, "nauc_mrr_at_1000_max": 0.036075706028644856, "nauc_mrr_at_1000_std": -0.32158632796741027, "nauc_mrr_at_100_diff1": 0.4107983157690925, "nauc_mrr_at_100_max": 0.036285694743612026, "nauc_mrr_at_100_std": -0.32093532373076283, "nauc_mrr_at_10_diff1": 0.41046754117231543, "nauc_mrr_at_10_max": 0.03410885855306805, "nauc_mrr_at_10_std": -0.32625815315272727, "nauc_mrr_at_1_diff1": 0.466179363499551, "nauc_mrr_at_1_max": 0.044401491514074486, "nauc_mrr_at_1_std": -0.31994123134797725, "nauc_mrr_at_20_diff1": 0.41037316727398276, "nauc_mrr_at_20_max": 0.03439737123523601, "nauc_mrr_at_20_std": -0.32329135614933974, "nauc_mrr_at_3_diff1": 0.41952574843966106, "nauc_mrr_at_3_max": 0.029129685849109452, "nauc_mrr_at_3_std": -0.33463121553924624, "nauc_mrr_at_5_diff1": 0.414221864309102, "nauc_mrr_at_5_max": 0.033148923369830066, "nauc_mrr_at_5_std": -0.3306114597523587, "nauc_ndcg_at_1000_diff1": 0.38549772415494227, "nauc_ndcg_at_1000_max": 0.0514557026520757, "nauc_ndcg_at_1000_std": -0.30279726116120725, "nauc_ndcg_at_100_diff1": 0.3727602829287285, "nauc_ndcg_at_100_max": 0.048233930311151973, "nauc_ndcg_at_100_std": -0.2962119804155439, "nauc_ndcg_at_10_diff1": 0.38051043963062375, "nauc_ndcg_at_10_max": 0.036367319868345106, "nauc_ndcg_at_10_std": -0.32456955734935256, "nauc_ndcg_at_1_diff1": 0.466179363499551, "nauc_ndcg_at_1_max": 0.044401491514074486, "nauc_ndcg_at_1_std": -0.31994123134797725, "nauc_ndcg_at_20_diff1": 0.378712854202551, "nauc_ndcg_at_20_max": 0.03184188908580192, "nauc_ndcg_at_20_std": -0.32546078957376057, "nauc_ndcg_at_3_diff1": 0.39511029275208914, "nauc_ndcg_at_3_max": 0.03176054328577763, "nauc_ndcg_at_3_std": -0.32941163777526294, "nauc_ndcg_at_5_diff1": 0.38647610324783105, "nauc_ndcg_at_5_max": 0.029787744926969575, "nauc_ndcg_at_5_std": -0.32996941608936253, "nauc_precision_at_1000_diff1": 0.0006998751582486866, "nauc_precision_at_1000_max": 0.11825407208946465, "nauc_precision_at_1000_std": 0.01712018373173514, "nauc_precision_at_100_diff1": 0.0549092317048009, "nauc_precision_at_100_max": 0.09826849400810544, "nauc_precision_at_100_std": -0.057210573901148136, "nauc_precision_at_10_diff1": 0.19800849972948234, "nauc_precision_at_10_max": 0.0471972091041214, "nauc_precision_at_10_std": -0.2682010255326668, "nauc_precision_at_1_diff1": 0.466179363499551, "nauc_precision_at_1_max": 0.044401491514074486, "nauc_precision_at_1_std": -0.31994123134797725, "nauc_precision_at_20_diff1": 0.17420748368672398, "nauc_precision_at_20_max": 0.03308766045486042, "nauc_precision_at_20_std": -0.25054427932796136, "nauc_precision_at_3_diff1": 0.2861768860728149, "nauc_precision_at_3_max": 0.030311130047706455, "nauc_precision_at_3_std": -0.34534427655896854, "nauc_precision_at_5_diff1": 0.2317301049279226, "nauc_precision_at_5_max": 0.030641536817061336, "nauc_precision_at_5_std": -0.3203145309958511, "nauc_recall_at_1000_diff1": 0.604153036897657, "nauc_recall_at_1000_max": 0.49836049087161766, "nauc_recall_at_1000_std": 0.4914469784736325, "nauc_recall_at_100_diff1": 0.19683162305052707, "nauc_recall_at_100_max": 0.07559457900716732, "nauc_recall_at_100_std": -0.10709365405290024, "nauc_recall_at_10_diff1": 0.3041464761589614, "nauc_recall_at_10_max": 0.02771192688008908, "nauc_recall_at_10_std": -0.30107264533528405, "nauc_recall_at_1_diff1": 0.46018940673767755, "nauc_recall_at_1_max": 0.026857409991088595, "nauc_recall_at_1_std": -0.26072768511708483, "nauc_recall_at_20_diff1": 0.29055647974103144, "nauc_recall_at_20_max": 0.011652203062635029, "nauc_recall_at_20_std": -0.30092242080512877, "nauc_recall_at_3_diff1": 0.34551445060774777, "nauc_recall_at_3_max": 0.011961831015589949, "nauc_recall_at_3_std": -0.3023740962483907, "nauc_recall_at_5_diff1": 0.32472988425610233, "nauc_recall_at_5_max": 0.016604695108971373, "nauc_recall_at_5_std": -0.31547424015605674, "ndcg_at_1": 0.29664, "ndcg_at_10": 0.36783, "ndcg_at_100": 0.44805, "ndcg_at_1000": 0.47639, "ndcg_at_20": 0.39491, "ndcg_at_3": 0.30936, "ndcg_at_5": 0.33167, "precision_at_1": 0.29664, "precision_at_10": 0.09771, "precision_at_100": 0.01621, "precision_at_1000": 0.00195, "precision_at_20": 0.0578, "precision_at_3": 0.1998, "precision_at_5": 0.15443, "recall_at_1": 0.17171, "recall_at_10": 0.47018, "recall_at_100": 0.80877, "recall_at_1000": 0.99745, "recall_at_20": 0.56017, "recall_at_3": 0.30961, "recall_at_5": 0.38124 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.34224, "map_at_1": 0.17882, "map_at_10": 0.29102, "map_at_100": 0.30843, "map_at_1000": 0.30981, "map_at_20": 0.29978, "map_at_3": 0.25808, "map_at_5": 0.27623, "mrr_at_1": 0.25895765472312704, "mrr_at_10": 0.3347745721524222, "mrr_at_100": 0.34704620639973177, "mrr_at_1000": 0.34790416138574237, "mrr_at_20": 0.3413059184607473, "mrr_at_3": 0.31026058631921827, "mrr_at_5": 0.32491856677524417, "nauc_map_at_1000_diff1": 0.4442606266867698, "nauc_map_at_1000_max": 0.1708185281377609, "nauc_map_at_1000_std": -0.2150207849454721, "nauc_map_at_100_diff1": 0.44381219715728043, "nauc_map_at_100_max": 0.1707274710341615, "nauc_map_at_100_std": -0.21455708234165588, "nauc_map_at_10_diff1": 0.4436015785611014, "nauc_map_at_10_max": 0.16655090962737581, "nauc_map_at_10_std": -0.22490018596457073, "nauc_map_at_1_diff1": 0.5389347444540163, "nauc_map_at_1_max": 0.1515899453580535, "nauc_map_at_1_std": -0.1767816810758001, "nauc_map_at_20_diff1": 0.4430880291336973, "nauc_map_at_20_max": 0.16720914284848754, "nauc_map_at_20_std": -0.22236181609399264, "nauc_map_at_3_diff1": 0.4540839727614088, "nauc_map_at_3_max": 0.1654335659396773, "nauc_map_at_3_std": -0.21649157978042835, "nauc_map_at_5_diff1": 0.446028297739072, "nauc_map_at_5_max": 0.16686509996341534, "nauc_map_at_5_std": -0.22249139471925608, "nauc_mrr_at_1000_diff1": 0.47729996007831915, "nauc_mrr_at_1000_max": 0.18441282412110485, "nauc_mrr_at_1000_std": -0.20627941403812264, "nauc_mrr_at_100_diff1": 0.4770835727205064, "nauc_mrr_at_100_max": 0.18479601696449416, "nauc_mrr_at_100_std": -0.20545326120890625, "nauc_mrr_at_10_diff1": 0.4777617449135116, "nauc_mrr_at_10_max": 0.17994374717776154, "nauc_mrr_at_10_std": -0.21547996625752097, "nauc_mrr_at_1_diff1": 0.5068810551795434, "nauc_mrr_at_1_max": 0.18618278045779485, "nauc_mrr_at_1_std": -0.19506776109325624, "nauc_mrr_at_20_diff1": 0.47678901427838954, "nauc_mrr_at_20_max": 0.18417263509377577, "nauc_mrr_at_20_std": -0.2085117124654718, "nauc_mrr_at_3_diff1": 0.4844441192011046, "nauc_mrr_at_3_max": 0.1792234137967968, "nauc_mrr_at_3_std": -0.21664277015642558, "nauc_mrr_at_5_diff1": 0.47820491737651394, "nauc_mrr_at_5_max": 0.17845755566828503, "nauc_mrr_at_5_std": -0.2157898644431357, "nauc_ndcg_at_1000_diff1": 0.4481873693885092, "nauc_ndcg_at_1000_max": 0.18787360902505088, "nauc_ndcg_at_1000_std": -0.19316289425075395, "nauc_ndcg_at_100_diff1": 0.4394537819468697, "nauc_ndcg_at_100_max": 0.19350311354898503, "nauc_ndcg_at_100_std": -0.17390263754390844, "nauc_ndcg_at_10_diff1": 0.4402180150810218, "nauc_ndcg_at_10_max": 0.16592842467392824, "nauc_ndcg_at_10_std": -0.23244228557748658, "nauc_ndcg_at_1_diff1": 0.5133942160277655, "nauc_ndcg_at_1_max": 0.19111862143910074, "nauc_ndcg_at_1_std": -0.19122036203008314, "nauc_ndcg_at_20_diff1": 0.43690865520526906, "nauc_ndcg_at_20_max": 0.17219255964138427, "nauc_ndcg_at_20_std": -0.21841228017116848, "nauc_ndcg_at_3_diff1": 0.44994103943624364, "nauc_ndcg_at_3_max": 0.16271644024934, "nauc_ndcg_at_3_std": -0.2257117973448687, "nauc_ndcg_at_5_diff1": 0.4419982475303065, "nauc_ndcg_at_5_max": 0.16545612756504927, "nauc_ndcg_at_5_std": -0.229005873748153, "nauc_precision_at_1000_diff1": 0.02730691276697214, "nauc_precision_at_1000_max": 0.19545611505796542, "nauc_precision_at_1000_std": 0.13374115163843245, "nauc_precision_at_100_diff1": 0.10858753845867798, "nauc_precision_at_100_max": 0.22412725787013638, "nauc_precision_at_100_std": 0.11797913299580332, "nauc_precision_at_10_diff1": 0.24374020626851647, "nauc_precision_at_10_max": 0.13709507229772783, "nauc_precision_at_10_std": -0.19945109358534438, "nauc_precision_at_1_diff1": 0.5133942160277655, "nauc_precision_at_1_max": 0.19111862143910074, "nauc_precision_at_1_std": -0.19122036203008314, "nauc_precision_at_20_diff1": 0.21489905139680565, "nauc_precision_at_20_max": 0.14602555693961008, "nauc_precision_at_20_std": -0.13637072628292257, "nauc_precision_at_3_diff1": 0.3042491138905916, "nauc_precision_at_3_max": 0.1595855715203587, "nauc_precision_at_3_std": -0.2197706922055072, "nauc_precision_at_5_diff1": 0.2733108865826472, "nauc_precision_at_5_max": 0.14881048996436866, "nauc_precision_at_5_std": -0.21817734063690133, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.31997277429981313, "nauc_recall_at_100_max": 0.25904864898037366, "nauc_recall_at_100_std": 0.06524456028606167, "nauc_recall_at_10_diff1": 0.3817894506993761, "nauc_recall_at_10_max": 0.14522958635782565, "nauc_recall_at_10_std": -0.25094904036841065, "nauc_recall_at_1_diff1": 0.5389347444540163, "nauc_recall_at_1_max": 0.1515899453580535, "nauc_recall_at_1_std": -0.1767816810758001, "nauc_recall_at_20_diff1": 0.358863155772006, "nauc_recall_at_20_max": 0.16139314884864522, "nauc_recall_at_20_std": -0.2036131731964118, "nauc_recall_at_3_diff1": 0.40411661757221, "nauc_recall_at_3_max": 0.1393116422204287, "nauc_recall_at_3_std": -0.23447710699237967, "nauc_recall_at_5_diff1": 0.3856211181381593, "nauc_recall_at_5_max": 0.14440756509928318, "nauc_recall_at_5_std": -0.2416137241970891, "ndcg_at_1": 0.25733, "ndcg_at_10": 0.34224, "ndcg_at_100": 0.42225, "ndcg_at_1000": 0.45136, "ndcg_at_20": 0.36854, "ndcg_at_3": 0.28972, "ndcg_at_5": 0.31342, "precision_at_1": 0.25733, "precision_at_10": 0.0759, "precision_at_100": 0.01362, "precision_at_1000": 0.00167, "precision_at_20": 0.04552, "precision_at_3": 0.16721, "precision_at_5": 0.1228, "recall_at_1": 0.17882, "recall_at_10": 0.44671, "recall_at_100": 0.79306, "recall_at_1000": 1.0, "recall_at_20": 0.53776, "recall_at_3": 0.31319, "recall_at_5": 0.37015 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/model_meta.json ================================================ {"name": "dragonkue/BGE-m3-ko", "revision": "no_revision_available", "release_date": null, "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "loader": null} ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 151.6671278476715, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.67849, "map_at_1": 0.50877, "map_at_10": 0.62073, "map_at_100": 0.62583, "map_at_1000": 0.62602, "map_at_20": 0.62309, "map_at_3": 0.59357, "map_at_5": 0.6076, "mrr_at_1": 0.5087719298245614, "mrr_at_10": 0.6207323865218602, "mrr_at_100": 0.6258257568399235, "mrr_at_1000": 0.6260219632471632, "mrr_at_20": 0.623094651834264, "mrr_at_3": 0.5935672514619883, "mrr_at_5": 0.6076023391812866, "nauc_map_at_1000_diff1": 0.582288233790788, "nauc_map_at_1000_max": 0.23044371638953698, "nauc_map_at_1000_std": -0.41208734259159496, "nauc_map_at_100_diff1": 0.5817918027086326, "nauc_map_at_100_max": 0.2304667287527998, "nauc_map_at_100_std": -0.41215377561858174, "nauc_map_at_10_diff1": 0.5846846399924914, "nauc_map_at_10_max": 0.23401910272999615, "nauc_map_at_10_std": -0.4140334389430552, "nauc_map_at_1_diff1": 0.6090728823575914, "nauc_map_at_1_max": 0.21248155213502273, "nauc_map_at_1_std": -0.35903170112785254, "nauc_map_at_20_diff1": 0.5826702654390756, "nauc_map_at_20_max": 0.22948806438882577, "nauc_map_at_20_std": -0.4144789094616613, "nauc_map_at_3_diff1": 0.5671123042755822, "nauc_map_at_3_max": 0.23142735508369172, "nauc_map_at_3_std": -0.39674000031475426, "nauc_map_at_5_diff1": 0.5794706593498045, "nauc_map_at_5_max": 0.23881979426157995, "nauc_map_at_5_std": -0.39372881376091706, "nauc_mrr_at_1000_diff1": 0.582288233790788, "nauc_mrr_at_1000_max": 0.23044371638953698, "nauc_mrr_at_1000_std": -0.41208734259159496, "nauc_mrr_at_100_diff1": 0.5817918027086326, "nauc_mrr_at_100_max": 0.2304667287527998, "nauc_mrr_at_100_std": -0.41215377561858174, "nauc_mrr_at_10_diff1": 0.5846846399924914, "nauc_mrr_at_10_max": 0.23401910272999615, "nauc_mrr_at_10_std": -0.4140334389430552, "nauc_mrr_at_1_diff1": 0.6090728823575914, "nauc_mrr_at_1_max": 0.21248155213502273, "nauc_mrr_at_1_std": -0.35903170112785254, "nauc_mrr_at_20_diff1": 0.5826702654390756, "nauc_mrr_at_20_max": 0.22948806438882577, "nauc_mrr_at_20_std": -0.4144789094616613, "nauc_mrr_at_3_diff1": 0.5671123042755822, "nauc_mrr_at_3_max": 0.23142735508369172, "nauc_mrr_at_3_std": -0.39674000031475426, "nauc_mrr_at_5_diff1": 0.5794706593498045, "nauc_mrr_at_5_max": 0.23881979426157995, "nauc_mrr_at_5_std": -0.39372881376091706, "nauc_ndcg_at_1000_diff1": 0.5733224625227847, "nauc_ndcg_at_1000_max": 0.23028659532099577, "nauc_ndcg_at_1000_std": -0.4246935854927388, "nauc_ndcg_at_100_diff1": 0.5614942543371718, "nauc_ndcg_at_100_max": 0.2305352837673783, "nauc_ndcg_at_100_std": -0.42624394933307364, "nauc_ndcg_at_10_diff1": 0.57543116454026, "nauc_ndcg_at_10_max": 0.24002962690822346, "nauc_ndcg_at_10_std": -0.4453247064049292, "nauc_ndcg_at_1_diff1": 0.6090728823575914, "nauc_ndcg_at_1_max": 0.21248155213502273, "nauc_ndcg_at_1_std": -0.35903170112785254, "nauc_ndcg_at_20_diff1": 0.565557539829234, "nauc_ndcg_at_20_max": 0.2218272444517468, "nauc_ndcg_at_20_std": -0.4474161022413809, "nauc_ndcg_at_3_diff1": 0.5438811101297, "nauc_ndcg_at_3_max": 0.23675676717529345, "nauc_ndcg_at_3_std": -0.40301874051380876, "nauc_ndcg_at_5_diff1": 0.5668612106583857, "nauc_ndcg_at_5_max": 0.2502845900216014, "nauc_ndcg_at_5_std": -0.3985522286869084, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": -0.15663902232913457, "nauc_precision_at_100_max": 0.24095945650359982, "nauc_precision_at_100_std": -0.5227782238828026, "nauc_precision_at_10_diff1": 0.5200476725127159, "nauc_precision_at_10_max": 0.2734245435140121, "nauc_precision_at_10_std": -0.6530978486978932, "nauc_precision_at_1_diff1": 0.6090728823575914, "nauc_precision_at_1_max": 0.21248155213502273, "nauc_precision_at_1_std": -0.35903170112785254, "nauc_precision_at_20_diff1": 0.4073558080534852, "nauc_precision_at_20_max": 0.11918508664313493, "nauc_precision_at_20_std": -0.7296703090179871, "nauc_precision_at_3_diff1": 0.45878133440822105, "nauc_precision_at_3_max": 0.25499625716929286, "nauc_precision_at_3_std": -0.42238881381557775, "nauc_precision_at_5_diff1": 0.5161458610560639, "nauc_precision_at_5_max": 0.2988483896664812, "nauc_precision_at_5_std": -0.41512583759679444, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": -0.1566390223291421, "nauc_recall_at_100_max": 0.24095945650358844, "nauc_recall_at_100_std": -0.5227782238828191, "nauc_recall_at_10_diff1": 0.520047672512716, "nauc_recall_at_10_max": 0.2734245435140122, "nauc_recall_at_10_std": -0.6530978486978949, "nauc_recall_at_1_diff1": 0.6090728823575914, "nauc_recall_at_1_max": 0.21248155213502273, "nauc_recall_at_1_std": -0.35903170112785254, "nauc_recall_at_20_diff1": 0.40735580805348587, "nauc_recall_at_20_max": 0.11918508664313392, "nauc_recall_at_20_std": -0.729670309017987, "nauc_recall_at_3_diff1": 0.45878133440822066, "nauc_recall_at_3_max": 0.2549962571692935, "nauc_recall_at_3_std": -0.4223888138155778, "nauc_recall_at_5_diff1": 0.5161458610560639, "nauc_recall_at_5_max": 0.29884838966647986, "nauc_recall_at_5_std": -0.4151258375967945, "ndcg_at_1": 0.50877, "ndcg_at_10": 0.67849, "ndcg_at_100": 0.70322, "ndcg_at_1000": 0.70694, "ndcg_at_20": 0.68726, "ndcg_at_3": 0.62343, "ndcg_at_5": 0.64872, "precision_at_1": 0.50877, "precision_at_10": 0.08596, "precision_at_100": 0.00974, "precision_at_1000": 0.001, "precision_at_20": 0.04474, "precision_at_3": 0.23684, "precision_at_5": 0.15439, "recall_at_1": 0.50877, "recall_at_10": 0.85965, "recall_at_100": 0.97368, "recall_at_1000": 1.0, "recall_at_20": 0.89474, "recall_at_3": 0.71053, "recall_at_5": 0.77193 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 151.1522355079651, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.92403, "map_at_1": 0.87, "map_at_10": 0.9083, "map_at_100": 0.90944, "map_at_1000": 0.90947, "map_at_20": 0.90918, "map_at_3": 0.90241, "map_at_5": 0.90624, "mrr_at_1": 0.87, "mrr_at_10": 0.9082954144620813, "mrr_at_100": 0.909443837395513, "mrr_at_1000": 0.9094705266851508, "mrr_at_20": 0.9091781330268173, "mrr_at_3": 0.9024074074074075, "mrr_at_5": 0.9062407407407409, "nauc_map_at_1000_diff1": 0.9087102061067024, "nauc_map_at_1000_max": 0.766168221985831, "nauc_map_at_1000_std": -0.1648800489468126, "nauc_map_at_100_diff1": 0.9087248992727647, "nauc_map_at_100_max": 0.7662386636292916, "nauc_map_at_100_std": -0.16484587582543506, "nauc_map_at_10_diff1": 0.9083597819986478, "nauc_map_at_10_max": 0.7657665753111865, "nauc_map_at_10_std": -0.16615363444355752, "nauc_map_at_1_diff1": 0.9120154368987957, "nauc_map_at_1_max": 0.7576958206818232, "nauc_map_at_1_std": -0.1504110197733845, "nauc_map_at_20_diff1": 0.9088535062630146, "nauc_map_at_20_max": 0.7664545921257993, "nauc_map_at_20_std": -0.16440632717917356, "nauc_map_at_3_diff1": 0.9045838102108751, "nauc_map_at_3_max": 0.763024859106417, "nauc_map_at_3_std": -0.1770295998568004, "nauc_map_at_5_diff1": 0.9085555990659762, "nauc_map_at_5_max": 0.7647866209411698, "nauc_map_at_5_std": -0.17732191665098596, "nauc_mrr_at_1000_diff1": 0.9087102061067024, "nauc_mrr_at_1000_max": 0.766168221985831, "nauc_mrr_at_1000_std": -0.1648800489468126, "nauc_mrr_at_100_diff1": 0.9087248992727647, "nauc_mrr_at_100_max": 0.7662386636292916, "nauc_mrr_at_100_std": -0.16484587582543506, "nauc_mrr_at_10_diff1": 0.9083597819986478, "nauc_mrr_at_10_max": 0.7657665753111865, "nauc_mrr_at_10_std": -0.16615363444355752, "nauc_mrr_at_1_diff1": 0.9120154368987957, "nauc_mrr_at_1_max": 0.7576958206818232, "nauc_mrr_at_1_std": -0.1504110197733845, "nauc_mrr_at_20_diff1": 0.9088535062630146, "nauc_mrr_at_20_max": 0.7664545921257993, "nauc_mrr_at_20_std": -0.16440632717917356, "nauc_mrr_at_3_diff1": 0.9045838102108751, "nauc_mrr_at_3_max": 0.763024859106417, "nauc_mrr_at_3_std": -0.1770295998568004, "nauc_mrr_at_5_diff1": 0.9085555990659762, "nauc_mrr_at_5_max": 0.7647866209411698, "nauc_mrr_at_5_std": -0.17732191665098596, "nauc_ndcg_at_1000_diff1": 0.9089940551247231, "nauc_ndcg_at_1000_max": 0.7684755456124541, "nauc_ndcg_at_1000_std": -0.1634540891579336, "nauc_ndcg_at_100_diff1": 0.9094774523703856, "nauc_ndcg_at_100_max": 0.7704651683565917, "nauc_ndcg_at_100_std": -0.16088569042832504, "nauc_ndcg_at_10_diff1": 0.9084825910977311, "nauc_ndcg_at_10_max": 0.7694146916366845, "nauc_ndcg_at_10_std": -0.16401000307785144, "nauc_ndcg_at_1_diff1": 0.9120154368987957, "nauc_ndcg_at_1_max": 0.7576958206818232, "nauc_ndcg_at_1_std": -0.1504110197733845, "nauc_ndcg_at_20_diff1": 0.9102490741608852, "nauc_ndcg_at_20_max": 0.772394437957629, "nauc_ndcg_at_20_std": -0.1554085967649596, "nauc_ndcg_at_3_diff1": 0.9013487734702595, "nauc_ndcg_at_3_max": 0.7633185143220046, "nauc_ndcg_at_3_std": -0.1918571927274191, "nauc_ndcg_at_5_diff1": 0.9092867089203742, "nauc_ndcg_at_5_max": 0.7665885728237049, "nauc_ndcg_at_5_std": -0.19472790384402736, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9673202614378776, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.18557422969188173, "nauc_precision_at_10_diff1": 0.9115779645191425, "nauc_precision_at_10_max": 0.8047619047619075, "nauc_precision_at_10_std": -0.11591036414566304, "nauc_precision_at_1_diff1": 0.9120154368987957, "nauc_precision_at_1_max": 0.7576958206818232, "nauc_precision_at_1_std": -0.1504110197733845, "nauc_precision_at_20_diff1": 0.9416433239962602, "nauc_precision_at_20_max": 0.8837868480725527, "nauc_precision_at_20_std": 0.08346672002133163, "nauc_precision_at_3_diff1": 0.8865719127157048, "nauc_precision_at_3_max": 0.763685721202064, "nauc_precision_at_3_std": -0.2611439637583455, "nauc_precision_at_5_diff1": 0.9160860925566805, "nauc_precision_at_5_max": 0.776628600158008, "nauc_precision_at_5_std": -0.30469965764083523, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9673202614379001, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.18557422969189222, "nauc_recall_at_10_diff1": 0.9115779645191382, "nauc_recall_at_10_max": 0.8047619047619022, "nauc_recall_at_10_std": -0.1159103641456544, "nauc_recall_at_1_diff1": 0.9120154368987957, "nauc_recall_at_1_max": 0.7576958206818232, "nauc_recall_at_1_std": -0.1504110197733845, "nauc_recall_at_20_diff1": 0.9416433239962608, "nauc_recall_at_20_max": 0.8837868480725666, "nauc_recall_at_20_std": 0.08346672002134754, "nauc_recall_at_3_diff1": 0.8865719127157022, "nauc_recall_at_3_max": 0.7636857212020612, "nauc_recall_at_3_std": -0.2611439637583427, "nauc_recall_at_5_diff1": 0.9160860925566815, "nauc_recall_at_5_max": 0.7766286001580116, "nauc_recall_at_5_std": -0.3046996576408323, "ndcg_at_1": 0.87, "ndcg_at_10": 0.92403, "ndcg_at_100": 0.92919, "ndcg_at_1000": 0.92979, "ndcg_at_20": 0.92716, "ndcg_at_3": 0.91213, "ndcg_at_5": 0.91901, "precision_at_1": 0.87, "precision_at_10": 0.09722, "precision_at_100": 0.00996, "precision_at_1000": 0.001, "precision_at_20": 0.04922, "precision_at_3": 0.31333, "precision_at_5": 0.19133, "recall_at_1": 0.87, "recall_at_10": 0.97222, "recall_at_100": 0.99556, "recall_at_1000": 1.0, "recall_at_20": 0.98444, "recall_at_3": 0.94, "recall_at_5": 0.95667 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.9245, "map_at_1": 0.86556, "map_at_10": 0.90772, "map_at_100": 0.90891, "map_at_1000": 0.90892, "map_at_20": 0.90867, "map_at_3": 0.90167, "map_at_5": 0.90572, "mrr_at_1": 0.8655555555555555, "mrr_at_10": 0.9077182539682542, "mrr_at_100": 0.9089099946621724, "mrr_at_1000": 0.9089200667955346, "mrr_at_20": 0.9086737418355068, "mrr_at_3": 0.9016666666666668, "mrr_at_5": 0.9057222222222224, "nauc_map_at_1000_diff1": 0.8907741421197661, "nauc_map_at_1000_max": 0.6626465117689059, "nauc_map_at_1000_std": -0.317136109087687, "nauc_map_at_100_diff1": 0.8907816742896304, "nauc_map_at_100_max": 0.6626847593795272, "nauc_map_at_100_std": -0.31699708730779125, "nauc_map_at_10_diff1": 0.890265368178002, "nauc_map_at_10_max": 0.664564904987638, "nauc_map_at_10_std": -0.3134820091469261, "nauc_map_at_1_diff1": 0.8942004455054253, "nauc_map_at_1_max": 0.6266975912367964, "nauc_map_at_1_std": -0.3639052782013713, "nauc_map_at_20_diff1": 0.8906411175863109, "nauc_map_at_20_max": 0.6628476087242807, "nauc_map_at_20_std": -0.31769612937316705, "nauc_map_at_3_diff1": 0.8925822549268371, "nauc_map_at_3_max": 0.6738326485879481, "nauc_map_at_3_std": -0.3032972154887417, "nauc_map_at_5_diff1": 0.8914108120590107, "nauc_map_at_5_max": 0.6684705992434418, "nauc_map_at_5_std": -0.3110940068516004, "nauc_mrr_at_1000_diff1": 0.8907741421197661, "nauc_mrr_at_1000_max": 0.6626465117689059, "nauc_mrr_at_1000_std": -0.317136109087687, "nauc_mrr_at_100_diff1": 0.8907816742896304, "nauc_mrr_at_100_max": 0.6626847593795272, "nauc_mrr_at_100_std": -0.31699708730779125, "nauc_mrr_at_10_diff1": 0.890265368178002, "nauc_mrr_at_10_max": 0.664564904987638, "nauc_mrr_at_10_std": -0.3134820091469261, "nauc_mrr_at_1_diff1": 0.8942004455054253, "nauc_mrr_at_1_max": 0.6266975912367964, "nauc_mrr_at_1_std": -0.3639052782013713, "nauc_mrr_at_20_diff1": 0.8906411175863109, "nauc_mrr_at_20_max": 0.6628476087242807, "nauc_mrr_at_20_std": -0.31769612937316705, "nauc_mrr_at_3_diff1": 0.8925822549268371, "nauc_mrr_at_3_max": 0.6738326485879481, "nauc_mrr_at_3_std": -0.3032972154887417, "nauc_mrr_at_5_diff1": 0.8914108120590107, "nauc_mrr_at_5_max": 0.6684705992434418, "nauc_mrr_at_5_std": -0.3110940068516004, "nauc_ndcg_at_1000_diff1": 0.8903604370829069, "nauc_ndcg_at_1000_max": 0.6680529698253163, "nauc_ndcg_at_1000_std": -0.3088091221018859, "nauc_ndcg_at_100_diff1": 0.8905603260103219, "nauc_ndcg_at_100_max": 0.6694191305565131, "nauc_ndcg_at_100_std": -0.30395669406253056, "nauc_ndcg_at_10_diff1": 0.8880903746847338, "nauc_ndcg_at_10_max": 0.6771885807878589, "nauc_ndcg_at_10_std": -0.2933321488974208, "nauc_ndcg_at_1_diff1": 0.8942004455054253, "nauc_ndcg_at_1_max": 0.6266975912367964, "nauc_ndcg_at_1_std": -0.3639052782013713, "nauc_ndcg_at_20_diff1": 0.8894301275693007, "nauc_ndcg_at_20_max": 0.6705724282044567, "nauc_ndcg_at_20_std": -0.3098295084689656, "nauc_ndcg_at_3_diff1": 0.8933950544996908, "nauc_ndcg_at_3_max": 0.6970050368317691, "nauc_ndcg_at_3_std": -0.2711580447806635, "nauc_ndcg_at_5_diff1": 0.8912467426056183, "nauc_ndcg_at_5_max": 0.6876593267266466, "nauc_ndcg_at_5_std": -0.28664735777263556, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9346405228757582, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.8611111111111238, "nauc_precision_at_10_diff1": 0.8628299804770346, "nauc_precision_at_10_max": 0.7927383074441847, "nauc_precision_at_10_std": -0.10024615906969267, "nauc_precision_at_1_diff1": 0.8942004455054253, "nauc_precision_at_1_max": 0.6266975912367964, "nauc_precision_at_1_std": -0.3639052782013713, "nauc_precision_at_20_diff1": 0.8648247177658893, "nauc_precision_at_20_max": 0.7647483235718557, "nauc_precision_at_20_std": -0.26215940921825764, "nauc_precision_at_3_diff1": 0.8981252631771672, "nauc_precision_at_3_max": 0.8105582102121879, "nauc_precision_at_3_std": -0.11257574925395346, "nauc_precision_at_5_diff1": 0.8909030278778126, "nauc_precision_at_5_max": 0.8157663065226051, "nauc_precision_at_5_std": -0.12546351874082617, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.934640522875857, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.8611111111111373, "nauc_recall_at_10_diff1": 0.8628299804770414, "nauc_recall_at_10_max": 0.7927383074441877, "nauc_recall_at_10_std": -0.10024615906969253, "nauc_recall_at_1_diff1": 0.8942004455054253, "nauc_recall_at_1_max": 0.6266975912367964, "nauc_recall_at_1_std": -0.3639052782013713, "nauc_recall_at_20_diff1": 0.8648247177658958, "nauc_recall_at_20_max": 0.764748323571863, "nauc_recall_at_20_std": -0.26215940921823416, "nauc_recall_at_3_diff1": 0.8981252631771675, "nauc_recall_at_3_max": 0.8105582102121894, "nauc_recall_at_3_std": -0.11257574925395204, "nauc_recall_at_5_diff1": 0.8909030278778138, "nauc_recall_at_5_max": 0.8157663065226111, "nauc_recall_at_5_std": -0.1254635187408309, "ndcg_at_1": 0.86556, "ndcg_at_10": 0.9245, "ndcg_at_100": 0.92955, "ndcg_at_1000": 0.92983, "ndcg_at_20": 0.92773, "ndcg_at_3": 0.91245, "ndcg_at_5": 0.91976, "precision_at_1": 0.86556, "precision_at_10": 0.09756, "precision_at_100": 0.00998, "precision_at_1000": 0.001, "precision_at_20": 0.04939, "precision_at_3": 0.31444, "precision_at_5": 0.19222, "recall_at_1": 0.86556, "recall_at_10": 0.97556, "recall_at_100": 0.99778, "recall_at_1000": 1.0, "recall_at_20": 0.98778, "recall_at_3": 0.94333, "recall_at_5": 0.96111 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.84232, "map_at_1": 0.74333, "map_at_10": 0.81242, "map_at_100": 0.81501, "map_at_1000": 0.81505, "map_at_20": 0.81411, "map_at_3": 0.80111, "map_at_5": 0.8075, "mrr_at_1": 0.7433333333333333, "mrr_at_10": 0.8124166666666669, "mrr_at_100": 0.8150097968662406, "mrr_at_1000": 0.8150519412628603, "mrr_at_20": 0.8141131871033833, "mrr_at_3": 0.8011111111111111, "mrr_at_5": 0.8074999999999999, "nauc_map_at_1000_diff1": 0.7479635701130208, "nauc_map_at_1000_max": 0.7868132208331684, "nauc_map_at_1000_std": 0.2267387051745361, "nauc_map_at_100_diff1": 0.7479575405061689, "nauc_map_at_100_max": 0.7868157515318333, "nauc_map_at_100_std": 0.2269098553049622, "nauc_map_at_10_diff1": 0.7476594262177979, "nauc_map_at_10_max": 0.7872075996532419, "nauc_map_at_10_std": 0.2254532093832398, "nauc_map_at_1_diff1": 0.7735585317934608, "nauc_map_at_1_max": 0.7814582351561841, "nauc_map_at_1_std": 0.20220053408369135, "nauc_map_at_20_diff1": 0.747521918347692, "nauc_map_at_20_max": 0.7868505598851331, "nauc_map_at_20_std": 0.22798988268232043, "nauc_map_at_3_diff1": 0.743714669014021, "nauc_map_at_3_max": 0.7877175737221868, "nauc_map_at_3_std": 0.21726579839289906, "nauc_map_at_5_diff1": 0.7453309061226726, "nauc_map_at_5_max": 0.787680630486119, "nauc_map_at_5_std": 0.22214427481440316, "nauc_mrr_at_1000_diff1": 0.7479635701130208, "nauc_mrr_at_1000_max": 0.7868132208331684, "nauc_mrr_at_1000_std": 0.2267387051745361, "nauc_mrr_at_100_diff1": 0.7479575405061689, "nauc_mrr_at_100_max": 0.7868157515318333, "nauc_mrr_at_100_std": 0.2269098553049622, "nauc_mrr_at_10_diff1": 0.7476594262177979, "nauc_mrr_at_10_max": 0.7872075996532419, "nauc_mrr_at_10_std": 0.2254532093832398, "nauc_mrr_at_1_diff1": 0.7735585317934608, "nauc_mrr_at_1_max": 0.7814582351561841, "nauc_mrr_at_1_std": 0.20220053408369135, "nauc_mrr_at_20_diff1": 0.747521918347692, "nauc_mrr_at_20_max": 0.7868505598851331, "nauc_mrr_at_20_std": 0.22798988268232043, "nauc_mrr_at_3_diff1": 0.743714669014021, "nauc_mrr_at_3_max": 0.7877175737221868, "nauc_mrr_at_3_std": 0.21726579839289906, "nauc_mrr_at_5_diff1": 0.7453309061226726, "nauc_mrr_at_5_max": 0.787680630486119, "nauc_mrr_at_5_std": 0.22214427481440316, "nauc_ndcg_at_1000_diff1": 0.7442360510715994, "nauc_ndcg_at_1000_max": 0.7868278968005287, "nauc_ndcg_at_1000_std": 0.23488350314372797, "nauc_ndcg_at_100_diff1": 0.7436547317586959, "nauc_ndcg_at_100_max": 0.7866619637977311, "nauc_ndcg_at_100_std": 0.24083458929790327, "nauc_ndcg_at_10_diff1": 0.7406837295936815, "nauc_ndcg_at_10_max": 0.7880027249441579, "nauc_ndcg_at_10_std": 0.23848502503860292, "nauc_ndcg_at_1_diff1": 0.7735585317934608, "nauc_ndcg_at_1_max": 0.7814582351561841, "nauc_ndcg_at_1_std": 0.20220053408369135, "nauc_ndcg_at_20_diff1": 0.739852926566927, "nauc_ndcg_at_20_max": 0.7864134670603612, "nauc_ndcg_at_20_std": 0.24891095256291795, "nauc_ndcg_at_3_diff1": 0.7327962541689025, "nauc_ndcg_at_3_max": 0.7895606209040222, "nauc_ndcg_at_3_std": 0.2206452903850603, "nauc_ndcg_at_5_diff1": 0.7351737326652631, "nauc_ndcg_at_5_max": 0.7889553644488616, "nauc_ndcg_at_5_std": 0.23093678509825147, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.6659663865546206, "nauc_precision_at_100_max": 0.7610877684407207, "nauc_precision_at_100_std": 0.9305555555555604, "nauc_precision_at_10_diff1": 0.6949706436246802, "nauc_precision_at_10_max": 0.7916726012438878, "nauc_precision_at_10_std": 0.3400750130560682, "nauc_precision_at_1_diff1": 0.7735585317934608, "nauc_precision_at_1_max": 0.7814582351561841, "nauc_precision_at_1_std": 0.20220053408369135, "nauc_precision_at_20_diff1": 0.6595041525382113, "nauc_precision_at_20_max": 0.7726792471374534, "nauc_precision_at_20_std": 0.5227775320654573, "nauc_precision_at_3_diff1": 0.6878544893694668, "nauc_precision_at_3_max": 0.7969252711532735, "nauc_precision_at_3_std": 0.2334986235032675, "nauc_precision_at_5_diff1": 0.6849258221807228, "nauc_precision_at_5_max": 0.7941280215790011, "nauc_precision_at_5_std": 0.27671438945948545, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.6659663865546264, "nauc_recall_at_100_max": 0.7610877684407182, "nauc_recall_at_100_std": 0.9305555555555559, "nauc_recall_at_10_diff1": 0.6949706436246803, "nauc_recall_at_10_max": 0.7916726012438876, "nauc_recall_at_10_std": 0.34007501305606896, "nauc_recall_at_1_diff1": 0.7735585317934608, "nauc_recall_at_1_max": 0.7814582351561841, "nauc_recall_at_1_std": 0.20220053408369135, "nauc_recall_at_20_diff1": 0.6595041525382099, "nauc_recall_at_20_max": 0.7726792471374523, "nauc_recall_at_20_std": 0.5227775320654581, "nauc_recall_at_3_diff1": 0.6878544893694643, "nauc_recall_at_3_max": 0.7969252711532728, "nauc_recall_at_3_std": 0.23349862350326753, "nauc_recall_at_5_diff1": 0.6849258221807234, "nauc_recall_at_5_max": 0.7941280215790028, "nauc_recall_at_5_std": 0.27671438945948845, "ndcg_at_1": 0.74333, "ndcg_at_10": 0.84232, "ndcg_at_100": 0.85458, "ndcg_at_1000": 0.85573, "ndcg_at_20": 0.84832, "ndcg_at_3": 0.81915, "ndcg_at_5": 0.83086, "precision_at_1": 0.74333, "precision_at_10": 0.09344, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "precision_at_20": 0.04789, "precision_at_3": 0.29037, "precision_at_5": 0.18, "recall_at_1": 0.74333, "recall_at_10": 0.93444, "recall_at_100": 0.99111, "recall_at_1000": 1.0, "recall_at_20": 0.95778, "recall_at_3": 0.87111, "recall_at_5": 0.9 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 1837.136996269226, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.79317, "map_at_1": 0.49126, "map_at_10": 0.7495, "map_at_100": 0.75455, "map_at_1000": 0.75486, "map_at_20": 0.7525, "map_at_3": 0.70867, "map_at_5": 0.73638, "mrr_at_1": 0.7584459459459459, "mrr_at_10": 0.8100875429000428, "mrr_at_100": 0.8117765496291576, "mrr_at_1000": 0.8119327347214493, "mrr_at_20": 0.8106196684321683, "mrr_at_3": 0.8037725225225222, "mrr_at_5": 0.8077421171171169, "nauc_map_at_1000_diff1": 0.5812617698951515, "nauc_map_at_1000_max": 0.6610692477766553, "nauc_map_at_1000_std": 0.27404270545327336, "nauc_map_at_100_diff1": 0.5811909048005637, "nauc_map_at_100_max": 0.6613587030835343, "nauc_map_at_100_std": 0.27427743469134663, "nauc_map_at_10_diff1": 0.5789105274892585, "nauc_map_at_10_max": 0.6596737849213944, "nauc_map_at_10_std": 0.2680503647538944, "nauc_map_at_1_diff1": 0.579615004172223, "nauc_map_at_1_max": 0.2908252007229241, "nauc_map_at_1_std": 0.034824910006952516, "nauc_map_at_20_diff1": 0.5815683585596179, "nauc_map_at_20_max": 0.6629871010695796, "nauc_map_at_20_std": 0.2750654700270136, "nauc_map_at_3_diff1": 0.5702846943143143, "nauc_map_at_3_max": 0.6094753352459681, "nauc_map_at_3_std": 0.2032967213655951, "nauc_map_at_5_diff1": 0.5800945455348301, "nauc_map_at_5_max": 0.6508414550197733, "nauc_map_at_5_std": 0.23763912270562051, "nauc_mrr_at_1000_diff1": 0.6731537330578673, "nauc_mrr_at_1000_max": 0.7153241852891469, "nauc_mrr_at_1000_std": 0.3516402936461557, "nauc_mrr_at_100_diff1": 0.6731642004771984, "nauc_mrr_at_100_max": 0.7155000144086914, "nauc_mrr_at_100_std": 0.35196033305591073, "nauc_mrr_at_10_diff1": 0.6734370672138152, "nauc_mrr_at_10_max": 0.7169363494428644, "nauc_mrr_at_10_std": 0.35223055736252545, "nauc_mrr_at_1_diff1": 0.66972023486555, "nauc_mrr_at_1_max": 0.6301506150713375, "nauc_mrr_at_1_std": 0.2455626983258168, "nauc_mrr_at_20_diff1": 0.6736684272381803, "nauc_mrr_at_20_max": 0.716476673430364, "nauc_mrr_at_20_std": 0.35288033831367044, "nauc_mrr_at_3_diff1": 0.6743708966504051, "nauc_mrr_at_3_max": 0.7240735657936037, "nauc_mrr_at_3_std": 0.35119546746688735, "nauc_mrr_at_5_diff1": 0.6745833502417148, "nauc_mrr_at_5_max": 0.720000059937184, "nauc_mrr_at_5_std": 0.3557423062678047, "nauc_ndcg_at_1000_diff1": 0.6070344828856682, "nauc_ndcg_at_1000_max": 0.706969512634442, "nauc_ndcg_at_1000_std": 0.34814483086269804, "nauc_ndcg_at_100_diff1": 0.6041752800901181, "nauc_ndcg_at_100_max": 0.713810430485531, "nauc_ndcg_at_100_std": 0.3561084263782227, "nauc_ndcg_at_10_diff1": 0.6002180308546468, "nauc_ndcg_at_10_max": 0.7147996091069331, "nauc_ndcg_at_10_std": 0.34075062854849214, "nauc_ndcg_at_1_diff1": 0.66972023486555, "nauc_ndcg_at_1_max": 0.6301506150713375, "nauc_ndcg_at_1_std": 0.2455626983258168, "nauc_ndcg_at_20_diff1": 0.6065870330170832, "nauc_ndcg_at_20_max": 0.7221616714472898, "nauc_ndcg_at_20_std": 0.35960801191696007, "nauc_ndcg_at_3_diff1": 0.5893874932024313, "nauc_ndcg_at_3_max": 0.6636957047260216, "nauc_ndcg_at_3_std": 0.2543432449264493, "nauc_ndcg_at_5_diff1": 0.6028280570221642, "nauc_ndcg_at_5_max": 0.7012799112010517, "nauc_ndcg_at_5_std": 0.2939975776821263, "nauc_precision_at_1000_diff1": -0.2810181647289382, "nauc_precision_at_1000_max": 0.04148207954816594, "nauc_precision_at_1000_std": 0.19379188361125566, "nauc_precision_at_100_diff1": -0.196651209135289, "nauc_precision_at_100_max": 0.17030537047692557, "nauc_precision_at_100_std": 0.271797207506889, "nauc_precision_at_10_diff1": -0.08287152921145098, "nauc_precision_at_10_max": 0.3031368102620088, "nauc_precision_at_10_std": 0.30266327719783204, "nauc_precision_at_1_diff1": 0.66972023486555, "nauc_precision_at_1_max": 0.6301506150713375, "nauc_precision_at_1_std": 0.2455626983258168, "nauc_precision_at_20_diff1": -0.10296139278278182, "nauc_precision_at_20_max": 0.27403214258088565, "nauc_precision_at_20_std": 0.3105317173548736, "nauc_precision_at_3_diff1": 0.05444938356841716, "nauc_precision_at_3_max": 0.43831031607539994, "nauc_precision_at_3_std": 0.2764742828725262, "nauc_precision_at_5_diff1": -0.01105899214358065, "nauc_precision_at_5_max": 0.38274325037052775, "nauc_precision_at_5_std": 0.28187720011150214, "nauc_recall_at_1000_diff1": 0.632396735941214, "nauc_recall_at_1000_max": 0.9304343829172986, "nauc_recall_at_1000_std": 0.8702336162445697, "nauc_recall_at_100_diff1": 0.5680963222825306, "nauc_recall_at_100_max": 0.8878032034318802, "nauc_recall_at_100_std": 0.6909240094984084, "nauc_recall_at_10_diff1": 0.5688992428586632, "nauc_recall_at_10_max": 0.8248338917311779, "nauc_recall_at_10_std": 0.4849807476605251, "nauc_recall_at_1_diff1": 0.579615004172223, "nauc_recall_at_1_max": 0.2908252007229241, "nauc_recall_at_1_std": 0.034824910006952516, "nauc_recall_at_20_diff1": 0.5974109466534318, "nauc_recall_at_20_max": 0.87467624395276, "nauc_recall_at_20_std": 0.5901505366736927, "nauc_recall_at_3_diff1": 0.5567183508386466, "nauc_recall_at_3_max": 0.6738919858737272, "nauc_recall_at_3_std": 0.25640288050963517, "nauc_recall_at_5_diff1": 0.5761474687351439, "nauc_recall_at_5_max": 0.7635092315346151, "nauc_recall_at_5_std": 0.3351164969852482, "ndcg_at_1": 0.75845, "ndcg_at_10": 0.79317, "ndcg_at_100": 0.81138, "ndcg_at_1000": 0.81813, "ndcg_at_20": 0.79982, "ndcg_at_3": 0.74868, "ndcg_at_5": 0.77519, "precision_at_1": 0.75845, "precision_at_10": 0.15828, "precision_at_100": 0.0174, "precision_at_1000": 0.00185, "precision_at_20": 0.08167, "precision_at_3": 0.44764, "precision_at_5": 0.29493, "recall_at_1": 0.49126, "recall_at_10": 0.85157, "recall_at_100": 0.92359, "recall_at_1000": 0.96523, "recall_at_20": 0.87176, "recall_at_3": 0.7543, "recall_at_5": 0.81055 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 133762.75083470345, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.58709, "map_at_1": 0.3504, "map_at_10": 0.49741, "map_at_100": 0.5241, "map_at_1000": 0.52528, "map_at_20": 0.51372, "map_at_3": 0.4474, "map_at_5": 0.47131, "mrr_at_1": 0.5774647887323944, "mrr_at_10": 0.6715887920113273, "mrr_at_100": 0.6762726144876823, "mrr_at_1000": 0.6764729104287056, "mrr_at_20": 0.6755270044236817, "mrr_at_3": 0.6494522691705791, "mrr_at_5": 0.6621283255086072, "nauc_map_at_1000_diff1": 0.34120094391499245, "nauc_map_at_1000_max": 0.366838210001104, "nauc_map_at_1000_std": 0.04078157589429166, "nauc_map_at_100_diff1": 0.34052366255683186, "nauc_map_at_100_max": 0.36606971433584523, "nauc_map_at_100_std": 0.039985295949039344, "nauc_map_at_10_diff1": 0.33104969057528355, "nauc_map_at_10_max": 0.35793214492098313, "nauc_map_at_10_std": 0.007120775723603169, "nauc_map_at_1_diff1": 0.36372917871224114, "nauc_map_at_1_max": 0.31168079591945574, "nauc_map_at_1_std": -0.11011005080540842, "nauc_map_at_20_diff1": 0.33556661225722967, "nauc_map_at_20_max": 0.3674680413455777, "nauc_map_at_20_std": 0.026355815955895336, "nauc_map_at_3_diff1": 0.34184935253502124, "nauc_map_at_3_max": 0.3434521708155092, "nauc_map_at_3_std": -0.010455588172976739, "nauc_map_at_5_diff1": 0.3254317733853242, "nauc_map_at_5_max": 0.34801016273643837, "nauc_map_at_5_std": -0.004311953264131259, "nauc_mrr_at_1000_diff1": 0.40435244338557286, "nauc_mrr_at_1000_max": 0.4287767488366848, "nauc_mrr_at_1000_std": 0.1619679098837795, "nauc_mrr_at_100_diff1": 0.404405312972269, "nauc_mrr_at_100_max": 0.42907346032870647, "nauc_mrr_at_100_std": 0.16233412877435102, "nauc_mrr_at_10_diff1": 0.4065372557651823, "nauc_mrr_at_10_max": 0.42877016983432525, "nauc_mrr_at_10_std": 0.16253075319632765, "nauc_mrr_at_1_diff1": 0.3828166786427259, "nauc_mrr_at_1_max": 0.3845585131308482, "nauc_mrr_at_1_std": 0.0617774532017859, "nauc_mrr_at_20_diff1": 0.4048662753915902, "nauc_mrr_at_20_max": 0.42937621763492295, "nauc_mrr_at_20_std": 0.1618932534605349, "nauc_mrr_at_3_diff1": 0.413269450193848, "nauc_mrr_at_3_max": 0.4276309165768991, "nauc_mrr_at_3_std": 0.16769288191570964, "nauc_mrr_at_5_diff1": 0.403247052594961, "nauc_mrr_at_5_max": 0.41734670566846804, "nauc_mrr_at_5_std": 0.1576369649565576, "nauc_ndcg_at_1000_diff1": 0.3572118952727939, "nauc_ndcg_at_1000_max": 0.4075069870231915, "nauc_ndcg_at_1000_std": 0.14022543880428157, "nauc_ndcg_at_100_diff1": 0.348312083182717, "nauc_ndcg_at_100_max": 0.4025977066232255, "nauc_ndcg_at_100_std": 0.13916190476906262, "nauc_ndcg_at_10_diff1": 0.33986492065971163, "nauc_ndcg_at_10_max": 0.3850661195073111, "nauc_ndcg_at_10_std": 0.0635710202558746, "nauc_ndcg_at_1_diff1": 0.3828166786427259, "nauc_ndcg_at_1_max": 0.3845585131308482, "nauc_ndcg_at_1_std": 0.0617774532017859, "nauc_ndcg_at_20_diff1": 0.3414557704155679, "nauc_ndcg_at_20_max": 0.40553747002915236, "nauc_ndcg_at_20_std": 0.09985925201518564, "nauc_ndcg_at_3_diff1": 0.3870890812358928, "nauc_ndcg_at_3_max": 0.3846874567549218, "nauc_ndcg_at_3_std": 0.12185414238392578, "nauc_ndcg_at_5_diff1": 0.3490173675040339, "nauc_ndcg_at_5_max": 0.3654450194590964, "nauc_ndcg_at_5_std": 0.07813419411057466, "nauc_precision_at_1000_diff1": -0.0032435172501387873, "nauc_precision_at_1000_max": 0.0007242613045474486, "nauc_precision_at_1000_std": 0.35043717667409324, "nauc_precision_at_100_diff1": 0.0016471763775988892, "nauc_precision_at_100_max": 0.01804901150100379, "nauc_precision_at_100_std": 0.36596154349162197, "nauc_precision_at_10_diff1": 0.05358186698251174, "nauc_precision_at_10_max": 0.15257889925441642, "nauc_precision_at_10_std": 0.3143054303042554, "nauc_precision_at_1_diff1": 0.3828166786427259, "nauc_precision_at_1_max": 0.3845585131308482, "nauc_precision_at_1_std": 0.0617774532017859, "nauc_precision_at_20_diff1": 0.02615997482157888, "nauc_precision_at_20_max": 0.11229379876131261, "nauc_precision_at_20_std": 0.3518356666804108, "nauc_precision_at_3_diff1": 0.18578823830575417, "nauc_precision_at_3_max": 0.2644100183947296, "nauc_precision_at_3_std": 0.3200450616684121, "nauc_precision_at_5_diff1": 0.06648526125065941, "nauc_precision_at_5_max": 0.19948594661617416, "nauc_precision_at_5_std": 0.30423437963537375, "nauc_recall_at_1000_diff1": 0.1751491317505002, "nauc_recall_at_1000_max": 0.6245630358343841, "nauc_recall_at_1000_std": 0.6668652808132978, "nauc_recall_at_100_diff1": 0.16490824289546413, "nauc_recall_at_100_max": 0.41167715005311006, "nauc_recall_at_100_std": 0.3490494435460939, "nauc_recall_at_10_diff1": 0.24803168616915408, "nauc_recall_at_10_max": 0.3346639265012337, "nauc_recall_at_10_std": 0.04329675173685037, "nauc_recall_at_1_diff1": 0.36372917871224114, "nauc_recall_at_1_max": 0.31168079591945574, "nauc_recall_at_1_std": -0.11011005080540842, "nauc_recall_at_20_diff1": 0.2127800805100239, "nauc_recall_at_20_max": 0.3910800445068547, "nauc_recall_at_20_std": 0.13021536861777655, "nauc_recall_at_3_diff1": 0.3307505105858759, "nauc_recall_at_3_max": 0.3423307880427305, "nauc_recall_at_3_std": 0.0678675786450557, "nauc_recall_at_5_diff1": 0.2785049573246195, "nauc_recall_at_5_max": 0.3273306187782061, "nauc_recall_at_5_std": 0.0512487582690908, "ndcg_at_1": 0.57746, "ndcg_at_10": 0.58709, "ndcg_at_100": 0.65388, "ndcg_at_1000": 0.66799, "ndcg_at_20": 0.62054, "ndcg_at_3": 0.55051, "ndcg_at_5": 0.5543, "precision_at_1": 0.57746, "precision_at_10": 0.14695, "precision_at_100": 0.02258, "precision_at_1000": 0.00251, "precision_at_20": 0.0892, "precision_at_3": 0.32394, "precision_at_5": 0.23474, "recall_at_1": 0.3504, "recall_at_10": 0.6752, "recall_at_100": 0.88583, "recall_at_1000": 0.96565, "recall_at_20": 0.76951, "recall_at_3": 0.50691, "recall_at_5": 0.56791 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 132131.55453968048, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.52444, "map_at_1": 0.33729, "map_at_10": 0.45818, "map_at_100": 0.46723, "map_at_1000": 0.46768, "map_at_20": 0.46381, "map_at_3": 0.42214, "map_at_5": 0.44596, "mrr_at_1": 0.3657957244655582, "mrr_at_10": 0.4820891301888926, "mrr_at_100": 0.48851420018324104, "mrr_at_1000": 0.48892493071547677, "mrr_at_20": 0.48601855229071494, "mrr_at_3": 0.4481393507521775, "mrr_at_5": 0.4705859065716547, "nauc_map_at_1000_diff1": 0.3747847738730098, "nauc_map_at_1000_max": 0.29507132501507005, "nauc_map_at_1000_std": 0.14692325040190168, "nauc_map_at_100_diff1": 0.3742629999508797, "nauc_map_at_100_max": 0.2947400320768413, "nauc_map_at_100_std": 0.14693435991367468, "nauc_map_at_10_diff1": 0.37386694267676973, "nauc_map_at_10_max": 0.293258808567027, "nauc_map_at_10_std": 0.14830624087068858, "nauc_map_at_1_diff1": 0.4420587257474029, "nauc_map_at_1_max": 0.2554211294897347, "nauc_map_at_1_std": 0.056532912085557045, "nauc_map_at_20_diff1": 0.3739942760636949, "nauc_map_at_20_max": 0.2951675912011361, "nauc_map_at_20_std": 0.14714098733319284, "nauc_map_at_3_diff1": 0.38800093134948943, "nauc_map_at_3_max": 0.3045256019708319, "nauc_map_at_3_std": 0.1320177261065248, "nauc_map_at_5_diff1": 0.3751952574138427, "nauc_map_at_5_max": 0.29712628643646033, "nauc_map_at_5_std": 0.15608824289829076, "nauc_mrr_at_1000_diff1": 0.3822359923443872, "nauc_mrr_at_1000_max": 0.30608038572405405, "nauc_mrr_at_1000_std": 0.14555736544858575, "nauc_mrr_at_100_diff1": 0.38178414099765895, "nauc_mrr_at_100_max": 0.30585503793768143, "nauc_mrr_at_100_std": 0.14552587623032825, "nauc_mrr_at_10_diff1": 0.3809713587160259, "nauc_mrr_at_10_max": 0.30546951362746794, "nauc_mrr_at_10_std": 0.14636774865250649, "nauc_mrr_at_1_diff1": 0.44387058387933104, "nauc_mrr_at_1_max": 0.2725338408610289, "nauc_mrr_at_1_std": 0.060316174675716386, "nauc_mrr_at_20_diff1": 0.38132444443784785, "nauc_mrr_at_20_max": 0.3062138527415912, "nauc_mrr_at_20_std": 0.14593941282612338, "nauc_mrr_at_3_diff1": 0.3970687243852113, "nauc_mrr_at_3_max": 0.31674852581835317, "nauc_mrr_at_3_std": 0.13436456094929625, "nauc_mrr_at_5_diff1": 0.38100138152706003, "nauc_mrr_at_5_max": 0.3073873170760276, "nauc_mrr_at_5_std": 0.15652611945867523, "nauc_ndcg_at_1000_diff1": 0.3574233343909868, "nauc_ndcg_at_1000_max": 0.30655290638796356, "nauc_ndcg_at_1000_std": 0.17337480036547953, "nauc_ndcg_at_100_diff1": 0.3457579810599418, "nauc_ndcg_at_100_max": 0.3006748110933195, "nauc_ndcg_at_100_std": 0.1733711765338111, "nauc_ndcg_at_10_diff1": 0.343048050237238, "nauc_ndcg_at_10_max": 0.29668116614702233, "nauc_ndcg_at_10_std": 0.17411425259884278, "nauc_ndcg_at_1_diff1": 0.44387058387933104, "nauc_ndcg_at_1_max": 0.2725338408610289, "nauc_ndcg_at_1_std": 0.060316174675716386, "nauc_ndcg_at_20_diff1": 0.3436895854878777, "nauc_ndcg_at_20_max": 0.3017373743013492, "nauc_ndcg_at_20_std": 0.17328905930923563, "nauc_ndcg_at_3_diff1": 0.3733606891804634, "nauc_ndcg_at_3_max": 0.3191160889522387, "nauc_ndcg_at_3_std": 0.14994318036782242, "nauc_ndcg_at_5_diff1": 0.3467270461764592, "nauc_ndcg_at_5_max": 0.3056709503441925, "nauc_ndcg_at_5_std": 0.1944296520827874, "nauc_precision_at_1000_diff1": -0.061608838519451925, "nauc_precision_at_1000_max": 0.08385509548569188, "nauc_precision_at_1000_std": 0.047319891697578075, "nauc_precision_at_100_diff1": -0.053599905790753355, "nauc_precision_at_100_max": 0.10630970652219067, "nauc_precision_at_100_std": 0.10934924444876068, "nauc_precision_at_10_diff1": 0.1060684588172504, "nauc_precision_at_10_max": 0.20995640854645897, "nauc_precision_at_10_std": 0.2090328516781496, "nauc_precision_at_1_diff1": 0.44387058387933104, "nauc_precision_at_1_max": 0.2725338408610289, "nauc_precision_at_1_std": 0.060316174675716386, "nauc_precision_at_20_diff1": 0.032771812626895394, "nauc_precision_at_20_max": 0.17315833562162938, "nauc_precision_at_20_std": 0.15246641294545024, "nauc_precision_at_3_diff1": 0.28712758486735523, "nauc_precision_at_3_max": 0.33407499991497946, "nauc_precision_at_3_std": 0.18741372094200553, "nauc_precision_at_5_diff1": 0.18376773280675154, "nauc_precision_at_5_max": 0.2771584276813803, "nauc_precision_at_5_std": 0.2790246716186412, "nauc_recall_at_1000_diff1": 0.21578878928600623, "nauc_recall_at_1000_max": 0.5562717296548295, "nauc_recall_at_1000_std": 0.8184097949860857, "nauc_recall_at_100_diff1": 0.12225055782258017, "nauc_recall_at_100_max": 0.3027932391305674, "nauc_recall_at_100_std": 0.38357849731180904, "nauc_recall_at_10_diff1": 0.20049493517074157, "nauc_recall_at_10_max": 0.274841557447575, "nauc_recall_at_10_std": 0.25749112463598584, "nauc_recall_at_1_diff1": 0.4420587257474029, "nauc_recall_at_1_max": 0.2554211294897347, "nauc_recall_at_1_std": 0.056532912085557045, "nauc_recall_at_20_diff1": 0.18594136122692378, "nauc_recall_at_20_max": 0.29793920886525604, "nauc_recall_at_20_std": 0.28441311234475547, "nauc_recall_at_3_diff1": 0.31239851875822006, "nauc_recall_at_3_max": 0.34248353513370267, "nauc_recall_at_3_std": 0.19423489820201745, "nauc_recall_at_5_diff1": 0.23334772142792784, "nauc_recall_at_5_max": 0.30945953842389773, "nauc_recall_at_5_std": 0.31199421183422255, "ndcg_at_1": 0.3658, "ndcg_at_10": 0.52444, "ndcg_at_100": 0.56324, "ndcg_at_1000": 0.57441, "ndcg_at_20": 0.54254, "ndcg_at_3": 0.45385, "ndcg_at_5": 0.49478, "precision_at_1": 0.3658, "precision_at_10": 0.07933, "precision_at_100": 0.01012, "precision_at_1000": 0.00111, "precision_at_20": 0.0443, "precision_at_3": 0.19082, "precision_at_5": 0.13729, "recall_at_1": 0.33729, "recall_at_10": 0.70586, "recall_at_100": 0.87688, "recall_at_1000": 0.96002, "recall_at_20": 0.77237, "recall_at_3": 0.52059, "recall_at_5": 0.6152 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 1299.6901452541351, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.2583, "map_at_1": 0.185, "map_at_10": 0.23136, "map_at_100": 0.24019, "map_at_1000": 0.24138, "map_at_20": 0.23595, "map_at_3": 0.21583, "map_at_5": 0.22283, "mrr_at_1": 0.185, "mrr_at_10": 0.23136309523809528, "mrr_at_100": 0.24019181484921373, "mrr_at_1000": 0.24138093143174497, "mrr_at_20": 0.23594689542483663, "mrr_at_3": 0.21583333333333335, "mrr_at_5": 0.22283333333333336, "nauc_map_at_1000_diff1": 0.4064546850003172, "nauc_map_at_1000_max": 0.35346841597131085, "nauc_map_at_1000_std": 0.1057915128161012, "nauc_map_at_100_diff1": 0.40622394733997047, "nauc_map_at_100_max": 0.3530956009604433, "nauc_map_at_100_std": 0.10514292708166764, "nauc_map_at_10_diff1": 0.41099525660975966, "nauc_map_at_10_max": 0.35835427780512297, "nauc_map_at_10_std": 0.10416922751037618, "nauc_map_at_1_diff1": 0.4786871453223328, "nauc_map_at_1_max": 0.395247211943784, "nauc_map_at_1_std": 0.09491839730766913, "nauc_map_at_20_diff1": 0.4061435472624891, "nauc_map_at_20_max": 0.3520833984840894, "nauc_map_at_20_std": 0.09980601023243292, "nauc_map_at_3_diff1": 0.42297446060463145, "nauc_map_at_3_max": 0.3652839616730233, "nauc_map_at_3_std": 0.10361345550841002, "nauc_map_at_5_diff1": 0.4180173430004609, "nauc_map_at_5_max": 0.363199626463278, "nauc_map_at_5_std": 0.0970030789067568, "nauc_mrr_at_1000_diff1": 0.4064546850003172, "nauc_mrr_at_1000_max": 0.35346841597131085, "nauc_mrr_at_1000_std": 0.1057915128161012, "nauc_mrr_at_100_diff1": 0.40622394733997047, "nauc_mrr_at_100_max": 0.3530956009604433, "nauc_mrr_at_100_std": 0.10514292708166764, "nauc_mrr_at_10_diff1": 0.41099525660975966, "nauc_mrr_at_10_max": 0.35835427780512297, "nauc_mrr_at_10_std": 0.10416922751037618, "nauc_mrr_at_1_diff1": 0.4786871453223328, "nauc_mrr_at_1_max": 0.395247211943784, "nauc_mrr_at_1_std": 0.09491839730766913, "nauc_mrr_at_20_diff1": 0.4061435472624891, "nauc_mrr_at_20_max": 0.3520833984840894, "nauc_mrr_at_20_std": 0.09980601023243292, "nauc_mrr_at_3_diff1": 0.42297446060463145, "nauc_mrr_at_3_max": 0.3652839616730233, "nauc_mrr_at_3_std": 0.10361345550841002, "nauc_mrr_at_5_diff1": 0.4180173430004609, "nauc_mrr_at_5_max": 0.363199626463278, "nauc_mrr_at_5_std": 0.0970030789067568, "nauc_ndcg_at_1000_diff1": 0.36119623623825786, "nauc_ndcg_at_1000_max": 0.3251781226472588, "nauc_ndcg_at_1000_std": 0.13460027984602643, "nauc_ndcg_at_100_diff1": 0.35902324967740534, "nauc_ndcg_at_100_max": 0.3166841266454237, "nauc_ndcg_at_100_std": 0.12282384956394275, "nauc_ndcg_at_10_diff1": 0.3808438900368939, "nauc_ndcg_at_10_max": 0.33894695737377734, "nauc_ndcg_at_10_std": 0.11168733011205549, "nauc_ndcg_at_1_diff1": 0.4786871453223328, "nauc_ndcg_at_1_max": 0.395247211943784, "nauc_ndcg_at_1_std": 0.09491839730766913, "nauc_ndcg_at_20_diff1": 0.36436154083113964, "nauc_ndcg_at_20_max": 0.31711980056360034, "nauc_ndcg_at_20_std": 0.0956758519192603, "nauc_ndcg_at_3_diff1": 0.4062661680506958, "nauc_ndcg_at_3_max": 0.3550415994920533, "nauc_ndcg_at_3_std": 0.10554574009628236, "nauc_ndcg_at_5_diff1": 0.39721165056474383, "nauc_ndcg_at_5_max": 0.3509364022396801, "nauc_ndcg_at_5_std": 0.09459482118303365, "nauc_precision_at_1000_diff1": 0.020582492678163988, "nauc_precision_at_1000_max": 0.15686625447445454, "nauc_precision_at_1000_std": 0.4483403839895867, "nauc_precision_at_100_diff1": 0.19440095241531105, "nauc_precision_at_100_max": 0.18578592301309582, "nauc_precision_at_100_std": 0.19901872361917824, "nauc_precision_at_10_diff1": 0.299049054010296, "nauc_precision_at_10_max": 0.2850723287272922, "nauc_precision_at_10_std": 0.13535285449006437, "nauc_precision_at_1_diff1": 0.4786871453223328, "nauc_precision_at_1_max": 0.395247211943784, "nauc_precision_at_1_std": 0.09491839730766913, "nauc_precision_at_20_diff1": 0.24434725789443065, "nauc_precision_at_20_max": 0.21071869885036665, "nauc_precision_at_20_std": 0.07820354400243278, "nauc_precision_at_3_diff1": 0.3618335192516606, "nauc_precision_at_3_max": 0.3274495650969326, "nauc_precision_at_3_std": 0.11049381365879976, "nauc_precision_at_5_diff1": 0.34233616576292103, "nauc_precision_at_5_max": 0.318257051641632, "nauc_precision_at_5_std": 0.08729284828842304, "nauc_recall_at_1000_diff1": 0.0205824926781645, "nauc_recall_at_1000_max": 0.1568662544744541, "nauc_recall_at_1000_std": 0.4483403839895864, "nauc_recall_at_100_diff1": 0.1944009524153109, "nauc_recall_at_100_max": 0.18578592301309568, "nauc_recall_at_100_std": 0.19901872361917805, "nauc_recall_at_10_diff1": 0.29904905401029597, "nauc_recall_at_10_max": 0.285072328727292, "nauc_recall_at_10_std": 0.1353528544900643, "nauc_recall_at_1_diff1": 0.4786871453223328, "nauc_recall_at_1_max": 0.395247211943784, "nauc_recall_at_1_std": 0.09491839730766913, "nauc_recall_at_20_diff1": 0.2443472578944304, "nauc_recall_at_20_max": 0.21071869885036645, "nauc_recall_at_20_std": 0.0782035440024324, "nauc_recall_at_3_diff1": 0.3618335192516606, "nauc_recall_at_3_max": 0.32744956509693285, "nauc_recall_at_3_std": 0.11049381365879964, "nauc_recall_at_5_diff1": 0.34233616576292075, "nauc_recall_at_5_max": 0.3182570516416317, "nauc_recall_at_5_std": 0.08729284828842285, "ndcg_at_1": 0.185, "ndcg_at_10": 0.2583, "ndcg_at_100": 0.30556, "ndcg_at_1000": 0.3397, "ndcg_at_20": 0.27566, "ndcg_at_3": 0.22589, "ndcg_at_5": 0.23837, "precision_at_1": 0.185, "precision_at_10": 0.0345, "precision_at_100": 0.00575, "precision_at_1000": 0.00085, "precision_at_20": 0.02075, "precision_at_3": 0.085, "precision_at_5": 0.057, "recall_at_1": 0.185, "recall_at_10": 0.345, "recall_at_100": 0.575, "recall_at_1000": 0.85, "recall_at_20": 0.415, "recall_at_3": 0.255, "recall_at_5": 0.285 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.26156, "map_at_1": 0.19, "map_at_10": 0.23683, "map_at_100": 0.24172, "map_at_1000": 0.24269, "map_at_20": 0.23826, "map_at_3": 0.22167, "map_at_5": 0.23242, "mrr_at_1": 0.19, "mrr_at_10": 0.23683134920634924, "mrr_at_100": 0.24171935489477314, "mrr_at_1000": 0.2426884955745824, "mrr_at_20": 0.23826008223802345, "mrr_at_3": 0.22166666666666668, "mrr_at_5": 0.23241666666666666, "nauc_map_at_1000_diff1": 0.6641907155449482, "nauc_map_at_1000_max": 0.48534236374028916, "nauc_map_at_1000_std": 0.06895415579732908, "nauc_map_at_100_diff1": 0.6640831057208721, "nauc_map_at_100_max": 0.4851182715062897, "nauc_map_at_100_std": 0.06883701661871983, "nauc_map_at_10_diff1": 0.6667368173006674, "nauc_map_at_10_max": 0.484255165057044, "nauc_map_at_10_std": 0.0647646791295488, "nauc_map_at_1_diff1": 0.7792611773409783, "nauc_map_at_1_max": 0.5408925058429875, "nauc_map_at_1_std": 0.03177071948615341, "nauc_map_at_20_diff1": 0.6661193271608087, "nauc_map_at_20_max": 0.483852716270148, "nauc_map_at_20_std": 0.06292547073334602, "nauc_map_at_3_diff1": 0.6930708041433866, "nauc_map_at_3_max": 0.5016790465935429, "nauc_map_at_3_std": 0.060037648187594314, "nauc_map_at_5_diff1": 0.672557899855475, "nauc_map_at_5_max": 0.4877426281333304, "nauc_map_at_5_std": 0.06386821043253478, "nauc_mrr_at_1000_diff1": 0.6641907155449482, "nauc_mrr_at_1000_max": 0.48534236374028916, "nauc_mrr_at_1000_std": 0.06895415579732908, "nauc_mrr_at_100_diff1": 0.6640831057208721, "nauc_mrr_at_100_max": 0.4851182715062897, "nauc_mrr_at_100_std": 0.06883701661871983, "nauc_mrr_at_10_diff1": 0.6667368173006674, "nauc_mrr_at_10_max": 0.484255165057044, "nauc_mrr_at_10_std": 0.0647646791295488, "nauc_mrr_at_1_diff1": 0.7792611773409783, "nauc_mrr_at_1_max": 0.5408925058429875, "nauc_mrr_at_1_std": 0.03177071948615341, "nauc_mrr_at_20_diff1": 0.6661193271608087, "nauc_mrr_at_20_max": 0.483852716270148, "nauc_mrr_at_20_std": 0.06292547073334602, "nauc_mrr_at_3_diff1": 0.6930708041433866, "nauc_mrr_at_3_max": 0.5016790465935429, "nauc_mrr_at_3_std": 0.060037648187594314, "nauc_mrr_at_5_diff1": 0.672557899855475, "nauc_mrr_at_5_max": 0.4877426281333304, "nauc_mrr_at_5_std": 0.06386821043253478, "nauc_ndcg_at_1000_diff1": 0.606836185050386, "nauc_ndcg_at_1000_max": 0.47775308221267027, "nauc_ndcg_at_1000_std": 0.11672644277568842, "nauc_ndcg_at_100_diff1": 0.6074586747481382, "nauc_ndcg_at_100_max": 0.47423018408455725, "nauc_ndcg_at_100_std": 0.11351861089617107, "nauc_ndcg_at_10_diff1": 0.6232461471680374, "nauc_ndcg_at_10_max": 0.45760043042235565, "nauc_ndcg_at_10_std": 0.07464143181827516, "nauc_ndcg_at_1_diff1": 0.7792611773409783, "nauc_ndcg_at_1_max": 0.5408925058429875, "nauc_ndcg_at_1_std": 0.03177071948615341, "nauc_ndcg_at_20_diff1": 0.62074672565798, "nauc_ndcg_at_20_max": 0.45647008618568724, "nauc_ndcg_at_20_std": 0.06806576617201257, "nauc_ndcg_at_3_diff1": 0.6706509022021202, "nauc_ndcg_at_3_max": 0.488536091665009, "nauc_ndcg_at_3_std": 0.06538872229154163, "nauc_ndcg_at_5_diff1": 0.6366605288401967, "nauc_ndcg_at_5_max": 0.4657904827493212, "nauc_ndcg_at_5_std": 0.07236457264680732, "nauc_precision_at_1000_diff1": 0.30311450207823304, "nauc_precision_at_1000_max": 0.5260376928770715, "nauc_precision_at_1000_std": 0.47865398849854834, "nauc_precision_at_100_diff1": 0.4309027801270645, "nauc_precision_at_100_max": 0.48441464421274844, "nauc_precision_at_100_std": 0.3121942638312848, "nauc_precision_at_10_diff1": 0.507081533691937, "nauc_precision_at_10_max": 0.3854633138132969, "nauc_precision_at_10_std": 0.10073133893319942, "nauc_precision_at_1_diff1": 0.7792611773409783, "nauc_precision_at_1_max": 0.5408925058429875, "nauc_precision_at_1_std": 0.03177071948615341, "nauc_precision_at_20_diff1": 0.49858290033065683, "nauc_precision_at_20_max": 0.3825644255497824, "nauc_precision_at_20_std": 0.07713352227995598, "nauc_precision_at_3_diff1": 0.6116114018097193, "nauc_precision_at_3_max": 0.45295741666907585, "nauc_precision_at_3_std": 0.07882108062790914, "nauc_precision_at_5_diff1": 0.5427227452539646, "nauc_precision_at_5_max": 0.4074817874881226, "nauc_precision_at_5_std": 0.09430071367437858, "nauc_recall_at_1000_diff1": 0.30311450207823193, "nauc_recall_at_1000_max": 0.5260376928770707, "nauc_recall_at_1000_std": 0.47865398849854796, "nauc_recall_at_100_diff1": 0.43090278012706423, "nauc_recall_at_100_max": 0.4844146442127485, "nauc_recall_at_100_std": 0.31219426383128435, "nauc_recall_at_10_diff1": 0.5070815336919365, "nauc_recall_at_10_max": 0.38546331381329646, "nauc_recall_at_10_std": 0.10073133893319892, "nauc_recall_at_1_diff1": 0.7792611773409783, "nauc_recall_at_1_max": 0.5408925058429875, "nauc_recall_at_1_std": 0.03177071948615341, "nauc_recall_at_20_diff1": 0.49858290033065683, "nauc_recall_at_20_max": 0.38256442554978215, "nauc_recall_at_20_std": 0.07713352227995594, "nauc_recall_at_3_diff1": 0.6116114018097193, "nauc_recall_at_3_max": 0.4529574166690759, "nauc_recall_at_3_std": 0.07882108062790905, "nauc_recall_at_5_diff1": 0.5427227452539644, "nauc_recall_at_5_max": 0.40748178748812197, "nauc_recall_at_5_std": 0.0943007136743786, "ndcg_at_1": 0.19, "ndcg_at_10": 0.26156, "ndcg_at_100": 0.29432, "ndcg_at_1000": 0.32578, "ndcg_at_20": 0.26668, "ndcg_at_3": 0.23155, "ndcg_at_5": 0.25049, "precision_at_1": 0.19, "precision_at_10": 0.034, "precision_at_100": 0.00515, "precision_at_1000": 0.00078, "precision_at_20": 0.018, "precision_at_3": 0.08667, "precision_at_5": 0.061, "recall_at_1": 0.19, "recall_at_10": 0.34, "recall_at_100": 0.515, "recall_at_1000": 0.775, "recall_at_20": 0.36, "recall_at_3": 0.26, "recall_at_5": 0.305 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 9.609801292419434, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.88733, "map_at_1": 0.77922, "map_at_10": 0.85491, "map_at_100": 0.85572, "map_at_1000": 0.85572, "map_at_20": 0.85572, "map_at_3": 0.84632, "map_at_5": 0.85216, "mrr_at_1": 0.7792207792207793, "mrr_at_10": 0.854906204906205, "mrr_at_100": 0.8557178932178933, "mrr_at_1000": 0.8557178932178933, "mrr_at_20": 0.8557178932178933, "mrr_at_3": 0.8463203463203463, "mrr_at_5": 0.8521645021645022, "nauc_map_at_1000_diff1": 0.7934322885790674, "nauc_map_at_1000_max": 0.42889149307066504, "nauc_map_at_1000_std": -0.14898589679329766, "nauc_map_at_100_diff1": 0.7934322885790674, "nauc_map_at_100_max": 0.42889149307066504, "nauc_map_at_100_std": -0.14898589679329766, "nauc_map_at_10_diff1": 0.7946746897029425, "nauc_map_at_10_max": 0.4315367969330446, "nauc_map_at_10_std": -0.15509289492565895, "nauc_map_at_1_diff1": 0.78246273000063, "nauc_map_at_1_max": 0.4139400142694634, "nauc_map_at_1_std": -0.14021417297508898, "nauc_map_at_20_diff1": 0.7934322885790674, "nauc_map_at_20_max": 0.42889149307066504, "nauc_map_at_20_std": -0.14898589679329766, "nauc_map_at_3_diff1": 0.8005529129446742, "nauc_map_at_3_max": 0.4368327376511254, "nauc_map_at_3_std": -0.13197873310791935, "nauc_map_at_5_diff1": 0.7945644457362998, "nauc_map_at_5_max": 0.42813807841768853, "nauc_map_at_5_std": -0.15147615627835584, "nauc_mrr_at_1000_diff1": 0.7934322885790674, "nauc_mrr_at_1000_max": 0.42889149307066504, "nauc_mrr_at_1000_std": -0.14898589679329766, "nauc_mrr_at_100_diff1": 0.7934322885790674, "nauc_mrr_at_100_max": 0.42889149307066504, "nauc_mrr_at_100_std": -0.14898589679329766, "nauc_mrr_at_10_diff1": 0.7946746897029425, "nauc_mrr_at_10_max": 0.4315367969330446, "nauc_mrr_at_10_std": -0.15509289492565895, "nauc_mrr_at_1_diff1": 0.78246273000063, "nauc_mrr_at_1_max": 0.4139400142694634, "nauc_mrr_at_1_std": -0.14021417297508898, "nauc_mrr_at_20_diff1": 0.7934322885790674, "nauc_mrr_at_20_max": 0.42889149307066504, "nauc_mrr_at_20_std": -0.14898589679329766, "nauc_mrr_at_3_diff1": 0.8005529129446742, "nauc_mrr_at_3_max": 0.4368327376511254, "nauc_mrr_at_3_std": -0.13197873310791935, "nauc_mrr_at_5_diff1": 0.7945644457362998, "nauc_mrr_at_5_max": 0.42813807841768853, "nauc_mrr_at_5_std": -0.15147615627835584, "nauc_ndcg_at_1000_diff1": 0.7950512425953983, "nauc_ndcg_at_1000_max": 0.4304110150291181, "nauc_ndcg_at_1000_std": -0.1537557702365057, "nauc_ndcg_at_100_diff1": 0.7950512425953983, "nauc_ndcg_at_100_max": 0.4304110150291181, "nauc_ndcg_at_100_std": -0.1537557702365057, "nauc_ndcg_at_10_diff1": 0.8012433401285308, "nauc_ndcg_at_10_max": 0.4436533826919126, "nauc_ndcg_at_10_std": -0.18428922374139559, "nauc_ndcg_at_1_diff1": 0.78246273000063, "nauc_ndcg_at_1_max": 0.4139400142694634, "nauc_ndcg_at_1_std": -0.14021417297508898, "nauc_ndcg_at_20_diff1": 0.7950512425953983, "nauc_ndcg_at_20_max": 0.4304110150291181, "nauc_ndcg_at_20_std": -0.1537557702365057, "nauc_ndcg_at_3_diff1": 0.8117962912647465, "nauc_ndcg_at_3_max": 0.4465068871741506, "nauc_ndcg_at_3_std": -0.13050839506151526, "nauc_ndcg_at_5_diff1": 0.7999319493116879, "nauc_ndcg_at_5_max": 0.4295215718616793, "nauc_ndcg_at_5_std": -0.17119373043019556, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 1.0, "nauc_precision_at_10_max": 0.868712702472296, "nauc_precision_at_10_std": -1.1643653934883855, "nauc_precision_at_1_diff1": 0.78246273000063, "nauc_precision_at_1_max": 0.4139400142694634, "nauc_precision_at_1_std": -0.14021417297508898, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_3_diff1": 0.8847118713466388, "nauc_precision_at_3_max": 0.5069794576105962, "nauc_precision_at_3_std": -0.12278837016399671, "nauc_precision_at_5_diff1": 0.8516155514202957, "nauc_precision_at_5_max": 0.43608962587044786, "nauc_precision_at_5_std": -0.3899875213628128, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 1.0, "nauc_recall_at_10_max": 0.868712702472291, "nauc_recall_at_10_std": -1.1643653934883855, "nauc_recall_at_1_diff1": 0.78246273000063, "nauc_recall_at_1_max": 0.4139400142694634, "nauc_recall_at_1_std": -0.14021417297508898, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_3_diff1": 0.8847118713466364, "nauc_recall_at_3_max": 0.5069794576105937, "nauc_recall_at_3_std": -0.12278837016399707, "nauc_recall_at_5_diff1": 0.8516155514202973, "nauc_recall_at_5_max": 0.4360896258704462, "nauc_recall_at_5_std": -0.3899875213628153, "ndcg_at_1": 0.77922, "ndcg_at_10": 0.88733, "ndcg_at_100": 0.8905, "ndcg_at_1000": 0.8905, "ndcg_at_20": 0.8905, "ndcg_at_3": 0.86905, "ndcg_at_5": 0.87966, "precision_at_1": 0.77922, "precision_at_10": 0.0987, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.05, "precision_at_3": 0.31169, "precision_at_5": 0.19221, "recall_at_1": 0.77922, "recall_at_10": 0.98701, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 1.0, "recall_at_3": 0.93506, "recall_at_5": 0.96104 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 94.5943055152893, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.39159, "map_at_1": 0.21735, "map_at_10": 0.33957, "map_at_100": 0.35468, "map_at_1000": 0.3557, "map_at_20": 0.34729, "map_at_3": 0.29888, "map_at_5": 0.32449, "mrr_at_1": 0.30275229357798167, "mrr_at_10": 0.37696410368428707, "mrr_at_100": 0.38764386663917805, "mrr_at_1000": 0.3882795810575206, "mrr_at_20": 0.38235277378984583, "mrr_at_3": 0.3491335372069318, "mrr_at_5": 0.3643476044852192, "nauc_map_at_1000_diff1": 0.3891943906806401, "nauc_map_at_1000_max": 0.10771105038677764, "nauc_map_at_1000_std": -0.3757363720697668, "nauc_map_at_100_diff1": 0.388758610656804, "nauc_map_at_100_max": 0.10796740105799091, "nauc_map_at_100_std": -0.3751089557776611, "nauc_map_at_10_diff1": 0.38874252303050344, "nauc_map_at_10_max": 0.10607471972305403, "nauc_map_at_10_std": -0.38029646262612327, "nauc_map_at_1_diff1": 0.4479663765266703, "nauc_map_at_1_max": 0.08344061958550404, "nauc_map_at_1_std": -0.3201619883425269, "nauc_map_at_20_diff1": 0.38851894744700904, "nauc_map_at_20_max": 0.10585408079995105, "nauc_map_at_20_std": -0.378891310149093, "nauc_map_at_3_diff1": 0.39575353829146903, "nauc_map_at_3_max": 0.10038991688314625, "nauc_map_at_3_std": -0.3661411633453306, "nauc_map_at_5_diff1": 0.39095681780893704, "nauc_map_at_5_max": 0.10539954891553001, "nauc_map_at_5_std": -0.3800485827939017, "nauc_mrr_at_1000_diff1": 0.4270858205611115, "nauc_mrr_at_1000_max": 0.11229534024796844, "nauc_mrr_at_1000_std": -0.3852042862872202, "nauc_mrr_at_100_diff1": 0.4269618416359274, "nauc_mrr_at_100_max": 0.11272734856956011, "nauc_mrr_at_100_std": -0.3845620305871939, "nauc_mrr_at_10_diff1": 0.42538900608630015, "nauc_mrr_at_10_max": 0.10762120716999009, "nauc_mrr_at_10_std": -0.38987371061908493, "nauc_mrr_at_1_diff1": 0.45149239085362464, "nauc_mrr_at_1_max": 0.11642553883873362, "nauc_mrr_at_1_std": -0.37054638932671113, "nauc_mrr_at_20_diff1": 0.4261287226213975, "nauc_mrr_at_20_max": 0.110564345143684, "nauc_mrr_at_20_std": -0.38730809677132183, "nauc_mrr_at_3_diff1": 0.4303268324883733, "nauc_mrr_at_3_max": 0.10677422466163869, "nauc_mrr_at_3_std": -0.39157421166114276, "nauc_mrr_at_5_diff1": 0.4292836011082437, "nauc_mrr_at_5_max": 0.10930394647740294, "nauc_mrr_at_5_std": -0.39440140898838644, "nauc_ndcg_at_1000_diff1": 0.3954317519730454, "nauc_ndcg_at_1000_max": 0.11984342194630304, "nauc_ndcg_at_1000_std": -0.36646537585088207, "nauc_ndcg_at_100_diff1": 0.38885374653122123, "nauc_ndcg_at_100_max": 0.13219229899594778, "nauc_ndcg_at_100_std": -0.34438208302966583, "nauc_ndcg_at_10_diff1": 0.38506936963711735, "nauc_ndcg_at_10_max": 0.10619487049571352, "nauc_ndcg_at_10_std": -0.38661887757098545, "nauc_ndcg_at_1_diff1": 0.4467339947356412, "nauc_ndcg_at_1_max": 0.11249725870947681, "nauc_ndcg_at_1_std": -0.3734570014392317, "nauc_ndcg_at_20_diff1": 0.3836944317399486, "nauc_ndcg_at_20_max": 0.10833552931509423, "nauc_ndcg_at_20_std": -0.38000077174364627, "nauc_ndcg_at_3_diff1": 0.3956121572163778, "nauc_ndcg_at_3_max": 0.09514136816101489, "nauc_ndcg_at_3_std": -0.3838862486285258, "nauc_ndcg_at_5_diff1": 0.392382534513423, "nauc_ndcg_at_5_max": 0.1062818876614154, "nauc_ndcg_at_5_std": -0.39182388867854756, "nauc_precision_at_1000_diff1": 0.024435045834388852, "nauc_precision_at_1000_max": 0.12304493138177797, "nauc_precision_at_1000_std": 0.00022105368792189542, "nauc_precision_at_100_diff1": 0.08531016504572342, "nauc_precision_at_100_max": 0.17074458774590445, "nauc_precision_at_100_std": -0.021180430234989018, "nauc_precision_at_10_diff1": 0.20730521488364595, "nauc_precision_at_10_max": 0.09347162454934668, "nauc_precision_at_10_std": -0.33210209387184353, "nauc_precision_at_1_diff1": 0.4467339947356412, "nauc_precision_at_1_max": 0.11249725870947681, "nauc_precision_at_1_std": -0.3734570014392317, "nauc_precision_at_20_diff1": 0.16466411716113014, "nauc_precision_at_20_max": 0.09453559326766833, "nauc_precision_at_20_std": -0.2651284275522984, "nauc_precision_at_3_diff1": 0.2571689526504932, "nauc_precision_at_3_max": 0.09392870266560792, "nauc_precision_at_3_std": -0.36403711358480606, "nauc_precision_at_5_diff1": 0.2313057952660572, "nauc_precision_at_5_max": 0.1005096228621213, "nauc_precision_at_5_std": -0.35598203053396, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.30446393761455715, "nauc_recall_at_100_max": 0.3113812040722389, "nauc_recall_at_100_std": -0.00927689901427721, "nauc_recall_at_10_diff1": 0.3210558561521383, "nauc_recall_at_10_max": 0.10285388430967961, "nauc_recall_at_10_std": -0.3662555708281694, "nauc_recall_at_1_diff1": 0.4479663765266703, "nauc_recall_at_1_max": 0.08344061958550404, "nauc_recall_at_1_std": -0.3201619883425269, "nauc_recall_at_20_diff1": 0.30936939279334214, "nauc_recall_at_20_max": 0.10513295799184674, "nauc_recall_at_20_std": -0.3444937107201154, "nauc_recall_at_3_diff1": 0.3687560732687906, "nauc_recall_at_3_max": 0.08633837741190495, "nauc_recall_at_3_std": -0.37472195139259223, "nauc_recall_at_5_diff1": 0.35030217548059306, "nauc_recall_at_5_max": 0.10254337088344546, "nauc_recall_at_5_std": -0.3922881466749867, "ndcg_at_1": 0.30428, "ndcg_at_10": 0.39159, "ndcg_at_100": 0.4628, "ndcg_at_1000": 0.48665, "ndcg_at_20": 0.41548, "ndcg_at_3": 0.32701, "ndcg_at_5": 0.35871, "precision_at_1": 0.30428, "precision_at_10": 0.0815, "precision_at_100": 0.01321, "precision_at_1000": 0.00156, "precision_at_20": 0.04771, "precision_at_3": 0.18145, "precision_at_5": 0.13486, "recall_at_1": 0.21735, "recall_at_10": 0.50956, "recall_at_100": 0.82378, "recall_at_1000": 1.0, "recall_at_20": 0.59286, "recall_at_3": 0.34207, "recall_at_5": 0.41758 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.30289, "map_at_1": 0.1449, "map_at_10": 0.25037, "map_at_100": 0.2719, "map_at_1000": 0.2738, "map_at_20": 0.26142, "map_at_3": 0.21433, "map_at_5": 0.23335, "mrr_at_1": 0.25535168195718655, "mrr_at_10": 0.31652407650114067, "mrr_at_100": 0.32893544150439313, "mrr_at_1000": 0.3298652873489474, "mrr_at_20": 0.3234923981116145, "mrr_at_3": 0.29332313965341494, "mrr_at_5": 0.3036442405708461, "nauc_map_at_1000_diff1": 0.43457739722073147, "nauc_map_at_1000_max": 0.24823568710933047, "nauc_map_at_1000_std": -0.25370312619154006, "nauc_map_at_100_diff1": 0.4345210829420494, "nauc_map_at_100_max": 0.24755923848907635, "nauc_map_at_100_std": -0.25456713066896886, "nauc_map_at_10_diff1": 0.43735696617222486, "nauc_map_at_10_max": 0.2533196612617186, "nauc_map_at_10_std": -0.2541926248122255, "nauc_map_at_1_diff1": 0.513895321278906, "nauc_map_at_1_max": 0.24223000175451737, "nauc_map_at_1_std": -0.24997815873894727, "nauc_map_at_20_diff1": 0.4344846713791249, "nauc_map_at_20_max": 0.24896924180209015, "nauc_map_at_20_std": -0.25480007677443606, "nauc_map_at_3_diff1": 0.45882356448306344, "nauc_map_at_3_max": 0.24690567099785007, "nauc_map_at_3_std": -0.2680891538294646, "nauc_map_at_5_diff1": 0.44188105522431037, "nauc_map_at_5_max": 0.2599847120407283, "nauc_map_at_5_std": -0.2540192836156989, "nauc_mrr_at_1000_diff1": 0.42743462569458335, "nauc_mrr_at_1000_max": 0.21087869381195828, "nauc_mrr_at_1000_std": -0.2545229580464646, "nauc_mrr_at_100_diff1": 0.4271337422418246, "nauc_mrr_at_100_max": 0.21071192703945432, "nauc_mrr_at_100_std": -0.25427735746920854, "nauc_mrr_at_10_diff1": 0.4289419050692238, "nauc_mrr_at_10_max": 0.21259491681047768, "nauc_mrr_at_10_std": -0.2552518780658438, "nauc_mrr_at_1_diff1": 0.49270414720254924, "nauc_mrr_at_1_max": 0.2244713219451547, "nauc_mrr_at_1_std": -0.27988538657714074, "nauc_mrr_at_20_diff1": 0.4268017541903574, "nauc_mrr_at_20_max": 0.2110709141279377, "nauc_mrr_at_20_std": -0.25370678720300754, "nauc_mrr_at_3_diff1": 0.4369492901076459, "nauc_mrr_at_3_max": 0.20897815691328095, "nauc_mrr_at_3_std": -0.2774844513650661, "nauc_mrr_at_5_diff1": 0.42602748195671786, "nauc_mrr_at_5_max": 0.21287624906794395, "nauc_mrr_at_5_std": -0.25665096900549267, "nauc_ndcg_at_1000_diff1": 0.41300503969412267, "nauc_ndcg_at_1000_max": 0.2300953503802467, "nauc_ndcg_at_1000_std": -0.23815762471091417, "nauc_ndcg_at_100_diff1": 0.41043075355739245, "nauc_ndcg_at_100_max": 0.22518919314821056, "nauc_ndcg_at_100_std": -0.2348691263548975, "nauc_ndcg_at_10_diff1": 0.41357986502094846, "nauc_ndcg_at_10_max": 0.23671628803070013, "nauc_ndcg_at_10_std": -0.24366171025120886, "nauc_ndcg_at_1_diff1": 0.49270414720254924, "nauc_ndcg_at_1_max": 0.2244713219451547, "nauc_ndcg_at_1_std": -0.27988538657714074, "nauc_ndcg_at_20_diff1": 0.4030648870554614, "nauc_ndcg_at_20_max": 0.22447653862887834, "nauc_ndcg_at_20_std": -0.24580859664545396, "nauc_ndcg_at_3_diff1": 0.43591498372541754, "nauc_ndcg_at_3_max": 0.23022420130524326, "nauc_ndcg_at_3_std": -0.275943597231683, "nauc_ndcg_at_5_diff1": 0.4163387471828433, "nauc_ndcg_at_5_max": 0.24558196251102155, "nauc_ndcg_at_5_std": -0.24487987161839542, "nauc_precision_at_1000_diff1": 0.02963318795790098, "nauc_precision_at_1000_max": 0.0345862628941379, "nauc_precision_at_1000_std": -0.0014912651423470648, "nauc_precision_at_100_diff1": 0.12575298288665557, "nauc_precision_at_100_max": 0.08109371158290818, "nauc_precision_at_100_std": -0.0663314119337897, "nauc_precision_at_10_diff1": 0.2513590557182598, "nauc_precision_at_10_max": 0.17806833312901688, "nauc_precision_at_10_std": -0.18802949109663442, "nauc_precision_at_1_diff1": 0.49270414720254924, "nauc_precision_at_1_max": 0.2244713219451547, "nauc_precision_at_1_std": -0.27988538657714074, "nauc_precision_at_20_diff1": 0.20635944393151526, "nauc_precision_at_20_max": 0.13210044050169437, "nauc_precision_at_20_std": -0.17156740340146884, "nauc_precision_at_3_diff1": 0.3224871343651754, "nauc_precision_at_3_max": 0.19129830622213678, "nauc_precision_at_3_std": -0.26513642202436344, "nauc_precision_at_5_diff1": 0.2711364203214068, "nauc_precision_at_5_max": 0.21229752056121295, "nauc_precision_at_5_std": -0.1969404484233047, "nauc_recall_at_1000_diff1": -0.15132967695572191, "nauc_recall_at_1000_max": 0.052355064645871356, "nauc_recall_at_1000_std": -0.5072703545177019, "nauc_recall_at_100_diff1": 0.3325446864488442, "nauc_recall_at_100_max": 0.16886679772995214, "nauc_recall_at_100_std": -0.12914544330636796, "nauc_recall_at_10_diff1": 0.3403494565493065, "nauc_recall_at_10_max": 0.20907004765501053, "nauc_recall_at_10_std": -0.2031678427789475, "nauc_recall_at_1_diff1": 0.513895321278906, "nauc_recall_at_1_max": 0.24223000175451737, "nauc_recall_at_1_std": -0.24997815873894727, "nauc_recall_at_20_diff1": 0.30140955640715916, "nauc_recall_at_20_max": 0.169749129114769, "nauc_recall_at_20_std": -0.21213574361710996, "nauc_recall_at_3_diff1": 0.3971050364106523, "nauc_recall_at_3_max": 0.2191021920161193, "nauc_recall_at_3_std": -0.2627217888076737, "nauc_recall_at_5_diff1": 0.35314778302843997, "nauc_recall_at_5_max": 0.23612552135699924, "nauc_recall_at_5_std": -0.2105323122479329, "ndcg_at_1": 0.25535, "ndcg_at_10": 0.30289, "ndcg_at_100": 0.39278, "ndcg_at_1000": 0.42741, "ndcg_at_20": 0.33458, "ndcg_at_3": 0.25616, "ndcg_at_5": 0.26714, "precision_at_1": 0.25535, "precision_at_10": 0.08073, "precision_at_100": 0.0155, "precision_at_1000": 0.00195, "precision_at_20": 0.05122, "precision_at_3": 0.16871, "precision_at_5": 0.12416, "recall_at_1": 0.1449, "recall_at_10": 0.38914, "recall_at_100": 0.75882, "recall_at_1000": 0.9949, "recall_at_20": 0.49281, "recall_at_3": 0.24873, "recall_at_5": 0.29348 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.33516, "map_at_1": 0.17789, "map_at_10": 0.28585, "map_at_100": 0.30276, "map_at_1000": 0.30426, "map_at_20": 0.29411, "map_at_3": 0.25084, "map_at_5": 0.27188, "mrr_at_1": 0.254071661237785, "mrr_at_10": 0.32666873481205716, "mrr_at_100": 0.33855970002567226, "mrr_at_1000": 0.339450666533862, "mrr_at_20": 0.33222574515850084, "mrr_at_3": 0.3023887079261674, "mrr_at_5": 0.3162323561346363, "nauc_map_at_1000_diff1": 0.3623821109817951, "nauc_map_at_1000_max": 0.12124235930285585, "nauc_map_at_1000_std": -0.27078210981271805, "nauc_map_at_100_diff1": 0.36182945188768245, "nauc_map_at_100_max": 0.12160822725357565, "nauc_map_at_100_std": -0.269756265087917, "nauc_map_at_10_diff1": 0.3594531472106089, "nauc_map_at_10_max": 0.11644267913960468, "nauc_map_at_10_std": -0.27938863769114985, "nauc_map_at_1_diff1": 0.47650799669579064, "nauc_map_at_1_max": 0.11520993165177627, "nauc_map_at_1_std": -0.23923864658257737, "nauc_map_at_20_diff1": 0.3614517084900315, "nauc_map_at_20_max": 0.11809940522998419, "nauc_map_at_20_std": -0.2790814301699091, "nauc_map_at_3_diff1": 0.3935368289346951, "nauc_map_at_3_max": 0.14310777366199712, "nauc_map_at_3_std": -0.254047650022876, "nauc_map_at_5_diff1": 0.3669713374981257, "nauc_map_at_5_max": 0.12493130042571256, "nauc_map_at_5_std": -0.2736289409957187, "nauc_mrr_at_1000_diff1": 0.4014125402711222, "nauc_mrr_at_1000_max": 0.1173756168405756, "nauc_mrr_at_1000_std": -0.2759369390780343, "nauc_mrr_at_100_diff1": 0.40111268807211997, "nauc_mrr_at_100_max": 0.1176174391512065, "nauc_mrr_at_100_std": -0.2751301876429702, "nauc_mrr_at_10_diff1": 0.39855921326635346, "nauc_mrr_at_10_max": 0.11145313304840819, "nauc_mrr_at_10_std": -0.28540293545438655, "nauc_mrr_at_1_diff1": 0.44667188955821263, "nauc_mrr_at_1_max": 0.13775938038737565, "nauc_mrr_at_1_std": -0.27021454271667966, "nauc_mrr_at_20_diff1": 0.40045821059527936, "nauc_mrr_at_20_max": 0.1151818052623516, "nauc_mrr_at_20_std": -0.2816545399445711, "nauc_mrr_at_3_diff1": 0.41784923702028376, "nauc_mrr_at_3_max": 0.12978561892786436, "nauc_mrr_at_3_std": -0.27246689193165147, "nauc_mrr_at_5_diff1": 0.4040329598691815, "nauc_mrr_at_5_max": 0.11773389677352912, "nauc_mrr_at_5_std": -0.28203728671218703, "nauc_ndcg_at_1000_diff1": 0.35829385829679256, "nauc_ndcg_at_1000_max": 0.12048030635559533, "nauc_ndcg_at_1000_std": -0.2575176498408982, "nauc_ndcg_at_100_diff1": 0.34480447134518943, "nauc_ndcg_at_100_max": 0.12848699770624952, "nauc_ndcg_at_100_std": -0.23286213613495696, "nauc_ndcg_at_10_diff1": 0.3397951173008745, "nauc_ndcg_at_10_max": 0.0956984587173686, "nauc_ndcg_at_10_std": -0.29463308607168387, "nauc_ndcg_at_1_diff1": 0.44667188955821263, "nauc_ndcg_at_1_max": 0.13775938038737565, "nauc_ndcg_at_1_std": -0.27021454271667966, "nauc_ndcg_at_20_diff1": 0.34546092462422556, "nauc_ndcg_at_20_max": 0.103825487525219, "nauc_ndcg_at_20_std": -0.287831046285804, "nauc_ndcg_at_3_diff1": 0.3870074552696127, "nauc_ndcg_at_3_max": 0.13717840697473688, "nauc_ndcg_at_3_std": -0.2623881440202472, "nauc_ndcg_at_5_diff1": 0.3533032866263907, "nauc_ndcg_at_5_max": 0.11164256923018963, "nauc_ndcg_at_5_std": -0.28502533536678604, "nauc_precision_at_1000_diff1": -0.028850941984775703, "nauc_precision_at_1000_max": 0.0731224248272666, "nauc_precision_at_1000_std": 0.06700364217691929, "nauc_precision_at_100_diff1": 0.018052177318166485, "nauc_precision_at_100_max": 0.1290489006189328, "nauc_precision_at_100_std": 0.06644339107714407, "nauc_precision_at_10_diff1": 0.13866559995100966, "nauc_precision_at_10_max": 0.049346880181758535, "nauc_precision_at_10_std": -0.2830185639913429, "nauc_precision_at_1_diff1": 0.44667188955821263, "nauc_precision_at_1_max": 0.13775938038737565, "nauc_precision_at_1_std": -0.27021454271667966, "nauc_precision_at_20_diff1": 0.13138509701167267, "nauc_precision_at_20_max": 0.05433073350201369, "nauc_precision_at_20_std": -0.23818889843069824, "nauc_precision_at_3_diff1": 0.26170965851451644, "nauc_precision_at_3_max": 0.15716134103441157, "nauc_precision_at_3_std": -0.2536287576129388, "nauc_precision_at_5_diff1": 0.1852260129744341, "nauc_precision_at_5_max": 0.09553256875989112, "nauc_precision_at_5_std": -0.2854262606252807, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.1918191910367033, "nauc_recall_at_100_max": 0.19270042996351033, "nauc_recall_at_100_std": 0.027761018698510682, "nauc_recall_at_10_diff1": 0.24171434563777844, "nauc_recall_at_10_max": 0.03862247472832488, "nauc_recall_at_10_std": -0.31189302591786877, "nauc_recall_at_1_diff1": 0.47650799669579064, "nauc_recall_at_1_max": 0.11520993165177627, "nauc_recall_at_1_std": -0.23923864658257737, "nauc_recall_at_20_diff1": 0.2499910733233376, "nauc_recall_at_20_max": 0.06571614435108895, "nauc_recall_at_20_std": -0.2827481665049045, "nauc_recall_at_3_diff1": 0.3514230751546961, "nauc_recall_at_3_max": 0.1405771234647246, "nauc_recall_at_3_std": -0.24502339482009927, "nauc_recall_at_5_diff1": 0.27781283077300545, "nauc_recall_at_5_max": 0.07995060250278149, "nauc_recall_at_5_std": -0.293485935538119, "ndcg_at_1": 0.25407, "ndcg_at_10": 0.33516, "ndcg_at_100": 0.41319, "ndcg_at_1000": 0.44511, "ndcg_at_20": 0.35931, "ndcg_at_3": 0.27928, "ndcg_at_5": 0.30723, "precision_at_1": 0.25407, "precision_at_10": 0.07248, "precision_at_100": 0.01322, "precision_at_1000": 0.00167, "precision_at_20": 0.04357, "precision_at_3": 0.15635, "precision_at_5": 0.11694, "recall_at_1": 0.17789, "recall_at_10": 0.43783, "recall_at_100": 0.7735, "recall_at_1000": 1.0, "recall_at_20": 0.51961, "recall_at_3": 0.2982, "recall_at_5": 0.36346 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/model_meta.json ================================================ {"name": "intfloat/e5-mistral-7b-instruct", "revision": "07163b72af1488142a360786df853f237b1a3ca1", "release_date": "2024-02-08", "languages": ["eng_Latn", "fra_Latn", "deu_Latn", "ita_Latn", "spa_Latn"], "n_parameters": 7111000000, "memory_usage": null, "max_tokens": 32768, "embed_dim": 4096, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["GritLM", "PyTorch"], "reference": "https://huggingface.co/intfloat/e5-mistral-7b-instruct", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 12.844457387924194, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.79752, "map_at_1": 0.64035, "map_at_10": 0.74901, "map_at_100": 0.75181, "map_at_1000": 0.75189, "map_at_20": 0.75103, "map_at_3": 0.72222, "map_at_5": 0.74196, "mrr_at_1": 0.6403508771929824, "mrr_at_10": 0.7490079365079366, "mrr_at_100": 0.751808506456676, "mrr_at_1000": 0.751886827258681, "mrr_at_20": 0.7510283941204995, "mrr_at_3": 0.7222222222222223, "mrr_at_5": 0.7419590643274855, "nauc_map_at_1000_diff1": 0.6723356332353837, "nauc_map_at_1000_max": 0.27297805472316405, "nauc_map_at_1000_std": -0.5726423683151028, "nauc_map_at_100_diff1": 0.6722918045747075, "nauc_map_at_100_max": 0.27314558762940006, "nauc_map_at_100_std": -0.5730836867712397, "nauc_map_at_10_diff1": 0.6658237829761234, "nauc_map_at_10_max": 0.27610706400977403, "nauc_map_at_10_std": -0.5733871417301553, "nauc_map_at_1_diff1": 0.7112013453493657, "nauc_map_at_1_max": 0.23271019903304435, "nauc_map_at_1_std": -0.5640115753195888, "nauc_map_at_20_diff1": 0.6702570196763116, "nauc_map_at_20_max": 0.2741873392047378, "nauc_map_at_20_std": -0.5717522349353029, "nauc_map_at_3_diff1": 0.6752697621006652, "nauc_map_at_3_max": 0.2570892542281406, "nauc_map_at_3_std": -0.571317005989482, "nauc_map_at_5_diff1": 0.6691055922420671, "nauc_map_at_5_max": 0.28519256115613123, "nauc_map_at_5_std": -0.5682331833213823, "nauc_mrr_at_1000_diff1": 0.6723356332353837, "nauc_mrr_at_1000_max": 0.27297805472316405, "nauc_mrr_at_1000_std": -0.5726423683151028, "nauc_mrr_at_100_diff1": 0.6722918045747075, "nauc_mrr_at_100_max": 0.27314558762940006, "nauc_mrr_at_100_std": -0.5730836867712397, "nauc_mrr_at_10_diff1": 0.6658237829761234, "nauc_mrr_at_10_max": 0.27610706400977403, "nauc_mrr_at_10_std": -0.5733871417301553, "nauc_mrr_at_1_diff1": 0.7112013453493657, "nauc_mrr_at_1_max": 0.23271019903304435, "nauc_mrr_at_1_std": -0.5640115753195888, "nauc_mrr_at_20_diff1": 0.6702570196763116, "nauc_mrr_at_20_max": 0.2741873392047378, "nauc_mrr_at_20_std": -0.5717522349353029, "nauc_mrr_at_3_diff1": 0.6752697621006652, "nauc_mrr_at_3_max": 0.2570892542281406, "nauc_mrr_at_3_std": -0.571317005989482, "nauc_mrr_at_5_diff1": 0.6691055922420671, "nauc_mrr_at_5_max": 0.28519256115613123, "nauc_mrr_at_5_std": -0.5682331833213823, "nauc_ndcg_at_1000_diff1": 0.6653614414214758, "nauc_ndcg_at_1000_max": 0.2807112227353227, "nauc_ndcg_at_1000_std": -0.5781138597877794, "nauc_ndcg_at_100_diff1": 0.6644906484817493, "nauc_ndcg_at_100_max": 0.2841887894561054, "nauc_ndcg_at_100_std": -0.5873918400849422, "nauc_ndcg_at_10_diff1": 0.6353600181941691, "nauc_ndcg_at_10_max": 0.29752591673557843, "nauc_ndcg_at_10_std": -0.5863892017597498, "nauc_ndcg_at_1_diff1": 0.7112013453493657, "nauc_ndcg_at_1_max": 0.23271019903304435, "nauc_ndcg_at_1_std": -0.5640115753195888, "nauc_ndcg_at_20_diff1": 0.651997768652368, "nauc_ndcg_at_20_max": 0.29042238107218876, "nauc_ndcg_at_20_std": -0.5788922792970898, "nauc_ndcg_at_3_diff1": 0.6616914652317964, "nauc_ndcg_at_3_max": 0.26336568473785094, "nauc_ndcg_at_3_std": -0.5769159630372578, "nauc_ndcg_at_5_diff1": 0.647354445410905, "nauc_ndcg_at_5_max": 0.3201079312415651, "nauc_ndcg_at_5_std": -0.5716641026307666, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 0.554759603617449, "nauc_precision_at_100_max": 0.7224066624409552, "nauc_precision_at_100_std": -1.7565359911142433, "nauc_precision_at_10_diff1": 0.25703130513687805, "nauc_precision_at_10_max": 0.5385542979657244, "nauc_precision_at_10_std": -0.7577965278796114, "nauc_precision_at_1_diff1": 0.7112013453493657, "nauc_precision_at_1_max": 0.23271019903304435, "nauc_precision_at_1_std": -0.5640115753195888, "nauc_precision_at_20_diff1": 0.2664089692459977, "nauc_precision_at_20_max": 0.6106419565585748, "nauc_precision_at_20_std": -0.7326974857052104, "nauc_precision_at_3_diff1": 0.6085926375473378, "nauc_precision_at_3_max": 0.28660305534992264, "nauc_precision_at_3_std": -0.6000744749701165, "nauc_precision_at_5_diff1": 0.5125707346224895, "nauc_precision_at_5_max": 0.5433636522514341, "nauc_precision_at_5_std": -0.5930010397419233, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.5547596036174103, "nauc_recall_at_100_max": 0.7224066624409369, "nauc_recall_at_100_std": -1.756535991114336, "nauc_recall_at_10_diff1": 0.25703130513688316, "nauc_recall_at_10_max": 0.5385542979657265, "nauc_recall_at_10_std": -0.7577965278796062, "nauc_recall_at_1_diff1": 0.7112013453493657, "nauc_recall_at_1_max": 0.23271019903304435, "nauc_recall_at_1_std": -0.5640115753195888, "nauc_recall_at_20_diff1": 0.26640896924600216, "nauc_recall_at_20_max": 0.6106419565585729, "nauc_recall_at_20_std": -0.7326974857051948, "nauc_recall_at_3_diff1": 0.6085926375473392, "nauc_recall_at_3_max": 0.2866030553499234, "nauc_recall_at_3_std": -0.6000744749701153, "nauc_recall_at_5_diff1": 0.5125707346224897, "nauc_recall_at_5_max": 0.5433636522514345, "nauc_recall_at_5_std": -0.5930010397419251, "ndcg_at_1": 0.64035, "ndcg_at_10": 0.79752, "ndcg_at_100": 0.80827, "ndcg_at_1000": 0.80956, "ndcg_at_20": 0.80442, "ndcg_at_3": 0.74645, "ndcg_at_5": 0.78045, "precision_at_1": 0.64035, "precision_at_10": 0.09474, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "precision_at_20": 0.04868, "precision_at_3": 0.27193, "precision_at_5": 0.17895, "recall_at_1": 0.64035, "recall_at_10": 0.94737, "recall_at_100": 0.99123, "recall_at_1000": 1.0, "recall_at_20": 0.97368, "recall_at_3": 0.81579, "recall_at_5": 0.89474 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 22.750174522399902, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.92868, "map_at_1": 0.87778, "map_at_10": 0.9138, "map_at_100": 0.91467, "map_at_1000": 0.91469, "map_at_20": 0.91439, "map_at_3": 0.90944, "map_at_5": 0.91133, "mrr_at_1": 0.8777777777777778, "mrr_at_10": 0.9137985008818343, "mrr_at_100": 0.9146685714948152, "mrr_at_1000": 0.9146886305512084, "mrr_at_20": 0.9143924634494812, "mrr_at_3": 0.9094444444444447, "mrr_at_5": 0.9113333333333336, "nauc_map_at_1000_diff1": 0.9175700238097899, "nauc_map_at_1000_max": 0.8209359588621885, "nauc_map_at_1000_std": 0.14758622633517648, "nauc_map_at_100_diff1": 0.9175628238240089, "nauc_map_at_100_max": 0.8209388488088828, "nauc_map_at_100_std": 0.14768093837442417, "nauc_map_at_10_diff1": 0.9173620069052005, "nauc_map_at_10_max": 0.8217179762219908, "nauc_map_at_10_std": 0.14776156586300634, "nauc_map_at_1_diff1": 0.9215546474219641, "nauc_map_at_1_max": 0.799260933831841, "nauc_map_at_1_std": 0.11366837666289938, "nauc_map_at_20_diff1": 0.9174648784829924, "nauc_map_at_20_max": 0.8212567907630953, "nauc_map_at_20_std": 0.1480711029323691, "nauc_map_at_3_diff1": 0.9191364960344942, "nauc_map_at_3_max": 0.8260591982830046, "nauc_map_at_3_std": 0.15475753853438384, "nauc_map_at_5_diff1": 0.9182178325391143, "nauc_map_at_5_max": 0.8243424595178229, "nauc_map_at_5_std": 0.15253983585873487, "nauc_mrr_at_1000_diff1": 0.9175700238097899, "nauc_mrr_at_1000_max": 0.8209359588621885, "nauc_mrr_at_1000_std": 0.14758622633517648, "nauc_mrr_at_100_diff1": 0.9175628238240089, "nauc_mrr_at_100_max": 0.8209388488088828, "nauc_mrr_at_100_std": 0.14768093837442417, "nauc_mrr_at_10_diff1": 0.9173620069052005, "nauc_mrr_at_10_max": 0.8217179762219908, "nauc_mrr_at_10_std": 0.14776156586300634, "nauc_mrr_at_1_diff1": 0.9215546474219641, "nauc_mrr_at_1_max": 0.799260933831841, "nauc_mrr_at_1_std": 0.11366837666289938, "nauc_mrr_at_20_diff1": 0.9174648784829924, "nauc_mrr_at_20_max": 0.8212567907630953, "nauc_mrr_at_20_std": 0.1480711029323691, "nauc_mrr_at_3_diff1": 0.9191364960344942, "nauc_mrr_at_3_max": 0.8260591982830046, "nauc_mrr_at_3_std": 0.15475753853438384, "nauc_mrr_at_5_diff1": 0.9182178325391143, "nauc_mrr_at_5_max": 0.8243424595178229, "nauc_mrr_at_5_std": 0.15253983585873487, "nauc_ndcg_at_1000_diff1": 0.9167576396553385, "nauc_ndcg_at_1000_max": 0.824184031484245, "nauc_ndcg_at_1000_std": 0.15512298069474917, "nauc_ndcg_at_100_diff1": 0.9167224172494681, "nauc_ndcg_at_100_max": 0.8247226421327326, "nauc_ndcg_at_100_std": 0.16028099913529092, "nauc_ndcg_at_10_diff1": 0.9150957154475938, "nauc_ndcg_at_10_max": 0.8289509732231671, "nauc_ndcg_at_10_std": 0.1596542610882336, "nauc_ndcg_at_1_diff1": 0.9215546474219641, "nauc_ndcg_at_1_max": 0.799260933831841, "nauc_ndcg_at_1_std": 0.11366837666289938, "nauc_ndcg_at_20_diff1": 0.9154318158435176, "nauc_ndcg_at_20_max": 0.82716744742434, "nauc_ndcg_at_20_std": 0.16310395329427177, "nauc_ndcg_at_3_diff1": 0.9190673661119936, "nauc_ndcg_at_3_max": 0.8379775133293821, "nauc_ndcg_at_3_std": 0.17381112073015367, "nauc_ndcg_at_5_diff1": 0.9172056481173958, "nauc_ndcg_at_5_max": 0.8350861313142242, "nauc_ndcg_at_5_std": 0.16991899186101822, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9183006535947483, "nauc_precision_at_100_max": 0.884687208216607, "nauc_precision_at_100_std": 0.6765639589169249, "nauc_precision_at_10_diff1": 0.8925222262818151, "nauc_precision_at_10_max": 0.8906954085982189, "nauc_precision_at_10_std": 0.2649697560183486, "nauc_precision_at_1_diff1": 0.9215546474219641, "nauc_precision_at_1_max": 0.799260933831841, "nauc_precision_at_1_std": 0.11366837666289938, "nauc_precision_at_20_diff1": 0.8873716153127876, "nauc_precision_at_20_max": 0.8911647992530264, "nauc_precision_at_20_std": 0.37704248366012, "nauc_precision_at_3_diff1": 0.9192442934620643, "nauc_precision_at_3_max": 0.8962492798537904, "nauc_precision_at_3_std": 0.26736794008383474, "nauc_precision_at_5_diff1": 0.911647992530343, "nauc_precision_at_5_max": 0.8948412698412691, "nauc_precision_at_5_std": 0.26683006535947623, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9183006535947761, "nauc_recall_at_100_max": 0.8846872082166294, "nauc_recall_at_100_std": 0.6765639589169213, "nauc_recall_at_10_diff1": 0.8925222262818165, "nauc_recall_at_10_max": 0.8906954085982204, "nauc_recall_at_10_std": 0.2649697560183457, "nauc_recall_at_1_diff1": 0.9215546474219641, "nauc_recall_at_1_max": 0.799260933831841, "nauc_recall_at_1_std": 0.11366837666289938, "nauc_recall_at_20_diff1": 0.8873716153127925, "nauc_recall_at_20_max": 0.891164799253035, "nauc_recall_at_20_std": 0.3770424836601331, "nauc_recall_at_3_diff1": 0.9192442934620657, "nauc_recall_at_3_max": 0.8962492798537856, "nauc_recall_at_3_std": 0.2673679400838344, "nauc_recall_at_5_diff1": 0.9116479925303471, "nauc_recall_at_5_max": 0.8948412698412709, "nauc_recall_at_5_std": 0.26683006535947723, "ndcg_at_1": 0.87778, "ndcg_at_10": 0.92868, "ndcg_at_100": 0.93292, "ndcg_at_1000": 0.9336, "ndcg_at_20": 0.93071, "ndcg_at_3": 0.91932, "ndcg_at_5": 0.92263, "precision_at_1": 0.87778, "precision_at_10": 0.09744, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "precision_at_20": 0.04911, "precision_at_3": 0.31593, "precision_at_5": 0.19111, "recall_at_1": 0.87778, "recall_at_10": 0.97444, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "recall_at_20": 0.98222, "recall_at_3": 0.94778, "recall_at_5": 0.95556 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.82283, "map_at_1": 0.72667, "map_at_10": 0.79085, "map_at_100": 0.79349, "map_at_1000": 0.79354, "map_at_20": 0.79205, "map_at_3": 0.77481, "map_at_5": 0.78537, "mrr_at_1": 0.7266666666666667, "mrr_at_10": 0.7908456790123461, "mrr_at_100": 0.7934928056608854, "mrr_at_1000": 0.793538990477385, "mrr_at_20": 0.7920526847879792, "mrr_at_3": 0.774814814814815, "mrr_at_5": 0.7853703703703707, "nauc_map_at_1000_diff1": 0.8234897859527498, "nauc_map_at_1000_max": 0.5617154161554866, "nauc_map_at_1000_std": 0.1527782088522249, "nauc_map_at_100_diff1": 0.8234845751578477, "nauc_map_at_100_max": 0.5616623686340935, "nauc_map_at_100_std": 0.15295319052202627, "nauc_map_at_10_diff1": 0.8217154451358325, "nauc_map_at_10_max": 0.5608609888078221, "nauc_map_at_10_std": 0.15473708791377086, "nauc_map_at_1_diff1": 0.8438314931821422, "nauc_map_at_1_max": 0.560422261720963, "nauc_map_at_1_std": 0.12586115183517807, "nauc_map_at_20_diff1": 0.8227647973076403, "nauc_map_at_20_max": 0.5619696743995988, "nauc_map_at_20_std": 0.15390905856653878, "nauc_map_at_3_diff1": 0.8260187927862362, "nauc_map_at_3_max": 0.5660999527289668, "nauc_map_at_3_std": 0.1494942169502371, "nauc_map_at_5_diff1": 0.8215064787749222, "nauc_map_at_5_max": 0.561880380615228, "nauc_map_at_5_std": 0.14899699856383378, "nauc_mrr_at_1000_diff1": 0.8234897859527498, "nauc_mrr_at_1000_max": 0.5617154161554866, "nauc_mrr_at_1000_std": 0.1527782088522249, "nauc_mrr_at_100_diff1": 0.8234845751578477, "nauc_mrr_at_100_max": 0.5616623686340935, "nauc_mrr_at_100_std": 0.15295319052202627, "nauc_mrr_at_10_diff1": 0.8217154451358325, "nauc_mrr_at_10_max": 0.5608609888078221, "nauc_mrr_at_10_std": 0.15473708791377086, "nauc_mrr_at_1_diff1": 0.8438314931821422, "nauc_mrr_at_1_max": 0.560422261720963, "nauc_mrr_at_1_std": 0.12586115183517807, "nauc_mrr_at_20_diff1": 0.8227647973076403, "nauc_mrr_at_20_max": 0.5619696743995988, "nauc_mrr_at_20_std": 0.15390905856653878, "nauc_mrr_at_3_diff1": 0.8260187927862362, "nauc_mrr_at_3_max": 0.5660999527289668, "nauc_mrr_at_3_std": 0.1494942169502371, "nauc_mrr_at_5_diff1": 0.8215064787749222, "nauc_mrr_at_5_max": 0.561880380615228, "nauc_mrr_at_5_std": 0.14899699856383378, "nauc_ndcg_at_1000_diff1": 0.8197499082937609, "nauc_ndcg_at_1000_max": 0.5606664898880551, "nauc_ndcg_at_1000_std": 0.15969401918873752, "nauc_ndcg_at_100_diff1": 0.8195860199960943, "nauc_ndcg_at_100_max": 0.5588226115554171, "nauc_ndcg_at_100_std": 0.16480730316388906, "nauc_ndcg_at_10_diff1": 0.8101250409144324, "nauc_ndcg_at_10_max": 0.5567023121807358, "nauc_ndcg_at_10_std": 0.17417787453234748, "nauc_ndcg_at_1_diff1": 0.8438314931821422, "nauc_ndcg_at_1_max": 0.560422261720963, "nauc_ndcg_at_1_std": 0.12586115183517807, "nauc_ndcg_at_20_diff1": 0.8146190720348256, "nauc_ndcg_at_20_max": 0.5624189349413797, "nauc_ndcg_at_20_std": 0.17099815701792506, "nauc_ndcg_at_3_diff1": 0.8195822806147087, "nauc_ndcg_at_3_max": 0.5679585650218014, "nauc_ndcg_at_3_std": 0.15628807596368502, "nauc_ndcg_at_5_diff1": 0.8095848513111588, "nauc_ndcg_at_5_max": 0.5587825702586793, "nauc_ndcg_at_5_std": 0.15612029203237102, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.7980859010270617, "nauc_precision_at_100_max": 0.3151260504201801, "nauc_precision_at_100_std": 0.8045634920635055, "nauc_precision_at_10_diff1": 0.7261735612119219, "nauc_precision_at_10_max": 0.5223142397055406, "nauc_precision_at_10_std": 0.3301871473226919, "nauc_precision_at_1_diff1": 0.8438314931821422, "nauc_precision_at_1_max": 0.560422261720963, "nauc_precision_at_1_std": 0.12586115183517807, "nauc_precision_at_20_diff1": 0.7486443295266831, "nauc_precision_at_20_max": 0.5787096890038043, "nauc_precision_at_20_std": 0.34637829490770417, "nauc_precision_at_3_diff1": 0.7956186663829573, "nauc_precision_at_3_max": 0.5747965263185904, "nauc_precision_at_3_std": 0.18084222145384055, "nauc_precision_at_5_diff1": 0.7492946102834129, "nauc_precision_at_5_max": 0.5403959168269141, "nauc_precision_at_5_std": 0.18949822978631403, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.7980859010270872, "nauc_recall_at_100_max": 0.31512605042016767, "nauc_recall_at_100_std": 0.8045634920634981, "nauc_recall_at_10_diff1": 0.7261735612119232, "nauc_recall_at_10_max": 0.5223142397055422, "nauc_recall_at_10_std": 0.330187147322698, "nauc_recall_at_1_diff1": 0.8438314931821422, "nauc_recall_at_1_max": 0.560422261720963, "nauc_recall_at_1_std": 0.12586115183517807, "nauc_recall_at_20_diff1": 0.7486443295266847, "nauc_recall_at_20_max": 0.5787096890038078, "nauc_recall_at_20_std": 0.3463782949077075, "nauc_recall_at_3_diff1": 0.7956186663829558, "nauc_recall_at_3_max": 0.5747965263185886, "nauc_recall_at_3_std": 0.18084222145383833, "nauc_recall_at_5_diff1": 0.7492946102834147, "nauc_recall_at_5_max": 0.5403959168269155, "nauc_recall_at_5_std": 0.18949822978631597, "ndcg_at_1": 0.72667, "ndcg_at_10": 0.82283, "ndcg_at_100": 0.83689, "ndcg_at_1000": 0.83805, "ndcg_at_20": 0.82747, "ndcg_at_3": 0.79042, "ndcg_at_5": 0.80955, "precision_at_1": 0.72667, "precision_at_10": 0.09233, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "precision_at_20": 0.04711, "precision_at_3": 0.27852, "precision_at_5": 0.17644, "recall_at_1": 0.72667, "recall_at_10": 0.92333, "recall_at_100": 0.99111, "recall_at_1000": 1.0, "recall_at_20": 0.94222, "recall_at_3": 0.83556, "recall_at_5": 0.88222 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.77524, "map_at_1": 0.66444, "map_at_10": 0.73824, "map_at_100": 0.74246, "map_at_1000": 0.74254, "map_at_20": 0.74128, "map_at_3": 0.71907, "map_at_5": 0.73046, "mrr_at_1": 0.6644444444444444, "mrr_at_10": 0.7382402998236333, "mrr_at_100": 0.7424622167121282, "mrr_at_1000": 0.7425399479231359, "mrr_at_20": 0.7412782930338401, "mrr_at_3": 0.7190740740740743, "mrr_at_5": 0.7304629629629632, "nauc_map_at_1000_diff1": 0.7899720871622794, "nauc_map_at_1000_max": 0.636679580496758, "nauc_map_at_1000_std": 0.20278639246259875, "nauc_map_at_100_diff1": 0.7899691659530444, "nauc_map_at_100_max": 0.6367077990429704, "nauc_map_at_100_std": 0.20293440007437535, "nauc_map_at_10_diff1": 0.7880583637680022, "nauc_map_at_10_max": 0.6359172299707678, "nauc_map_at_10_std": 0.20454031162914182, "nauc_map_at_1_diff1": 0.8195281021775603, "nauc_map_at_1_max": 0.6255213235759166, "nauc_map_at_1_std": 0.16194235584310274, "nauc_map_at_20_diff1": 0.7896294447842023, "nauc_map_at_20_max": 0.6373774697210272, "nauc_map_at_20_std": 0.20461194000085434, "nauc_map_at_3_diff1": 0.786610187688215, "nauc_map_at_3_max": 0.6360744018755625, "nauc_map_at_3_std": 0.19083591004405367, "nauc_map_at_5_diff1": 0.7877762737592369, "nauc_map_at_5_max": 0.6363189564576122, "nauc_map_at_5_std": 0.19808412603436282, "nauc_mrr_at_1000_diff1": 0.7899720871622794, "nauc_mrr_at_1000_max": 0.636679580496758, "nauc_mrr_at_1000_std": 0.20278639246259875, "nauc_mrr_at_100_diff1": 0.7899691659530444, "nauc_mrr_at_100_max": 0.6367077990429704, "nauc_mrr_at_100_std": 0.20293440007437535, "nauc_mrr_at_10_diff1": 0.7880583637680022, "nauc_mrr_at_10_max": 0.6359172299707678, "nauc_mrr_at_10_std": 0.20454031162914182, "nauc_mrr_at_1_diff1": 0.8195281021775603, "nauc_mrr_at_1_max": 0.6255213235759166, "nauc_mrr_at_1_std": 0.16194235584310274, "nauc_mrr_at_20_diff1": 0.7896294447842023, "nauc_mrr_at_20_max": 0.6373774697210272, "nauc_mrr_at_20_std": 0.20461194000085434, "nauc_mrr_at_3_diff1": 0.786610187688215, "nauc_mrr_at_3_max": 0.6360744018755625, "nauc_mrr_at_3_std": 0.19083591004405367, "nauc_mrr_at_5_diff1": 0.7877762737592369, "nauc_mrr_at_5_max": 0.6363189564576122, "nauc_mrr_at_5_std": 0.19808412603436282, "nauc_ndcg_at_1000_diff1": 0.7861236191707828, "nauc_ndcg_at_1000_max": 0.6399061578992061, "nauc_ndcg_at_1000_std": 0.21488055303395148, "nauc_ndcg_at_100_diff1": 0.7861182735886454, "nauc_ndcg_at_100_max": 0.641147678776563, "nauc_ndcg_at_100_std": 0.21969864365016378, "nauc_ndcg_at_10_diff1": 0.776980790774603, "nauc_ndcg_at_10_max": 0.6401755531081292, "nauc_ndcg_at_10_std": 0.23416053780066517, "nauc_ndcg_at_1_diff1": 0.8195281021775603, "nauc_ndcg_at_1_max": 0.6255213235759166, "nauc_ndcg_at_1_std": 0.16194235584310274, "nauc_ndcg_at_20_diff1": 0.782997924515294, "nauc_ndcg_at_20_max": 0.6474238317445925, "nauc_ndcg_at_20_std": 0.23531316447325576, "nauc_ndcg_at_3_diff1": 0.7757766754986003, "nauc_ndcg_at_3_max": 0.6396965790529899, "nauc_ndcg_at_3_std": 0.19901557248954965, "nauc_ndcg_at_5_diff1": 0.777266463466251, "nauc_ndcg_at_5_max": 0.6412025750548266, "nauc_ndcg_at_5_std": 0.2148997990723823, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.7858309990662994, "nauc_precision_at_100_max": 0.7740040460628638, "nauc_precision_at_100_std": 0.7100451291627877, "nauc_precision_at_10_diff1": 0.7104132857823072, "nauc_precision_at_10_max": 0.6661694539730024, "nauc_precision_at_10_std": 0.4357866811781495, "nauc_precision_at_1_diff1": 0.8195281021775603, "nauc_precision_at_1_max": 0.6255213235759166, "nauc_precision_at_1_std": 0.16194235584310274, "nauc_precision_at_20_diff1": 0.7337215587989555, "nauc_precision_at_20_max": 0.7727816272707906, "nauc_precision_at_20_std": 0.5824692450079442, "nauc_precision_at_3_diff1": 0.7381955824071126, "nauc_precision_at_3_max": 0.6523252831018433, "nauc_precision_at_3_std": 0.2268243904737471, "nauc_precision_at_5_diff1": 0.7334373079388574, "nauc_precision_at_5_max": 0.6629504556598959, "nauc_precision_at_5_std": 0.28992393552693146, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.7858309990662884, "nauc_recall_at_100_max": 0.7740040460628713, "nauc_recall_at_100_std": 0.7100451291627723, "nauc_recall_at_10_diff1": 0.7104132857823081, "nauc_recall_at_10_max": 0.6661694539730049, "nauc_recall_at_10_std": 0.43578668117815195, "nauc_recall_at_1_diff1": 0.8195281021775603, "nauc_recall_at_1_max": 0.6255213235759166, "nauc_recall_at_1_std": 0.16194235584310274, "nauc_recall_at_20_diff1": 0.733721558798958, "nauc_recall_at_20_max": 0.7727816272707918, "nauc_recall_at_20_std": 0.5824692450079454, "nauc_recall_at_3_diff1": 0.7381955824071117, "nauc_recall_at_3_max": 0.6523252831018428, "nauc_recall_at_3_std": 0.2268243904737469, "nauc_recall_at_5_diff1": 0.7334373079388581, "nauc_recall_at_5_max": 0.6629504556598974, "nauc_recall_at_5_std": 0.2899239355269335, "ndcg_at_1": 0.66444, "ndcg_at_10": 0.77524, "ndcg_at_100": 0.79551, "ndcg_at_1000": 0.79729, "ndcg_at_20": 0.78642, "ndcg_at_3": 0.73594, "ndcg_at_5": 0.7565, "precision_at_1": 0.66444, "precision_at_10": 0.08922, "precision_at_100": 0.00987, "precision_at_1000": 0.001, "precision_at_20": 0.04683, "precision_at_3": 0.26148, "precision_at_5": 0.16689, "recall_at_1": 0.66444, "recall_at_10": 0.89222, "recall_at_100": 0.98667, "recall_at_1000": 1.0, "recall_at_20": 0.93667, "recall_at_3": 0.78444, "recall_at_5": 0.83444 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 69.76440715789795, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.76355, "map_at_1": 0.48055, "map_at_10": 0.71596, "map_at_100": 0.72269, "map_at_1000": 0.72304, "map_at_20": 0.72063, "map_at_3": 0.6768, "map_at_5": 0.70509, "mrr_at_1": 0.7364864864864865, "mrr_at_10": 0.7903548637923634, "mrr_at_100": 0.7925485119906738, "mrr_at_1000": 0.7927242938243799, "mrr_at_20": 0.7919637160083169, "mrr_at_3": 0.784065315315315, "mrr_at_5": 0.788119369369369, "nauc_map_at_1000_diff1": 0.6120886347168524, "nauc_map_at_1000_max": 0.6567934929218409, "nauc_map_at_1000_std": 0.3328898180127869, "nauc_map_at_100_diff1": 0.6119223965896734, "nauc_map_at_100_max": 0.657123268389259, "nauc_map_at_100_std": 0.33351708700930727, "nauc_map_at_10_diff1": 0.6065961876568182, "nauc_map_at_10_max": 0.6528343968354539, "nauc_map_at_10_std": 0.3226467274838311, "nauc_map_at_1_diff1": 0.6018674995640366, "nauc_map_at_1_max": 0.32940697064641544, "nauc_map_at_1_std": 0.11559889953342109, "nauc_map_at_20_diff1": 0.6107444546155024, "nauc_map_at_20_max": 0.6570571699925494, "nauc_map_at_20_std": 0.33129771879965536, "nauc_map_at_3_diff1": 0.6073294597606239, "nauc_map_at_3_max": 0.5870903892260443, "nauc_map_at_3_std": 0.25101234031587527, "nauc_map_at_5_diff1": 0.6037720970775445, "nauc_map_at_5_max": 0.6388561752349948, "nauc_map_at_5_std": 0.30314656514004323, "nauc_mrr_at_1000_diff1": 0.7133054627456897, "nauc_mrr_at_1000_max": 0.7553376166599368, "nauc_mrr_at_1000_std": 0.46353672735350226, "nauc_mrr_at_100_diff1": 0.7133556295202433, "nauc_mrr_at_100_max": 0.755503983145516, "nauc_mrr_at_100_std": 0.46382569286212155, "nauc_mrr_at_10_diff1": 0.7130938288592268, "nauc_mrr_at_10_max": 0.7571695366472795, "nauc_mrr_at_10_std": 0.46358167542606876, "nauc_mrr_at_1_diff1": 0.7039530934167848, "nauc_mrr_at_1_max": 0.6858079847057547, "nauc_mrr_at_1_std": 0.3765388440967593, "nauc_mrr_at_20_diff1": 0.7134291595723251, "nauc_mrr_at_20_max": 0.7560514342722913, "nauc_mrr_at_20_std": 0.4641020982610976, "nauc_mrr_at_3_diff1": 0.7087505118984042, "nauc_mrr_at_3_max": 0.7518004256417345, "nauc_mrr_at_3_std": 0.45155163434304196, "nauc_mrr_at_5_diff1": 0.7149092216925744, "nauc_mrr_at_5_max": 0.7571960903936084, "nauc_mrr_at_5_std": 0.45999482268765096, "nauc_ndcg_at_1000_diff1": 0.64424755653158, "nauc_ndcg_at_1000_max": 0.7179995530658594, "nauc_ndcg_at_1000_std": 0.4261845949670468, "nauc_ndcg_at_100_diff1": 0.642682319085353, "nauc_ndcg_at_100_max": 0.7256084162632179, "nauc_ndcg_at_100_std": 0.44181624785201484, "nauc_ndcg_at_10_diff1": 0.6278521640047413, "nauc_ndcg_at_10_max": 0.7180933779096977, "nauc_ndcg_at_10_std": 0.4070068316170624, "nauc_ndcg_at_1_diff1": 0.7039530934167848, "nauc_ndcg_at_1_max": 0.6858079847057547, "nauc_ndcg_at_1_std": 0.3765388440967593, "nauc_ndcg_at_20_diff1": 0.6373392342929015, "nauc_ndcg_at_20_max": 0.7258280384831197, "nauc_ndcg_at_20_std": 0.42983545353306446, "nauc_ndcg_at_3_diff1": 0.6264794579305172, "nauc_ndcg_at_3_max": 0.636290626191759, "nauc_ndcg_at_3_std": 0.3056776331064701, "nauc_ndcg_at_5_diff1": 0.625869075116017, "nauc_ndcg_at_5_max": 0.6952466157901112, "nauc_ndcg_at_5_std": 0.37196337183328204, "nauc_precision_at_1000_diff1": -0.21994374172961523, "nauc_precision_at_1000_max": 0.09521843731394646, "nauc_precision_at_1000_std": 0.19015616075822983, "nauc_precision_at_100_diff1": -0.11260045412571985, "nauc_precision_at_100_max": 0.24458203344170185, "nauc_precision_at_100_std": 0.3186218386991172, "nauc_precision_at_10_diff1": -0.027536142258204773, "nauc_precision_at_10_max": 0.3792982449178664, "nauc_precision_at_10_std": 0.33785925216888235, "nauc_precision_at_1_diff1": 0.7039530934167848, "nauc_precision_at_1_max": 0.6858079847057547, "nauc_precision_at_1_std": 0.3765388440967593, "nauc_precision_at_20_diff1": -0.06191201640332512, "nauc_precision_at_20_max": 0.3372528758223194, "nauc_precision_at_20_std": 0.34947043305267833, "nauc_precision_at_3_diff1": 0.1328685927957571, "nauc_precision_at_3_max": 0.46127555813841487, "nauc_precision_at_3_std": 0.29389070032941417, "nauc_precision_at_5_diff1": 0.02893102490247378, "nauc_precision_at_5_max": 0.41494796228331343, "nauc_precision_at_5_std": 0.3198824993031576, "nauc_recall_at_1000_diff1": 0.6723513136663745, "nauc_recall_at_1000_max": 0.8988977438568299, "nauc_recall_at_1000_std": 0.8912083424270899, "nauc_recall_at_100_diff1": 0.6400112549494104, "nauc_recall_at_100_max": 0.8895950225937042, "nauc_recall_at_100_std": 0.842089393056033, "nauc_recall_at_10_diff1": 0.5794086907748127, "nauc_recall_at_10_max": 0.7969474008126761, "nauc_recall_at_10_std": 0.5315121876669305, "nauc_recall_at_1_diff1": 0.6018674995640366, "nauc_recall_at_1_max": 0.32940697064641544, "nauc_recall_at_1_std": 0.11559889953342109, "nauc_recall_at_20_diff1": 0.6099124795283541, "nauc_recall_at_20_max": 0.8447530472239648, "nauc_recall_at_20_std": 0.6616877200292428, "nauc_recall_at_3_diff1": 0.5897862739896276, "nauc_recall_at_3_max": 0.6127205173754705, "nauc_recall_at_3_std": 0.2903940935173204, "nauc_recall_at_5_diff1": 0.5875005857489918, "nauc_recall_at_5_max": 0.7295534419284448, "nauc_recall_at_5_std": 0.42258075167199766, "ndcg_at_1": 0.73649, "ndcg_at_10": 0.76355, "ndcg_at_100": 0.78456, "ndcg_at_1000": 0.79272, "ndcg_at_20": 0.77514, "ndcg_at_3": 0.71865, "ndcg_at_5": 0.7468, "precision_at_1": 0.73649, "precision_at_10": 0.15084, "precision_at_100": 0.01681, "precision_at_1000": 0.00181, "precision_at_20": 0.07922, "precision_at_3": 0.42342, "precision_at_5": 0.28378, "recall_at_1": 0.48055, "recall_at_10": 0.82061, "recall_at_100": 0.89764, "recall_at_1000": 0.95011, "recall_at_20": 0.85825, "recall_at_3": 0.72275, "recall_at_5": 0.77861 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 10329.316873311996, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.6227, "map_at_1": 0.3612, "map_at_10": 0.5315, "map_at_100": 0.55512, "map_at_1000": 0.55614, "map_at_20": 0.54556, "map_at_3": 0.47674, "map_at_5": 0.504, "mrr_at_1": 0.5868544600938967, "mrr_at_10": 0.6973116476637602, "mrr_at_100": 0.7002104887131142, "mrr_at_1000": 0.700290787057861, "mrr_at_20": 0.6987682505288138, "mrr_at_3": 0.6784037558685446, "mrr_at_5": 0.6906103286384975, "nauc_map_at_1000_diff1": 0.38764854132379667, "nauc_map_at_1000_max": 0.3113912044438687, "nauc_map_at_1000_std": -0.08692573899236988, "nauc_map_at_100_diff1": 0.3875727487743683, "nauc_map_at_100_max": 0.31084917316415417, "nauc_map_at_100_std": -0.08769034073115611, "nauc_map_at_10_diff1": 0.38338772006257255, "nauc_map_at_10_max": 0.2976298129992829, "nauc_map_at_10_std": -0.1240776524025288, "nauc_map_at_1_diff1": 0.48702256214431855, "nauc_map_at_1_max": 0.21748772342571, "nauc_map_at_1_std": -0.20305923228318654, "nauc_map_at_20_diff1": 0.38264216102917853, "nauc_map_at_20_max": 0.30415154290066665, "nauc_map_at_20_std": -0.09912139862787113, "nauc_map_at_3_diff1": 0.4037090768200007, "nauc_map_at_3_max": 0.27642763386759206, "nauc_map_at_3_std": -0.1810076463974879, "nauc_map_at_5_diff1": 0.3966359976722569, "nauc_map_at_5_max": 0.286925678426597, "nauc_map_at_5_std": -0.15198298329553037, "nauc_mrr_at_1000_diff1": 0.41284389043719655, "nauc_mrr_at_1000_max": 0.42117004847800693, "nauc_mrr_at_1000_std": 0.18800705458167233, "nauc_mrr_at_100_diff1": 0.41281828202187665, "nauc_mrr_at_100_max": 0.4212382136233428, "nauc_mrr_at_100_std": 0.18814564685189714, "nauc_mrr_at_10_diff1": 0.4120268901923224, "nauc_mrr_at_10_max": 0.422986788214752, "nauc_mrr_at_10_std": 0.18811292303183347, "nauc_mrr_at_1_diff1": 0.4487596742093259, "nauc_mrr_at_1_max": 0.3610578163366466, "nauc_mrr_at_1_std": 0.13225259650622362, "nauc_mrr_at_20_diff1": 0.41392097470507366, "nauc_mrr_at_20_max": 0.42359215144179657, "nauc_mrr_at_20_std": 0.19097457002225834, "nauc_mrr_at_3_diff1": 0.39752061736475947, "nauc_mrr_at_3_max": 0.41555272255813586, "nauc_mrr_at_3_std": 0.17157649602889363, "nauc_mrr_at_5_diff1": 0.4135361139700035, "nauc_mrr_at_5_max": 0.42384273859241994, "nauc_mrr_at_5_std": 0.18309769773126172, "nauc_ndcg_at_1000_diff1": 0.38558364488923336, "nauc_ndcg_at_1000_max": 0.3735443153990054, "nauc_ndcg_at_1000_std": 0.04997815724278142, "nauc_ndcg_at_100_diff1": 0.38109696465625686, "nauc_ndcg_at_100_max": 0.3708687670036079, "nauc_ndcg_at_100_std": 0.04744258222085883, "nauc_ndcg_at_10_diff1": 0.3655827203180003, "nauc_ndcg_at_10_max": 0.3384549245590441, "nauc_ndcg_at_10_std": -0.03945198134422429, "nauc_ndcg_at_1_diff1": 0.4487596742093259, "nauc_ndcg_at_1_max": 0.3610578163366466, "nauc_ndcg_at_1_std": 0.13225259650622362, "nauc_ndcg_at_20_diff1": 0.37125748141706266, "nauc_ndcg_at_20_max": 0.3566575938179187, "nauc_ndcg_at_20_std": 0.01618259053944107, "nauc_ndcg_at_3_diff1": 0.3744884533156548, "nauc_ndcg_at_3_max": 0.36185198492264975, "nauc_ndcg_at_3_std": -0.022525070094551996, "nauc_ndcg_at_5_diff1": 0.38897678279830183, "nauc_ndcg_at_5_max": 0.3506678858363011, "nauc_ndcg_at_5_std": -0.04899222506336306, "nauc_precision_at_1000_diff1": -0.13646893822028466, "nauc_precision_at_1000_max": 0.12066222521534775, "nauc_precision_at_1000_std": 0.45459076243223984, "nauc_precision_at_100_diff1": -0.12546392835057266, "nauc_precision_at_100_max": 0.1385245545085449, "nauc_precision_at_100_std": 0.4657881556877765, "nauc_precision_at_10_diff1": -0.142336052719195, "nauc_precision_at_10_max": 0.19599132675355047, "nauc_precision_at_10_std": 0.41448405304099284, "nauc_precision_at_1_diff1": 0.4487596742093259, "nauc_precision_at_1_max": 0.3610578163366466, "nauc_precision_at_1_std": 0.13225259650622362, "nauc_precision_at_20_diff1": -0.13734760079468608, "nauc_precision_at_20_max": 0.17268688686350742, "nauc_precision_at_20_std": 0.4652441859623385, "nauc_precision_at_3_diff1": -0.01385469062182767, "nauc_precision_at_3_max": 0.2865712095373297, "nauc_precision_at_3_std": 0.25408063870871217, "nauc_precision_at_5_diff1": -0.0725893843825658, "nauc_precision_at_5_max": 0.24986208705825372, "nauc_precision_at_5_std": 0.33964597495569715, "nauc_recall_at_1000_diff1": 0.43281509968634124, "nauc_recall_at_1000_max": 0.7773981194084716, "nauc_recall_at_1000_std": 0.7037305348610524, "nauc_recall_at_100_diff1": 0.24616775664220159, "nauc_recall_at_100_max": 0.4172532948489134, "nauc_recall_at_100_std": 0.23724557640896274, "nauc_recall_at_10_diff1": 0.26010019285097696, "nauc_recall_at_10_max": 0.28609614869854244, "nauc_recall_at_10_std": -0.09819697382128093, "nauc_recall_at_1_diff1": 0.48702256214431855, "nauc_recall_at_1_max": 0.21748772342571, "nauc_recall_at_1_std": -0.20305923228318654, "nauc_recall_at_20_diff1": 0.26857159375544104, "nauc_recall_at_20_max": 0.34876939981112565, "nauc_recall_at_20_std": 0.07342454797424504, "nauc_recall_at_3_diff1": 0.33046767471167193, "nauc_recall_at_3_max": 0.29281890571905794, "nauc_recall_at_3_std": -0.18490968486067033, "nauc_recall_at_5_diff1": 0.3280496356900794, "nauc_recall_at_5_max": 0.2887980433798689, "nauc_recall_at_5_std": -0.1628636660793256, "ndcg_at_1": 0.58685, "ndcg_at_10": 0.6227, "ndcg_at_100": 0.68376, "ndcg_at_1000": 0.69545, "ndcg_at_20": 0.64963, "ndcg_at_3": 0.58584, "ndcg_at_5": 0.59175, "precision_at_1": 0.58685, "precision_at_10": 0.1554, "precision_at_100": 0.0231, "precision_at_1000": 0.00253, "precision_at_20": 0.09108, "precision_at_3": 0.35055, "precision_at_5": 0.24977, "recall_at_1": 0.3612, "recall_at_10": 0.71449, "recall_at_100": 0.91555, "recall_at_1000": 0.98075, "recall_at_20": 0.78997, "recall_at_3": 0.54734, "recall_at_5": 0.617 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 10726.250535488129, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.58082, "map_at_1": 0.43191, "map_at_10": 0.5278, "map_at_100": 0.53635, "map_at_1000": 0.53672, "map_at_20": 0.53278, "map_at_3": 0.49637, "map_at_5": 0.51579, "mrr_at_1": 0.4655581947743468, "mrr_at_10": 0.5515750480714849, "mrr_at_100": 0.5582255676972496, "mrr_at_1000": 0.5585150923017264, "mrr_at_20": 0.5555904681690624, "mrr_at_3": 0.5241488519398259, "mrr_at_5": 0.5395882818685669, "nauc_map_at_1000_diff1": 0.4185245174995054, "nauc_map_at_1000_max": 0.40612966246897475, "nauc_map_at_1000_std": 0.048795216003709785, "nauc_map_at_100_diff1": 0.41815378361431316, "nauc_map_at_100_max": 0.4058548055479725, "nauc_map_at_100_std": 0.04868018064681098, "nauc_map_at_10_diff1": 0.4175731631836936, "nauc_map_at_10_max": 0.4036531634839384, "nauc_map_at_10_std": 0.045235507009600294, "nauc_map_at_1_diff1": 0.4730432659584148, "nauc_map_at_1_max": 0.4116036341712533, "nauc_map_at_1_std": 0.05590340783512278, "nauc_map_at_20_diff1": 0.4172847311708526, "nauc_map_at_20_max": 0.40438924348554933, "nauc_map_at_20_std": 0.04522925358150473, "nauc_map_at_3_diff1": 0.42754403200834207, "nauc_map_at_3_max": 0.41230793882923544, "nauc_map_at_3_std": 0.056123248270589544, "nauc_map_at_5_diff1": 0.414617225536972, "nauc_map_at_5_max": 0.4048782050759969, "nauc_map_at_5_std": 0.048760891180866864, "nauc_mrr_at_1000_diff1": 0.4391148789521434, "nauc_mrr_at_1000_max": 0.43037019215564914, "nauc_mrr_at_1000_std": 0.051646723959669176, "nauc_mrr_at_100_diff1": 0.4389393375094754, "nauc_mrr_at_100_max": 0.4301686328622996, "nauc_mrr_at_100_std": 0.051420021276688044, "nauc_mrr_at_10_diff1": 0.43897367091817935, "nauc_mrr_at_10_max": 0.43005799762942876, "nauc_mrr_at_10_std": 0.05127538650954391, "nauc_mrr_at_1_diff1": 0.4974245661547362, "nauc_mrr_at_1_max": 0.43988998412009594, "nauc_mrr_at_1_std": 0.05137648799598571, "nauc_mrr_at_20_diff1": 0.437611596877886, "nauc_mrr_at_20_max": 0.42924562204504574, "nauc_mrr_at_20_std": 0.04957879150663721, "nauc_mrr_at_3_diff1": 0.4389111677101245, "nauc_mrr_at_3_max": 0.4333270992230221, "nauc_mrr_at_3_std": 0.05629367157218375, "nauc_mrr_at_5_diff1": 0.4359000531297272, "nauc_mrr_at_5_max": 0.4303383727731648, "nauc_mrr_at_5_std": 0.055756374223376357, "nauc_ndcg_at_1000_diff1": 0.41343247137897293, "nauc_ndcg_at_1000_max": 0.4156390723661495, "nauc_ndcg_at_1000_std": 0.05808562888891213, "nauc_ndcg_at_100_diff1": 0.40359731894318585, "nauc_ndcg_at_100_max": 0.40760147296813215, "nauc_ndcg_at_100_std": 0.05489362113880834, "nauc_ndcg_at_10_diff1": 0.4019025213111373, "nauc_ndcg_at_10_max": 0.40016778751314225, "nauc_ndcg_at_10_std": 0.0381379917740792, "nauc_ndcg_at_1_diff1": 0.4974245661547362, "nauc_ndcg_at_1_max": 0.43988998412009594, "nauc_ndcg_at_1_std": 0.05137648799598571, "nauc_ndcg_at_20_diff1": 0.3985959250925596, "nauc_ndcg_at_20_max": 0.4006483776741761, "nauc_ndcg_at_20_std": 0.035963086709961394, "nauc_ndcg_at_3_diff1": 0.41339294018327666, "nauc_ndcg_at_3_max": 0.4154232762476081, "nauc_ndcg_at_3_std": 0.05749303837057296, "nauc_ndcg_at_5_diff1": 0.3948810565781645, "nauc_ndcg_at_5_max": 0.40297811344864953, "nauc_ndcg_at_5_std": 0.048612021354227104, "nauc_precision_at_1000_diff1": -0.023872356592962996, "nauc_precision_at_1000_max": 0.08608031696531497, "nauc_precision_at_1000_std": 0.07184301921198236, "nauc_precision_at_100_diff1": -0.010290794691752425, "nauc_precision_at_100_max": 0.11286433340170968, "nauc_precision_at_100_std": 0.07282243793129542, "nauc_precision_at_10_diff1": 0.17239822969194146, "nauc_precision_at_10_max": 0.25491440587453856, "nauc_precision_at_10_std": -0.01136630280583889, "nauc_precision_at_1_diff1": 0.4974245661547362, "nauc_precision_at_1_max": 0.43988998412009594, "nauc_precision_at_1_std": 0.05137648799598571, "nauc_precision_at_20_diff1": 0.09807534995702752, "nauc_precision_at_20_max": 0.20243662205690297, "nauc_precision_at_20_std": -0.009308783166631366, "nauc_precision_at_3_diff1": 0.30403982399381463, "nauc_precision_at_3_max": 0.376503959358296, "nauc_precision_at_3_std": 0.05426784715124511, "nauc_precision_at_5_diff1": 0.20336094469303898, "nauc_precision_at_5_max": 0.301871787288159, "nauc_precision_at_5_std": 0.031832913255998734, "nauc_recall_at_1000_diff1": 0.4741332939793106, "nauc_recall_at_1000_max": 0.6762796719315332, "nauc_recall_at_1000_std": 0.439130909069932, "nauc_recall_at_100_diff1": 0.24460260903502526, "nauc_recall_at_100_max": 0.3520020445129798, "nauc_recall_at_100_std": 0.13716817293473227, "nauc_recall_at_10_diff1": 0.3106688966871937, "nauc_recall_at_10_max": 0.3366772175043759, "nauc_recall_at_10_std": -0.006124281457655288, "nauc_recall_at_1_diff1": 0.4730432659584148, "nauc_recall_at_1_max": 0.4116036341712533, "nauc_recall_at_1_std": 0.05590340783512278, "nauc_recall_at_20_diff1": 0.28199555631776185, "nauc_recall_at_20_max": 0.3313221032143139, "nauc_recall_at_20_std": -0.029706893887273177, "nauc_recall_at_3_diff1": 0.3587914593094187, "nauc_recall_at_3_max": 0.39924211690350325, "nauc_recall_at_3_std": 0.05969615552219959, "nauc_recall_at_5_diff1": 0.3017520123963702, "nauc_recall_at_5_max": 0.3579689669410457, "nauc_recall_at_5_std": 0.036385597059597576, "ndcg_at_1": 0.46556, "ndcg_at_10": 0.58082, "ndcg_at_100": 0.61867, "ndcg_at_1000": 0.62788, "ndcg_at_20": 0.59634, "ndcg_at_3": 0.52094, "ndcg_at_5": 0.55244, "precision_at_1": 0.46556, "precision_at_10": 0.08171, "precision_at_100": 0.01036, "precision_at_1000": 0.00112, "precision_at_20": 0.04466, "precision_at_3": 0.20982, "precision_at_5": 0.14394, "recall_at_1": 0.43191, "recall_at_10": 0.72169, "recall_at_100": 0.88876, "recall_at_1000": 0.95922, "recall_at_20": 0.77672, "recall_at_3": 0.56532, "recall_at_5": 0.63737 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 365.5563635826111, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.23766, "map_at_1": 0.16, "map_at_10": 0.20903, "map_at_100": 0.21578, "map_at_1000": 0.21688, "map_at_20": 0.21168, "map_at_3": 0.19167, "map_at_5": 0.20242, "mrr_at_1": 0.16, "mrr_at_10": 0.20903174603174604, "mrr_at_100": 0.215783785522135, "mrr_at_1000": 0.2168796089420726, "mrr_at_20": 0.21167943861693864, "mrr_at_3": 0.19166666666666668, "mrr_at_5": 0.20241666666666663, "nauc_map_at_1000_diff1": 0.4056527499371557, "nauc_map_at_1000_max": 0.45971918394837746, "nauc_map_at_1000_std": 0.10888775044447328, "nauc_map_at_100_diff1": 0.4057775934260692, "nauc_map_at_100_max": 0.4593519956984368, "nauc_map_at_100_std": 0.10843762042187415, "nauc_map_at_10_diff1": 0.4111243025689681, "nauc_map_at_10_max": 0.46241285085930794, "nauc_map_at_10_std": 0.09961547366420115, "nauc_map_at_1_diff1": 0.4773870794284836, "nauc_map_at_1_max": 0.5224592871408816, "nauc_map_at_1_std": 0.11363112613304648, "nauc_map_at_20_diff1": 0.40796056090406, "nauc_map_at_20_max": 0.46179111942523543, "nauc_map_at_20_std": 0.10283069379805279, "nauc_map_at_3_diff1": 0.4344793643094571, "nauc_map_at_3_max": 0.48414054809763624, "nauc_map_at_3_std": 0.10250367348103344, "nauc_map_at_5_diff1": 0.42545269261198565, "nauc_map_at_5_max": 0.4792715776070097, "nauc_map_at_5_std": 0.10650287220411736, "nauc_mrr_at_1000_diff1": 0.4056527499371557, "nauc_mrr_at_1000_max": 0.45971918394837746, "nauc_mrr_at_1000_std": 0.10888775044447328, "nauc_mrr_at_100_diff1": 0.4057775934260692, "nauc_mrr_at_100_max": 0.4593519956984368, "nauc_mrr_at_100_std": 0.10843762042187415, "nauc_mrr_at_10_diff1": 0.4111243025689681, "nauc_mrr_at_10_max": 0.46241285085930794, "nauc_mrr_at_10_std": 0.09961547366420115, "nauc_mrr_at_1_diff1": 0.4773870794284836, "nauc_mrr_at_1_max": 0.5224592871408816, "nauc_mrr_at_1_std": 0.11363112613304648, "nauc_mrr_at_20_diff1": 0.40796056090406, "nauc_mrr_at_20_max": 0.46179111942523543, "nauc_mrr_at_20_std": 0.10283069379805279, "nauc_mrr_at_3_diff1": 0.4344793643094571, "nauc_mrr_at_3_max": 0.48414054809763624, "nauc_mrr_at_3_std": 0.10250367348103344, "nauc_mrr_at_5_diff1": 0.42545269261198565, "nauc_mrr_at_5_max": 0.4792715776070097, "nauc_mrr_at_5_std": 0.10650287220411736, "nauc_ndcg_at_1000_diff1": 0.357629836385146, "nauc_ndcg_at_1000_max": 0.42579081969985966, "nauc_ndcg_at_1000_std": 0.15118971276323878, "nauc_ndcg_at_100_diff1": 0.3582291264246711, "nauc_ndcg_at_100_max": 0.4188747272395376, "nauc_ndcg_at_100_std": 0.14084513620696146, "nauc_ndcg_at_10_diff1": 0.37899637323724716, "nauc_ndcg_at_10_max": 0.43004170234676586, "nauc_ndcg_at_10_std": 0.09353667167876474, "nauc_ndcg_at_1_diff1": 0.4773870794284836, "nauc_ndcg_at_1_max": 0.5224592871408816, "nauc_ndcg_at_1_std": 0.11363112613304648, "nauc_ndcg_at_20_diff1": 0.36734478358022304, "nauc_ndcg_at_20_max": 0.4274639301201697, "nauc_ndcg_at_20_std": 0.10651189016151837, "nauc_ndcg_at_3_diff1": 0.42527666873945985, "nauc_ndcg_at_3_max": 0.47536644100741793, "nauc_ndcg_at_3_std": 0.10162267713534295, "nauc_ndcg_at_5_diff1": 0.41059087810133726, "nauc_ndcg_at_5_max": 0.4675405208954469, "nauc_ndcg_at_5_std": 0.10930382183498212, "nauc_precision_at_1000_diff1": 0.12434933487565131, "nauc_precision_at_1000_max": 0.296620672560522, "nauc_precision_at_1000_std": 0.48898148038856926, "nauc_precision_at_100_diff1": 0.2187673059173419, "nauc_precision_at_100_max": 0.3049123166854683, "nauc_precision_at_100_std": 0.2800601647694252, "nauc_precision_at_10_diff1": 0.29423786129861146, "nauc_precision_at_10_max": 0.3431321643966932, "nauc_precision_at_10_std": 0.07613398836388804, "nauc_precision_at_1_diff1": 0.4773870794284836, "nauc_precision_at_1_max": 0.5224592871408816, "nauc_precision_at_1_std": 0.11363112613304648, "nauc_precision_at_20_diff1": 0.25535068360777796, "nauc_precision_at_20_max": 0.33514409955732005, "nauc_precision_at_20_std": 0.12291228665822236, "nauc_precision_at_3_diff1": 0.4023504827565167, "nauc_precision_at_3_max": 0.45330303076524175, "nauc_precision_at_3_std": 0.09998322173243263, "nauc_precision_at_5_diff1": 0.3740083244874019, "nauc_precision_at_5_max": 0.4389421196714906, "nauc_precision_at_5_std": 0.11802335325995887, "nauc_recall_at_1000_diff1": 0.12434933487565095, "nauc_recall_at_1000_max": 0.2966206725605223, "nauc_recall_at_1000_std": 0.48898148038856976, "nauc_recall_at_100_diff1": 0.21876730591734192, "nauc_recall_at_100_max": 0.304912316685468, "nauc_recall_at_100_std": 0.2800601647694252, "nauc_recall_at_10_diff1": 0.2942378612986113, "nauc_recall_at_10_max": 0.34313216439669286, "nauc_recall_at_10_std": 0.0761339883638881, "nauc_recall_at_1_diff1": 0.4773870794284836, "nauc_recall_at_1_max": 0.5224592871408816, "nauc_recall_at_1_std": 0.11363112613304648, "nauc_recall_at_20_diff1": 0.25535068360777774, "nauc_recall_at_20_max": 0.33514409955731994, "nauc_recall_at_20_std": 0.12291228665822235, "nauc_recall_at_3_diff1": 0.40235048275651664, "nauc_recall_at_3_max": 0.45330303076524137, "nauc_recall_at_3_std": 0.0999832217324324, "nauc_recall_at_5_diff1": 0.3740083244874014, "nauc_recall_at_5_max": 0.4389421196714899, "nauc_recall_at_5_std": 0.11802335325995847, "ndcg_at_1": 0.16, "ndcg_at_10": 0.23766, "ndcg_at_100": 0.2765, "ndcg_at_1000": 0.30876, "ndcg_at_20": 0.24761, "ndcg_at_3": 0.20155, "ndcg_at_5": 0.22155, "precision_at_1": 0.16, "precision_at_10": 0.033, "precision_at_100": 0.00525, "precision_at_1000": 0.00079, "precision_at_20": 0.0185, "precision_at_3": 0.07667, "precision_at_5": 0.056, "recall_at_1": 0.16, "recall_at_10": 0.33, "recall_at_100": 0.525, "recall_at_1000": 0.785, "recall_at_20": 0.37, "recall_at_3": 0.23, "recall_at_5": 0.28 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.22486, "map_at_1": 0.17, "map_at_10": 0.20452, "map_at_100": 0.21295, "map_at_1000": 0.21427, "map_at_20": 0.20945, "map_at_3": 0.19417, "map_at_5": 0.19992, "mrr_at_1": 0.17, "mrr_at_10": 0.20451984126984127, "mrr_at_100": 0.2129479826793562, "mrr_at_1000": 0.2142721941179602, "mrr_at_20": 0.20945311712281525, "mrr_at_3": 0.19416666666666665, "mrr_at_5": 0.19991666666666666, "nauc_map_at_1000_diff1": 0.5087580113845908, "nauc_map_at_1000_max": 0.5179525106865259, "nauc_map_at_1000_std": 0.0847410446203826, "nauc_map_at_100_diff1": 0.5089305311988365, "nauc_map_at_100_max": 0.5178971110642436, "nauc_map_at_100_std": 0.08419962372042392, "nauc_map_at_10_diff1": 0.5182162044962908, "nauc_map_at_10_max": 0.5259011127234215, "nauc_map_at_10_std": 0.08045608374070097, "nauc_map_at_1_diff1": 0.5934927299730468, "nauc_map_at_1_max": 0.5180696382405251, "nauc_map_at_1_std": 0.023432772878123738, "nauc_map_at_20_diff1": 0.5113697633838645, "nauc_map_at_20_max": 0.5210348343486106, "nauc_map_at_20_std": 0.08549340137837198, "nauc_map_at_3_diff1": 0.525610817973501, "nauc_map_at_3_max": 0.5258851216290541, "nauc_map_at_3_std": 0.07157074713282353, "nauc_map_at_5_diff1": 0.524481893074894, "nauc_map_at_5_max": 0.5200983900181767, "nauc_map_at_5_std": 0.08209387595688832, "nauc_mrr_at_1000_diff1": 0.5087580113845908, "nauc_mrr_at_1000_max": 0.5179525106865259, "nauc_mrr_at_1000_std": 0.0847410446203826, "nauc_mrr_at_100_diff1": 0.5089305311988365, "nauc_mrr_at_100_max": 0.5178971110642436, "nauc_mrr_at_100_std": 0.08419962372042392, "nauc_mrr_at_10_diff1": 0.5182162044962908, "nauc_mrr_at_10_max": 0.5259011127234215, "nauc_mrr_at_10_std": 0.08045608374070097, "nauc_mrr_at_1_diff1": 0.5934927299730468, "nauc_mrr_at_1_max": 0.5180696382405251, "nauc_mrr_at_1_std": 0.023432772878123738, "nauc_mrr_at_20_diff1": 0.5113697633838645, "nauc_mrr_at_20_max": 0.5210348343486106, "nauc_mrr_at_20_std": 0.08549340137837198, "nauc_mrr_at_3_diff1": 0.525610817973501, "nauc_mrr_at_3_max": 0.5258851216290541, "nauc_mrr_at_3_std": 0.07157074713282353, "nauc_mrr_at_5_diff1": 0.524481893074894, "nauc_mrr_at_5_max": 0.5200983900181767, "nauc_mrr_at_5_std": 0.08209387595688832, "nauc_ndcg_at_1000_diff1": 0.46291688826516936, "nauc_ndcg_at_1000_max": 0.5076348172971382, "nauc_ndcg_at_1000_std": 0.13996126632352657, "nauc_ndcg_at_100_diff1": 0.45761311543315036, "nauc_ndcg_at_100_max": 0.49986715299588624, "nauc_ndcg_at_100_std": 0.11573952558693551, "nauc_ndcg_at_10_diff1": 0.48659095860512674, "nauc_ndcg_at_10_max": 0.5306997041080913, "nauc_ndcg_at_10_std": 0.09827948773797913, "nauc_ndcg_at_1_diff1": 0.5934927299730468, "nauc_ndcg_at_1_max": 0.5180696382405251, "nauc_ndcg_at_1_std": 0.023432772878123738, "nauc_ndcg_at_20_diff1": 0.4662179168588857, "nauc_ndcg_at_20_max": 0.5166720253899405, "nauc_ndcg_at_20_std": 0.11924750957281219, "nauc_ndcg_at_3_diff1": 0.5024480480565693, "nauc_ndcg_at_3_max": 0.5281856243842368, "nauc_ndcg_at_3_std": 0.08633380109016073, "nauc_ndcg_at_5_diff1": 0.5002530444688492, "nauc_ndcg_at_5_max": 0.5181217847759284, "nauc_ndcg_at_5_std": 0.10348046484708498, "nauc_precision_at_1000_diff1": 0.28709694740515956, "nauc_precision_at_1000_max": 0.4903482159637017, "nauc_precision_at_1000_std": 0.6286546567508802, "nauc_precision_at_100_diff1": 0.3174783690136539, "nauc_precision_at_100_max": 0.4325266189155581, "nauc_precision_at_100_std": 0.20916714883315735, "nauc_precision_at_10_diff1": 0.40477040781286505, "nauc_precision_at_10_max": 0.5445586033542574, "nauc_precision_at_10_std": 0.1402895485076166, "nauc_precision_at_1_diff1": 0.5934927299730468, "nauc_precision_at_1_max": 0.5180696382405251, "nauc_precision_at_1_std": 0.023432772878123738, "nauc_precision_at_20_diff1": 0.3454784640456575, "nauc_precision_at_20_max": 0.502143450857708, "nauc_precision_at_20_std": 0.21520901100195702, "nauc_precision_at_3_diff1": 0.4409481246472753, "nauc_precision_at_3_max": 0.5342047863821483, "nauc_precision_at_3_std": 0.12512774362807152, "nauc_precision_at_5_diff1": 0.4373746874020287, "nauc_precision_at_5_max": 0.5119097093112275, "nauc_precision_at_5_std": 0.15933565184423698, "nauc_recall_at_1000_diff1": 0.28709694740516095, "nauc_recall_at_1000_max": 0.49034821596370226, "nauc_recall_at_1000_std": 0.628654656750879, "nauc_recall_at_100_diff1": 0.3174783690136534, "nauc_recall_at_100_max": 0.43252661891555766, "nauc_recall_at_100_std": 0.20916714883315712, "nauc_recall_at_10_diff1": 0.40477040781286494, "nauc_recall_at_10_max": 0.5445586033542575, "nauc_recall_at_10_std": 0.14028954850761638, "nauc_recall_at_1_diff1": 0.5934927299730468, "nauc_recall_at_1_max": 0.5180696382405251, "nauc_recall_at_1_std": 0.023432772878123738, "nauc_recall_at_20_diff1": 0.34547846404565713, "nauc_recall_at_20_max": 0.5021434508577074, "nauc_recall_at_20_std": 0.21520901100195683, "nauc_recall_at_3_diff1": 0.44094812464727506, "nauc_recall_at_3_max": 0.534204786382148, "nauc_recall_at_3_std": 0.12512774362807147, "nauc_recall_at_5_diff1": 0.43737468740202834, "nauc_recall_at_5_max": 0.5119097093112275, "nauc_recall_at_5_std": 0.15933565184423693, "ndcg_at_1": 0.17, "ndcg_at_10": 0.22486, "ndcg_at_100": 0.26791, "ndcg_at_1000": 0.30737, "ndcg_at_20": 0.24349, "ndcg_at_3": 0.20327, "ndcg_at_5": 0.2136, "precision_at_1": 0.17, "precision_at_10": 0.029, "precision_at_100": 0.00495, "precision_at_1000": 0.00082, "precision_at_20": 0.01825, "precision_at_3": 0.07667, "precision_at_5": 0.051, "recall_at_1": 0.17, "recall_at_10": 0.29, "recall_at_100": 0.495, "recall_at_1000": 0.815, "recall_at_20": 0.365, "recall_at_3": 0.23, "recall_at_5": 0.255 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 2.6252050399780273, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.77203, "map_at_1": 0.58442, "map_at_10": 0.7159, "map_at_100": 0.71802, "map_at_1000": 0.71802, "map_at_20": 0.71677, "map_at_3": 0.69264, "map_at_5": 0.70238, "mrr_at_1": 0.5844155844155844, "mrr_at_10": 0.7159039373325088, "mrr_at_100": 0.7180190902896827, "mrr_at_1000": 0.7180190902896827, "mrr_at_20": 0.7167697381983097, "mrr_at_3": 0.6926406926406926, "mrr_at_5": 0.7023809523809523, "nauc_map_at_1000_diff1": 0.7049122955851462, "nauc_map_at_1000_max": 0.1749739403055711, "nauc_map_at_1000_std": -0.11753044732058268, "nauc_map_at_100_diff1": 0.7049122955851462, "nauc_map_at_100_max": 0.1749739403055711, "nauc_map_at_100_std": -0.11753044732058268, "nauc_map_at_10_diff1": 0.7059280638537124, "nauc_map_at_10_max": 0.17576332204748918, "nauc_map_at_10_std": -0.11259325010112323, "nauc_map_at_1_diff1": 0.7730849905549931, "nauc_map_at_1_max": 0.16323654437643398, "nauc_map_at_1_std": -0.15463541123486912, "nauc_map_at_20_diff1": 0.7048274115980733, "nauc_map_at_20_max": 0.17371661217517576, "nauc_map_at_20_std": -0.11626608139650574, "nauc_map_at_3_diff1": 0.6895156048217616, "nauc_map_at_3_max": 0.18438045879970635, "nauc_map_at_3_std": -0.0969266403909441, "nauc_map_at_5_diff1": 0.689721270225883, "nauc_map_at_5_max": 0.18085499936323893, "nauc_map_at_5_std": -0.11320280218313829, "nauc_mrr_at_1000_diff1": 0.7049122955851462, "nauc_mrr_at_1000_max": 0.1749739403055711, "nauc_mrr_at_1000_std": -0.11753044732058268, "nauc_mrr_at_100_diff1": 0.7049122955851462, "nauc_mrr_at_100_max": 0.1749739403055711, "nauc_mrr_at_100_std": -0.11753044732058268, "nauc_mrr_at_10_diff1": 0.7059280638537124, "nauc_mrr_at_10_max": 0.17576332204748918, "nauc_mrr_at_10_std": -0.11259325010112323, "nauc_mrr_at_1_diff1": 0.7730849905549931, "nauc_mrr_at_1_max": 0.16323654437643398, "nauc_mrr_at_1_std": -0.15463541123486912, "nauc_mrr_at_20_diff1": 0.7048274115980733, "nauc_mrr_at_20_max": 0.17371661217517576, "nauc_mrr_at_20_std": -0.11626608139650574, "nauc_mrr_at_3_diff1": 0.6895156048217616, "nauc_mrr_at_3_max": 0.18438045879970635, "nauc_mrr_at_3_std": -0.0969266403909441, "nauc_mrr_at_5_diff1": 0.689721270225883, "nauc_mrr_at_5_max": 0.18085499936323893, "nauc_mrr_at_5_std": -0.11320280218313829, "nauc_ndcg_at_1000_diff1": 0.6972930152328541, "nauc_ndcg_at_1000_max": 0.17338517487228328, "nauc_ndcg_at_1000_std": -0.1123379011258602, "nauc_ndcg_at_100_diff1": 0.6972930152328541, "nauc_ndcg_at_100_max": 0.17338517487228328, "nauc_ndcg_at_100_std": -0.1123379011258602, "nauc_ndcg_at_10_diff1": 0.7000221387691848, "nauc_ndcg_at_10_max": 0.17175618246048177, "nauc_ndcg_at_10_std": -0.08579200170290963, "nauc_ndcg_at_1_diff1": 0.7730849905549931, "nauc_ndcg_at_1_max": 0.16323654437643398, "nauc_ndcg_at_1_std": -0.15463541123486912, "nauc_ndcg_at_20_diff1": 0.6947883208861702, "nauc_ndcg_at_20_max": 0.16214537299119408, "nauc_ndcg_at_20_std": -0.10244557663552743, "nauc_ndcg_at_3_diff1": 0.6574658310748505, "nauc_ndcg_at_3_max": 0.19094549989384266, "nauc_ndcg_at_3_std": -0.07151120593375672, "nauc_ndcg_at_5_diff1": 0.6557380415427442, "nauc_ndcg_at_5_max": 0.18463280081684605, "nauc_ndcg_at_5_std": -0.1009792734356102, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.7120602067801151, "nauc_precision_at_10_max": 0.09791382994502315, "nauc_precision_at_10_std": 0.31479849081120154, "nauc_precision_at_1_diff1": 0.7730849905549931, "nauc_precision_at_1_max": 0.16323654437643398, "nauc_precision_at_1_std": -0.15463541123486912, "nauc_precision_at_20_diff1": 0.616080275706823, "nauc_precision_at_20_max": -0.11031577525540454, "nauc_precision_at_20_std": 0.1301604202575047, "nauc_precision_at_3_diff1": 0.5312918444354878, "nauc_precision_at_3_max": 0.2160886881649667, "nauc_precision_at_3_std": 0.03003794963804101, "nauc_precision_at_5_diff1": 0.4967070686214641, "nauc_precision_at_5_max": 0.20013872348340253, "nauc_precision_at_5_std": -0.04801064361272453, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.7120602067801166, "nauc_recall_at_10_max": 0.09791382994501428, "nauc_recall_at_10_std": 0.3147984908111977, "nauc_recall_at_1_diff1": 0.7730849905549931, "nauc_recall_at_1_max": 0.16323654437643398, "nauc_recall_at_1_std": -0.15463541123486912, "nauc_recall_at_20_diff1": 0.6160802757068197, "nauc_recall_at_20_max": -0.1103157752554104, "nauc_recall_at_20_std": 0.13016042025750219, "nauc_recall_at_3_diff1": 0.5312918444354875, "nauc_recall_at_3_max": 0.2160886881649647, "nauc_recall_at_3_std": 0.03003794963803853, "nauc_recall_at_5_diff1": 0.49670706862146197, "nauc_recall_at_5_max": 0.20013872348340017, "nauc_recall_at_5_std": -0.048010643612726514, "ndcg_at_1": 0.58442, "ndcg_at_10": 0.77203, "ndcg_at_100": 0.78303, "ndcg_at_1000": 0.78303, "ndcg_at_20": 0.77528, "ndcg_at_3": 0.72201, "ndcg_at_5": 0.73879, "precision_at_1": 0.58442, "precision_at_10": 0.09481, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04805, "precision_at_3": 0.2684, "precision_at_5": 0.16883, "recall_at_1": 0.58442, "recall_at_10": 0.94805, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.96104, "recall_at_3": 0.80519, "recall_at_5": 0.84416 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 16.628807067871094, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.3607, "map_at_1": 0.21481, "map_at_10": 0.3182, "map_at_100": 0.33387, "map_at_1000": 0.33517, "map_at_20": 0.32628, "map_at_3": 0.2905, "map_at_5": 0.30788, "mrr_at_1": 0.30275229357798167, "mrr_at_10": 0.36139568467550115, "mrr_at_100": 0.3727459173479044, "mrr_at_1000": 0.3735149401359442, "mrr_at_20": 0.3669863817077084, "mrr_at_3": 0.3440366972477066, "mrr_at_5": 0.35328746177370046, "nauc_map_at_1000_diff1": 0.42963585760689943, "nauc_map_at_1000_max": 0.20892123488970232, "nauc_map_at_1000_std": -0.2431027065544824, "nauc_map_at_100_diff1": 0.42937913438871533, "nauc_map_at_100_max": 0.2090349260230252, "nauc_map_at_100_std": -0.24287170422372528, "nauc_map_at_10_diff1": 0.4304977743091833, "nauc_map_at_10_max": 0.20818062207572038, "nauc_map_at_10_std": -0.2470407473817399, "nauc_map_at_1_diff1": 0.48514432281307956, "nauc_map_at_1_max": 0.15959686126723213, "nauc_map_at_1_std": -0.20796331653269645, "nauc_map_at_20_diff1": 0.4305756124522112, "nauc_map_at_20_max": 0.20730123860183938, "nauc_map_at_20_std": -0.24760022109278698, "nauc_map_at_3_diff1": 0.4426742232767405, "nauc_map_at_3_max": 0.21074344546441529, "nauc_map_at_3_std": -0.2386688960545258, "nauc_map_at_5_diff1": 0.4332746920752696, "nauc_map_at_5_max": 0.2133348686910522, "nauc_map_at_5_std": -0.24223475070935185, "nauc_mrr_at_1000_diff1": 0.44625683270525074, "nauc_mrr_at_1000_max": 0.2106574277517107, "nauc_mrr_at_1000_std": -0.2677092441216044, "nauc_mrr_at_100_diff1": 0.4461128558106053, "nauc_mrr_at_100_max": 0.21078043248107986, "nauc_mrr_at_100_std": -0.2674632995361709, "nauc_mrr_at_10_diff1": 0.444987740964291, "nauc_mrr_at_10_max": 0.20799867789822493, "nauc_mrr_at_10_std": -0.2729638141198885, "nauc_mrr_at_1_diff1": 0.48504640241831776, "nauc_mrr_at_1_max": 0.22365383306396908, "nauc_mrr_at_1_std": -0.26645605434126035, "nauc_mrr_at_20_diff1": 0.4466793381008902, "nauc_mrr_at_20_max": 0.21003273255434113, "nauc_mrr_at_20_std": -0.2696298492568519, "nauc_mrr_at_3_diff1": 0.4504803999602948, "nauc_mrr_at_3_max": 0.20896914537040218, "nauc_mrr_at_3_std": -0.2730896968385306, "nauc_mrr_at_5_diff1": 0.44887921995905455, "nauc_mrr_at_5_max": 0.20983553173203578, "nauc_mrr_at_5_std": -0.2729058093449394, "nauc_ndcg_at_1000_diff1": 0.42053386492803835, "nauc_ndcg_at_1000_max": 0.21026761107634181, "nauc_ndcg_at_1000_std": -0.23853119910664403, "nauc_ndcg_at_100_diff1": 0.4124642636421199, "nauc_ndcg_at_100_max": 0.21297954341153955, "nauc_ndcg_at_100_std": -0.2270552761293387, "nauc_ndcg_at_10_diff1": 0.4163086383546011, "nauc_ndcg_at_10_max": 0.1999755251857054, "nauc_ndcg_at_10_std": -0.2601310454387032, "nauc_ndcg_at_1_diff1": 0.48504640241831776, "nauc_ndcg_at_1_max": 0.22365383306396908, "nauc_ndcg_at_1_std": -0.26645605434126035, "nauc_ndcg_at_20_diff1": 0.41834947593121535, "nauc_ndcg_at_20_max": 0.2003429851075697, "nauc_ndcg_at_20_std": -0.25606159299219916, "nauc_ndcg_at_3_diff1": 0.43553101733767347, "nauc_ndcg_at_3_max": 0.20549898662174343, "nauc_ndcg_at_3_std": -0.2533596592071201, "nauc_ndcg_at_5_diff1": 0.4225304579550925, "nauc_ndcg_at_5_max": 0.2068825667359891, "nauc_ndcg_at_5_std": -0.2544813137224287, "nauc_precision_at_1000_diff1": -0.018914124872467415, "nauc_precision_at_1000_max": 0.13281207973414894, "nauc_precision_at_1000_std": 0.03702777512602303, "nauc_precision_at_100_diff1": 0.08052988850710768, "nauc_precision_at_100_max": 0.1831172991717435, "nauc_precision_at_100_std": 0.0015248727721582668, "nauc_precision_at_10_diff1": 0.2383661017481684, "nauc_precision_at_10_max": 0.1794621672513022, "nauc_precision_at_10_std": -0.23894429362629943, "nauc_precision_at_1_diff1": 0.48504640241831776, "nauc_precision_at_1_max": 0.22365383306396908, "nauc_precision_at_1_std": -0.26645605434126035, "nauc_precision_at_20_diff1": 0.2085622758297434, "nauc_precision_at_20_max": 0.16392545018550997, "nauc_precision_at_20_std": -0.20028651530585947, "nauc_precision_at_3_diff1": 0.3197277868626114, "nauc_precision_at_3_max": 0.23054771411644534, "nauc_precision_at_3_std": -0.26043053473303907, "nauc_precision_at_5_diff1": 0.27302904750591267, "nauc_precision_at_5_max": 0.211260301971109, "nauc_precision_at_5_std": -0.24206268208121878, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.28168730538400805, "nauc_recall_at_100_max": 0.22809734919206476, "nauc_recall_at_100_std": -0.0639689151589093, "nauc_recall_at_10_diff1": 0.34935722006409087, "nauc_recall_at_10_max": 0.17132518446252754, "nauc_recall_at_10_std": -0.2645301871034324, "nauc_recall_at_1_diff1": 0.48514432281307956, "nauc_recall_at_1_max": 0.15959686126723213, "nauc_recall_at_1_std": -0.20796331653269645, "nauc_recall_at_20_diff1": 0.3551238752877474, "nauc_recall_at_20_max": 0.17073061417496116, "nauc_recall_at_20_std": -0.25110422810336486, "nauc_recall_at_3_diff1": 0.3936308854125494, "nauc_recall_at_3_max": 0.18498685148507174, "nauc_recall_at_3_std": -0.24862344210225615, "nauc_recall_at_5_diff1": 0.3677562165817916, "nauc_recall_at_5_max": 0.18766267088899702, "nauc_recall_at_5_std": -0.25256901516345587, "ndcg_at_1": 0.30275, "ndcg_at_10": 0.3607, "ndcg_at_100": 0.43626, "ndcg_at_1000": 0.46606, "ndcg_at_20": 0.38572, "ndcg_at_3": 0.31866, "ndcg_at_5": 0.33908, "precision_at_1": 0.30275, "precision_at_10": 0.07141, "precision_at_100": 0.01248, "precision_at_1000": 0.00156, "precision_at_20": 0.04297, "precision_at_3": 0.17482, "precision_at_5": 0.12263, "recall_at_1": 0.21481, "recall_at_10": 0.44429, "recall_at_100": 0.78193, "recall_at_1000": 1.0, "recall_at_20": 0.53168, "recall_at_3": 0.33372, "recall_at_5": 0.38509 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.20148, "map_at_1": 0.09414, "map_at_10": 0.15803, "map_at_100": 0.1757, "map_at_1000": 0.17866, "map_at_20": 0.16692, "map_at_3": 0.13541, "map_at_5": 0.14789, "mrr_at_1": 0.17889908256880735, "mrr_at_10": 0.2286703315373039, "mrr_at_100": 0.241428725969669, "mrr_at_1000": 0.2427723235325885, "mrr_at_20": 0.23509999732785672, "mrr_at_3": 0.20998980632008152, "mrr_at_5": 0.21847604485219163, "nauc_map_at_1000_diff1": 0.30539350162203316, "nauc_map_at_1000_max": 0.24034173633656167, "nauc_map_at_1000_std": -0.08103295111936148, "nauc_map_at_100_diff1": 0.30489058048932155, "nauc_map_at_100_max": 0.24119416853033185, "nauc_map_at_100_std": -0.08083640255787466, "nauc_map_at_10_diff1": 0.3056511815165923, "nauc_map_at_10_max": 0.22596714633779053, "nauc_map_at_10_std": -0.10241596640938393, "nauc_map_at_1_diff1": 0.42383841325418287, "nauc_map_at_1_max": 0.250405206326011, "nauc_map_at_1_std": -0.13520336912467026, "nauc_map_at_20_diff1": 0.3057776643050341, "nauc_map_at_20_max": 0.23641728576352428, "nauc_map_at_20_std": -0.09051814637606816, "nauc_map_at_3_diff1": 0.3308851492829308, "nauc_map_at_3_max": 0.22854232581291226, "nauc_map_at_3_std": -0.10839033935341609, "nauc_map_at_5_diff1": 0.31457141895678165, "nauc_map_at_5_max": 0.2243829020636881, "nauc_map_at_5_std": -0.11013395271043995, "nauc_mrr_at_1000_diff1": 0.31889757879784597, "nauc_mrr_at_1000_max": 0.22436637919163335, "nauc_mrr_at_1000_std": -0.07825944228943346, "nauc_mrr_at_100_diff1": 0.31855404076186933, "nauc_mrr_at_100_max": 0.22480172438918175, "nauc_mrr_at_100_std": -0.07726923319110378, "nauc_mrr_at_10_diff1": 0.3177409232150682, "nauc_mrr_at_10_max": 0.21907309827774715, "nauc_mrr_at_10_std": -0.08561257776101205, "nauc_mrr_at_1_diff1": 0.3948794314506051, "nauc_mrr_at_1_max": 0.2504301745012543, "nauc_mrr_at_1_std": -0.10573813009386544, "nauc_mrr_at_20_diff1": 0.31854162686673243, "nauc_mrr_at_20_max": 0.22334733477690014, "nauc_mrr_at_20_std": -0.07996475055973579, "nauc_mrr_at_3_diff1": 0.3306660624982572, "nauc_mrr_at_3_max": 0.22066001330319143, "nauc_mrr_at_3_std": -0.0935538960245645, "nauc_mrr_at_5_diff1": 0.3215903882649284, "nauc_mrr_at_5_max": 0.21498576898930763, "nauc_mrr_at_5_std": -0.09712281554924904, "nauc_ndcg_at_1000_diff1": 0.28568574379509243, "nauc_ndcg_at_1000_max": 0.24343162091807738, "nauc_ndcg_at_1000_std": -0.03778789550660657, "nauc_ndcg_at_100_diff1": 0.2725495542324967, "nauc_ndcg_at_100_max": 0.26547890081249387, "nauc_ndcg_at_100_std": -0.01087246106286027, "nauc_ndcg_at_10_diff1": 0.2790712927892886, "nauc_ndcg_at_10_max": 0.2204802656578329, "nauc_ndcg_at_10_std": -0.08315669038036567, "nauc_ndcg_at_1_diff1": 0.4033369499664993, "nauc_ndcg_at_1_max": 0.25281952813321773, "nauc_ndcg_at_1_std": -0.10322278163978298, "nauc_ndcg_at_20_diff1": 0.28192700653801284, "nauc_ndcg_at_20_max": 0.24675490880933862, "nauc_ndcg_at_20_std": -0.05094524679791449, "nauc_ndcg_at_3_diff1": 0.30540317541255185, "nauc_ndcg_at_3_max": 0.21933385751984275, "nauc_ndcg_at_3_std": -0.09785653718888704, "nauc_ndcg_at_5_diff1": 0.290223062445283, "nauc_ndcg_at_5_max": 0.21300270312838201, "nauc_ndcg_at_5_std": -0.10367682596724503, "nauc_precision_at_1000_diff1": 0.02383291229758173, "nauc_precision_at_1000_max": 0.027010048687200763, "nauc_precision_at_1000_std": 0.09102651003545263, "nauc_precision_at_100_diff1": 0.09759601754061357, "nauc_precision_at_100_max": 0.2321220026260983, "nauc_precision_at_100_std": 0.17556095540343836, "nauc_precision_at_10_diff1": 0.18029298355404208, "nauc_precision_at_10_max": 0.19742365753024946, "nauc_precision_at_10_std": -0.022400643864277075, "nauc_precision_at_1_diff1": 0.4033369499664993, "nauc_precision_at_1_max": 0.25281952813321773, "nauc_precision_at_1_std": -0.10322278163978298, "nauc_precision_at_20_diff1": 0.17529097352342055, "nauc_precision_at_20_max": 0.23367065227326117, "nauc_precision_at_20_std": 0.04838154362383047, "nauc_precision_at_3_diff1": 0.24438827121376572, "nauc_precision_at_3_max": 0.21076596647953377, "nauc_precision_at_3_std": -0.06626938359139384, "nauc_precision_at_5_diff1": 0.20700221690766976, "nauc_precision_at_5_max": 0.18749098508944212, "nauc_precision_at_5_std": -0.07188348358340925, "nauc_recall_at_1000_diff1": 0.5758326388163159, "nauc_recall_at_1000_max": 0.8156349912513978, "nauc_recall_at_1000_std": 0.7555715146667914, "nauc_recall_at_100_diff1": 0.1418479527830795, "nauc_recall_at_100_max": 0.31793376334708096, "nauc_recall_at_100_std": 0.17142940080470698, "nauc_recall_at_10_diff1": 0.1946626084556666, "nauc_recall_at_10_max": 0.19458983448728884, "nauc_recall_at_10_std": -0.051337513468547806, "nauc_recall_at_1_diff1": 0.42383841325418287, "nauc_recall_at_1_max": 0.250405206326011, "nauc_recall_at_1_std": -0.13520336912467026, "nauc_recall_at_20_diff1": 0.20122402738932615, "nauc_recall_at_20_max": 0.2629741535771735, "nauc_recall_at_20_std": 0.034103975248139094, "nauc_recall_at_3_diff1": 0.24753555717964568, "nauc_recall_at_3_max": 0.19120399927021967, "nauc_recall_at_3_std": -0.08494071412694883, "nauc_recall_at_5_diff1": 0.21679989815496223, "nauc_recall_at_5_max": 0.18119340444328566, "nauc_recall_at_5_std": -0.09814578598765797, "ndcg_at_1": 0.17737, "ndcg_at_10": 0.20148, "ndcg_at_100": 0.28543, "ndcg_at_1000": 0.34227, "ndcg_at_20": 0.22923, "ndcg_at_3": 0.16856, "ndcg_at_5": 0.17856, "precision_at_1": 0.17737, "precision_at_10": 0.05291, "precision_at_100": 0.01229, "precision_at_1000": 0.00194, "precision_at_20": 0.0354, "precision_at_3": 0.11009, "precision_at_5": 0.08135, "recall_at_1": 0.09414, "recall_at_10": 0.26055, "recall_at_100": 0.61476, "recall_at_1000": 0.99592, "recall_at_20": 0.35497, "recall_at_3": 0.16374, "recall_at_5": 0.20064 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.29263, "map_at_1": 0.14722, "map_at_10": 0.24572, "map_at_100": 0.26066, "map_at_1000": 0.26234, "map_at_20": 0.25264, "map_at_3": 0.2117, "map_at_5": 0.23153, "mrr_at_1": 0.2182410423452769, "mrr_at_10": 0.2844462540716614, "mrr_at_100": 0.29499249139067935, "mrr_at_1000": 0.2960265190976579, "mrr_at_20": 0.2901179168210185, "mrr_at_3": 0.26112920738327916, "mrr_at_5": 0.2736699239956571, "nauc_map_at_1000_diff1": 0.36364393096869635, "nauc_map_at_1000_max": 0.1801080153267461, "nauc_map_at_1000_std": -0.08999792998748074, "nauc_map_at_100_diff1": 0.3630141643535187, "nauc_map_at_100_max": 0.17991407524169148, "nauc_map_at_100_std": -0.08982299640070364, "nauc_map_at_10_diff1": 0.3658732019343366, "nauc_map_at_10_max": 0.17822484528135826, "nauc_map_at_10_std": -0.09279843246339788, "nauc_map_at_1_diff1": 0.5070734238811194, "nauc_map_at_1_max": 0.24529236270282476, "nauc_map_at_1_std": -0.004688212359841111, "nauc_map_at_20_diff1": 0.36207907758581914, "nauc_map_at_20_max": 0.17525189915636535, "nauc_map_at_20_std": -0.09435244487831614, "nauc_map_at_3_diff1": 0.40295825342965635, "nauc_map_at_3_max": 0.20104012126833412, "nauc_map_at_3_std": -0.07628803518006758, "nauc_map_at_5_diff1": 0.37612971547715945, "nauc_map_at_5_max": 0.19077273609554368, "nauc_map_at_5_std": -0.08499496025998478, "nauc_mrr_at_1000_diff1": 0.3798619434872802, "nauc_mrr_at_1000_max": 0.19885025712420953, "nauc_mrr_at_1000_std": -0.10458091620949982, "nauc_mrr_at_100_diff1": 0.37958597939618843, "nauc_mrr_at_100_max": 0.19890610868394065, "nauc_mrr_at_100_std": -0.10431192413324031, "nauc_mrr_at_10_diff1": 0.3816613970826813, "nauc_mrr_at_10_max": 0.19735653109654772, "nauc_mrr_at_10_std": -0.10730444084305658, "nauc_mrr_at_1_diff1": 0.4446874864765066, "nauc_mrr_at_1_max": 0.23916999910934264, "nauc_mrr_at_1_std": -0.08078870533756279, "nauc_mrr_at_20_diff1": 0.37807188953601084, "nauc_mrr_at_20_max": 0.19650524967244096, "nauc_mrr_at_20_std": -0.10639377645047687, "nauc_mrr_at_3_diff1": 0.3904534717117424, "nauc_mrr_at_3_max": 0.21086282986637608, "nauc_mrr_at_3_std": -0.1023624177806897, "nauc_mrr_at_5_diff1": 0.38401972122916894, "nauc_mrr_at_5_max": 0.20353606462281326, "nauc_mrr_at_5_std": -0.10743586414085929, "nauc_ndcg_at_1000_diff1": 0.3514196895142937, "nauc_ndcg_at_1000_max": 0.1808114461842303, "nauc_ndcg_at_1000_std": -0.08996523853646911, "nauc_ndcg_at_100_diff1": 0.3395612777980135, "nauc_ndcg_at_100_max": 0.1796525016824039, "nauc_ndcg_at_100_std": -0.08122542384351605, "nauc_ndcg_at_10_diff1": 0.34342650182736895, "nauc_ndcg_at_10_max": 0.1598603925312353, "nauc_ndcg_at_10_std": -0.10739253057810857, "nauc_ndcg_at_1_diff1": 0.4446874864765066, "nauc_ndcg_at_1_max": 0.23916999910934264, "nauc_ndcg_at_1_std": -0.08078870533756279, "nauc_ndcg_at_20_diff1": 0.3305328263689251, "nauc_ndcg_at_20_max": 0.15180270912134258, "nauc_ndcg_at_20_std": -0.10958156492422332, "nauc_ndcg_at_3_diff1": 0.38377007939763225, "nauc_ndcg_at_3_max": 0.19204739459298753, "nauc_ndcg_at_3_std": -0.09359252897467076, "nauc_ndcg_at_5_diff1": 0.3557709083054129, "nauc_ndcg_at_5_max": 0.17921601582787194, "nauc_ndcg_at_5_std": -0.1014947108412644, "nauc_precision_at_1000_diff1": 0.006633055518163547, "nauc_precision_at_1000_max": 0.08759636628632125, "nauc_precision_at_1000_std": -0.014889287290916334, "nauc_precision_at_100_diff1": 0.0617682685125337, "nauc_precision_at_100_max": 0.11877558098393813, "nauc_precision_at_100_std": -0.016417726786710003, "nauc_precision_at_10_diff1": 0.13223329223196534, "nauc_precision_at_10_max": 0.05442432624317679, "nauc_precision_at_10_std": -0.17842192575142593, "nauc_precision_at_1_diff1": 0.4446874864765066, "nauc_precision_at_1_max": 0.23916999910934264, "nauc_precision_at_1_std": -0.08078870533756279, "nauc_precision_at_20_diff1": 0.09794018553176256, "nauc_precision_at_20_max": 0.031054840058235694, "nauc_precision_at_20_std": -0.16778270792399078, "nauc_precision_at_3_diff1": 0.2205084203050175, "nauc_precision_at_3_max": 0.1352735356527621, "nauc_precision_at_3_std": -0.1604897997884064, "nauc_precision_at_5_diff1": 0.15387150754118664, "nauc_precision_at_5_max": 0.1003284578634536, "nauc_precision_at_5_std": -0.16309786348639163, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.23543072434985665, "nauc_recall_at_100_max": 0.1641127988377667, "nauc_recall_at_100_std": 0.0010560715079287173, "nauc_recall_at_10_diff1": 0.2634681061506411, "nauc_recall_at_10_max": 0.09287986292947803, "nauc_recall_at_10_std": -0.11951474652894345, "nauc_recall_at_1_diff1": 0.5070734238811194, "nauc_recall_at_1_max": 0.24529236270282476, "nauc_recall_at_1_std": -0.004688212359841111, "nauc_recall_at_20_diff1": 0.2156825749351031, "nauc_recall_at_20_max": 0.06392647995897967, "nauc_recall_at_20_std": -0.12668293400209021, "nauc_recall_at_3_diff1": 0.35395455611773263, "nauc_recall_at_3_max": 0.1744699445576355, "nauc_recall_at_3_std": -0.08642018776428495, "nauc_recall_at_5_diff1": 0.2927550794400118, "nauc_recall_at_5_max": 0.1343446345451416, "nauc_recall_at_5_std": -0.11348758147177891, "ndcg_at_1": 0.21824, "ndcg_at_10": 0.29263, "ndcg_at_100": 0.3642, "ndcg_at_1000": 0.40446, "ndcg_at_20": 0.31466, "ndcg_at_3": 0.23881, "ndcg_at_5": 0.26406, "precision_at_1": 0.21824, "precision_at_10": 0.06759, "precision_at_100": 0.01236, "precision_at_1000": 0.00167, "precision_at_20": 0.04072, "precision_at_3": 0.14224, "precision_at_5": 0.10749, "recall_at_1": 0.14722, "recall_at_10": 0.39091, "recall_at_100": 0.70168, "recall_at_1000": 1.0, "recall_at_20": 0.466, "recall_at_3": 0.24872, "recall_at_5": 0.31177 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/model_meta.json ================================================ {"name": "intfloat/multilingual-e5-base", "revision": "d13f1b27baf31030b7fd040960d60d909913633f", "release_date": "2024-02-08", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": 278000000, "memory_usage": null, "max_tokens": 514, "embed_dim": 768, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/intfloat/multilingual-e5-base", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "sentence_transformers_loader"} ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 18.56531572341919, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.81337, "map_at_1": 0.67544, "map_at_10": 0.77253, "map_at_100": 0.77541, "map_at_1000": 0.77541, "map_at_20": 0.77446, "map_at_3": 0.75585, "map_at_5": 0.76637, "mrr_at_1": 0.6754385964912281, "mrr_at_10": 0.7725250626566417, "mrr_at_100": 0.77540714333768, "mrr_at_1000": 0.77540714333768, "mrr_at_20": 0.7744555517581834, "mrr_at_3": 0.7558479532163744, "mrr_at_5": 0.766374269005848, "nauc_map_at_1000_diff1": 0.6882626899425819, "nauc_map_at_1000_max": 0.13028712605706078, "nauc_map_at_1000_std": -0.6333480833690052, "nauc_map_at_100_diff1": 0.6882626899425819, "nauc_map_at_100_max": 0.13028712605706078, "nauc_map_at_100_std": -0.6333480833690052, "nauc_map_at_10_diff1": 0.6837187235611711, "nauc_map_at_10_max": 0.13476353281143322, "nauc_map_at_10_std": -0.63489540113429, "nauc_map_at_1_diff1": 0.7283217227535123, "nauc_map_at_1_max": 0.08574937709970755, "nauc_map_at_1_std": -0.6246105689080378, "nauc_map_at_20_diff1": 0.6838263576041446, "nauc_map_at_20_max": 0.1313812162496074, "nauc_map_at_20_std": -0.6315026415983667, "nauc_map_at_3_diff1": 0.6952310170994932, "nauc_map_at_3_max": 0.11334045087831807, "nauc_map_at_3_std": -0.6298824326505806, "nauc_map_at_5_diff1": 0.6844736565258235, "nauc_map_at_5_max": 0.13942038403835927, "nauc_map_at_5_std": -0.6274947494869828, "nauc_mrr_at_1000_diff1": 0.6882626899425819, "nauc_mrr_at_1000_max": 0.13028712605706078, "nauc_mrr_at_1000_std": -0.6333480833690052, "nauc_mrr_at_100_diff1": 0.6882626899425819, "nauc_mrr_at_100_max": 0.13028712605706078, "nauc_mrr_at_100_std": -0.6333480833690052, "nauc_mrr_at_10_diff1": 0.6837187235611711, "nauc_mrr_at_10_max": 0.13476353281143322, "nauc_mrr_at_10_std": -0.63489540113429, "nauc_mrr_at_1_diff1": 0.7283217227535123, "nauc_mrr_at_1_max": 0.08574937709970755, "nauc_mrr_at_1_std": -0.6246105689080378, "nauc_mrr_at_20_diff1": 0.6838263576041446, "nauc_mrr_at_20_max": 0.1313812162496074, "nauc_mrr_at_20_std": -0.6315026415983667, "nauc_mrr_at_3_diff1": 0.6952310170994932, "nauc_mrr_at_3_max": 0.11334045087831807, "nauc_mrr_at_3_std": -0.6298824326505806, "nauc_mrr_at_5_diff1": 0.6844736565258235, "nauc_mrr_at_5_max": 0.13942038403835927, "nauc_mrr_at_5_std": -0.6274947494869828, "nauc_ndcg_at_1000_diff1": 0.6798177496497114, "nauc_ndcg_at_1000_max": 0.13930225357339482, "nauc_ndcg_at_1000_std": -0.6349323465994774, "nauc_ndcg_at_100_diff1": 0.6798177496497114, "nauc_ndcg_at_100_max": 0.13930225357339482, "nauc_ndcg_at_100_std": -0.6349323465994774, "nauc_ndcg_at_10_diff1": 0.6534305228357417, "nauc_ndcg_at_10_max": 0.16337846477607224, "nauc_ndcg_at_10_std": -0.6439472422010555, "nauc_ndcg_at_1_diff1": 0.7283217227535123, "nauc_ndcg_at_1_max": 0.08574937709970755, "nauc_ndcg_at_1_std": -0.6246105689080378, "nauc_ndcg_at_20_diff1": 0.6532504040101259, "nauc_ndcg_at_20_max": 0.15154965569273673, "nauc_ndcg_at_20_std": -0.62529669477198, "nauc_ndcg_at_3_diff1": 0.6802176246485789, "nauc_ndcg_at_3_max": 0.11876515078143252, "nauc_ndcg_at_3_std": -0.6295132308461416, "nauc_ndcg_at_5_diff1": 0.65782040297904, "nauc_ndcg_at_5_max": 0.1710880020518264, "nauc_ndcg_at_5_std": -0.623544324755987, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.3576107205133529, "nauc_precision_at_10_max": 0.4417872910038361, "nauc_precision_at_10_std": -0.7449071731142188, "nauc_precision_at_1_diff1": 0.7283217227535123, "nauc_precision_at_1_max": 0.08574937709970755, "nauc_precision_at_1_std": -0.6246105689080378, "nauc_precision_at_20_diff1": 0.13838797981732007, "nauc_precision_at_20_max": 0.4593222978687597, "nauc_precision_at_20_std": -0.4558779704077904, "nauc_precision_at_3_diff1": 0.6150080982680898, "nauc_precision_at_3_max": 0.13934586595258922, "nauc_precision_at_3_std": -0.6267492848843961, "nauc_precision_at_5_diff1": 0.5019388400138272, "nauc_precision_at_5_max": 0.3582921076932564, "nauc_precision_at_5_std": -0.5967482390704195, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.3576107205133534, "nauc_recall_at_10_max": 0.44178729100383596, "nauc_recall_at_10_std": -0.7449071731142182, "nauc_recall_at_1_diff1": 0.7283217227535123, "nauc_recall_at_1_max": 0.08574937709970755, "nauc_recall_at_1_std": -0.6246105689080378, "nauc_recall_at_20_diff1": 0.13838797981732298, "nauc_recall_at_20_max": 0.4593222978687574, "nauc_recall_at_20_std": -0.45587797040779265, "nauc_recall_at_3_diff1": 0.6150080982680903, "nauc_recall_at_3_max": 0.13934586595258888, "nauc_recall_at_3_std": -0.6267492848843966, "nauc_recall_at_5_diff1": 0.5019388400138263, "nauc_recall_at_5_max": 0.35829210769325465, "nauc_recall_at_5_std": -0.5967482390704182, "ndcg_at_1": 0.67544, "ndcg_at_10": 0.81337, "ndcg_at_100": 0.82677, "ndcg_at_1000": 0.82677, "ndcg_at_20": 0.82015, "ndcg_at_3": 0.78039, "ndcg_at_5": 0.79889, "precision_at_1": 0.67544, "precision_at_10": 0.09386, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04825, "precision_at_3": 0.28363, "precision_at_5": 0.17895, "recall_at_1": 0.67544, "recall_at_10": 0.9386, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.96491, "recall_at_3": 0.85088, "recall_at_5": 0.89474 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 45.867305755615234, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.94499, "map_at_1": 0.90556, "map_at_10": 0.93387, "map_at_100": 0.93468, "map_at_1000": 0.93472, "map_at_20": 0.93447, "map_at_3": 0.93093, "map_at_5": 0.93248, "mrr_at_1": 0.9055555555555556, "mrr_at_10": 0.9338730158730159, "mrr_at_100": 0.9346750637504948, "mrr_at_1000": 0.9347165982343034, "mrr_at_20": 0.9344692159692158, "mrr_at_3": 0.9309259259259258, "mrr_at_5": 0.9324814814814814, "nauc_map_at_1000_diff1": 0.930227758709326, "nauc_map_at_1000_max": 0.8265114039753071, "nauc_map_at_1000_std": 0.20890868954346548, "nauc_map_at_100_diff1": 0.9302376031901283, "nauc_map_at_100_max": 0.8265563740476617, "nauc_map_at_100_std": 0.20923759828213495, "nauc_map_at_10_diff1": 0.9304875708217899, "nauc_map_at_10_max": 0.8282339553413531, "nauc_map_at_10_std": 0.2077632421187409, "nauc_map_at_1_diff1": 0.9256329982973587, "nauc_map_at_1_max": 0.7965178228154025, "nauc_map_at_1_std": 0.18032075575328102, "nauc_map_at_20_diff1": 0.9304567434737736, "nauc_map_at_20_max": 0.8268189860529359, "nauc_map_at_20_std": 0.20806536983957427, "nauc_map_at_3_diff1": 0.931054638119769, "nauc_map_at_3_max": 0.83250476240541, "nauc_map_at_3_std": 0.21031933774403588, "nauc_map_at_5_diff1": 0.9310392469293455, "nauc_map_at_5_max": 0.8305281154334097, "nauc_map_at_5_std": 0.21305404590067914, "nauc_mrr_at_1000_diff1": 0.930227758709326, "nauc_mrr_at_1000_max": 0.8265114039753071, "nauc_mrr_at_1000_std": 0.20890868954346548, "nauc_mrr_at_100_diff1": 0.9302376031901283, "nauc_mrr_at_100_max": 0.8265563740476617, "nauc_mrr_at_100_std": 0.20923759828213495, "nauc_mrr_at_10_diff1": 0.9304875708217899, "nauc_mrr_at_10_max": 0.8282339553413531, "nauc_mrr_at_10_std": 0.2077632421187409, "nauc_mrr_at_1_diff1": 0.9256329982973587, "nauc_mrr_at_1_max": 0.7965178228154025, "nauc_mrr_at_1_std": 0.18032075575328102, "nauc_mrr_at_20_diff1": 0.9304567434737736, "nauc_mrr_at_20_max": 0.8268189860529359, "nauc_mrr_at_20_std": 0.20806536983957427, "nauc_mrr_at_3_diff1": 0.931054638119769, "nauc_mrr_at_3_max": 0.83250476240541, "nauc_mrr_at_3_std": 0.21031933774403588, "nauc_mrr_at_5_diff1": 0.9310392469293455, "nauc_mrr_at_5_max": 0.8305281154334097, "nauc_mrr_at_5_std": 0.21305404590067914, "nauc_ndcg_at_1000_diff1": 0.9311047013723796, "nauc_ndcg_at_1000_max": 0.8308367681918263, "nauc_ndcg_at_1000_std": 0.21661583693444478, "nauc_ndcg_at_100_diff1": 0.9311689837709294, "nauc_ndcg_at_100_max": 0.8318347592999481, "nauc_ndcg_at_100_std": 0.2253928671613762, "nauc_ndcg_at_10_diff1": 0.9327072401603272, "nauc_ndcg_at_10_max": 0.8392724958204727, "nauc_ndcg_at_10_std": 0.2174409681677675, "nauc_ndcg_at_1_diff1": 0.9256329982973587, "nauc_ndcg_at_1_max": 0.7965178228154025, "nauc_ndcg_at_1_std": 0.18032075575328102, "nauc_ndcg_at_20_diff1": 0.9328766716345652, "nauc_ndcg_at_20_max": 0.8337744839486585, "nauc_ndcg_at_20_std": 0.2183668124786509, "nauc_ndcg_at_3_diff1": 0.93419337625773, "nauc_ndcg_at_3_max": 0.8486196504258102, "nauc_ndcg_at_3_std": 0.2244037110917959, "nauc_ndcg_at_5_diff1": 0.9343058150714209, "nauc_ndcg_at_5_max": 0.845513221904975, "nauc_ndcg_at_5_std": 0.23060011579652634, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9319172113289566, "nauc_precision_at_100_max": 0.8856209150326616, "nauc_precision_at_100_std": 0.7176314970432794, "nauc_precision_at_10_diff1": 0.9509803921568587, "nauc_precision_at_10_max": 0.9286205710354282, "nauc_precision_at_10_std": 0.29566563467491047, "nauc_precision_at_1_diff1": 0.9256329982973587, "nauc_precision_at_1_max": 0.7965178228154025, "nauc_precision_at_1_std": 0.18032075575328102, "nauc_precision_at_20_diff1": 0.965958605664485, "nauc_precision_at_20_max": 0.8978758169934673, "nauc_precision_at_20_std": 0.3496342981637061, "nauc_precision_at_3_diff1": 0.9505135387488309, "nauc_precision_at_3_max": 0.9304388422035507, "nauc_precision_at_3_std": 0.2962651727357632, "nauc_precision_at_5_diff1": 0.9543610547667292, "nauc_precision_at_5_max": 0.9346405228758145, "nauc_precision_at_5_std": 0.33776039151291765, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9319172113289615, "nauc_recall_at_100_max": 0.8856209150326758, "nauc_recall_at_100_std": 0.7176314970432669, "nauc_recall_at_10_diff1": 0.9509803921568633, "nauc_recall_at_10_max": 0.9286205710354343, "nauc_recall_at_10_std": 0.29566563467492946, "nauc_recall_at_1_diff1": 0.9256329982973587, "nauc_recall_at_1_max": 0.7965178228154025, "nauc_recall_at_1_std": 0.18032075575328102, "nauc_recall_at_20_diff1": 0.9659586056644808, "nauc_recall_at_20_max": 0.8978758169934619, "nauc_recall_at_20_std": 0.349634298163714, "nauc_recall_at_3_diff1": 0.9505135387488312, "nauc_recall_at_3_max": 0.9304388422035459, "nauc_recall_at_3_std": 0.29626517273576003, "nauc_recall_at_5_diff1": 0.9543610547667313, "nauc_recall_at_5_max": 0.9346405228758145, "nauc_recall_at_5_std": 0.3377603915129274, "ndcg_at_1": 0.90556, "ndcg_at_10": 0.94499, "ndcg_at_100": 0.94834, "ndcg_at_1000": 0.94925, "ndcg_at_20": 0.94703, "ndcg_at_3": 0.93872, "ndcg_at_5": 0.94149, "precision_at_1": 0.90556, "precision_at_10": 0.09789, "precision_at_100": 0.00993, "precision_at_1000": 0.001, "precision_at_20": 0.04933, "precision_at_3": 0.32037, "precision_at_5": 0.19356, "recall_at_1": 0.90556, "recall_at_10": 0.97889, "recall_at_100": 0.99333, "recall_at_1000": 1.0, "recall_at_20": 0.98667, "recall_at_3": 0.96111, "recall_at_5": 0.96778 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.92031, "map_at_1": 0.85667, "map_at_10": 0.90067, "map_at_100": 0.90155, "map_at_1000": 0.90157, "map_at_20": 0.90143, "map_at_3": 0.89296, "map_at_5": 0.89763, "mrr_at_1": 0.8566666666666667, "mrr_at_10": 0.9006693121693128, "mrr_at_100": 0.9015503056196907, "mrr_at_1000": 0.9015654504046813, "mrr_at_20": 0.9014305278055285, "mrr_at_3": 0.8929629629629634, "mrr_at_5": 0.8976296296296301, "nauc_map_at_1000_diff1": 0.92182025415712, "nauc_map_at_1000_max": 0.6206471258944869, "nauc_map_at_1000_std": 0.06596374693316626, "nauc_map_at_100_diff1": 0.9218178893846458, "nauc_map_at_100_max": 0.6206524888282047, "nauc_map_at_100_std": 0.06610573144199076, "nauc_map_at_10_diff1": 0.9218303582707491, "nauc_map_at_10_max": 0.6203003887986447, "nauc_map_at_10_std": 0.0620874618663675, "nauc_map_at_1_diff1": 0.9290876344207575, "nauc_map_at_1_max": 0.6200235118576627, "nauc_map_at_1_std": 0.07183550742658158, "nauc_map_at_20_diff1": 0.9217873327372612, "nauc_map_at_20_max": 0.6202010281677299, "nauc_map_at_20_std": 0.06578026510307823, "nauc_map_at_3_diff1": 0.9181874296864884, "nauc_map_at_3_max": 0.6291171775700318, "nauc_map_at_3_std": 0.05984820578172109, "nauc_map_at_5_diff1": 0.9211955728710669, "nauc_map_at_5_max": 0.6236460096499931, "nauc_map_at_5_std": 0.0627327727332104, "nauc_mrr_at_1000_diff1": 0.92182025415712, "nauc_mrr_at_1000_max": 0.6206471258944869, "nauc_mrr_at_1000_std": 0.06596374693316626, "nauc_mrr_at_100_diff1": 0.9218178893846458, "nauc_mrr_at_100_max": 0.6206524888282047, "nauc_mrr_at_100_std": 0.06610573144199076, "nauc_mrr_at_10_diff1": 0.9218303582707491, "nauc_mrr_at_10_max": 0.6203003887986447, "nauc_mrr_at_10_std": 0.0620874618663675, "nauc_mrr_at_1_diff1": 0.9290876344207575, "nauc_mrr_at_1_max": 0.6200235118576627, "nauc_mrr_at_1_std": 0.07183550742658158, "nauc_mrr_at_20_diff1": 0.9217873327372612, "nauc_mrr_at_20_max": 0.6202010281677299, "nauc_mrr_at_20_std": 0.06578026510307823, "nauc_mrr_at_3_diff1": 0.9181874296864884, "nauc_mrr_at_3_max": 0.6291171775700318, "nauc_mrr_at_3_std": 0.05984820578172109, "nauc_mrr_at_5_diff1": 0.9211955728710669, "nauc_mrr_at_5_max": 0.6236460096499931, "nauc_mrr_at_5_std": 0.0627327727332104, "nauc_ndcg_at_1000_diff1": 0.9214103710428968, "nauc_ndcg_at_1000_max": 0.6185534050632435, "nauc_ndcg_at_1000_std": 0.06803705027165456, "nauc_ndcg_at_100_diff1": 0.9214411175848705, "nauc_ndcg_at_100_max": 0.6181840377291484, "nauc_ndcg_at_100_std": 0.07163635632943753, "nauc_ndcg_at_10_diff1": 0.9213837866465251, "nauc_ndcg_at_10_max": 0.6140930950462312, "nauc_ndcg_at_10_std": 0.05677801630688054, "nauc_ndcg_at_1_diff1": 0.9290876344207575, "nauc_ndcg_at_1_max": 0.6200235118576627, "nauc_ndcg_at_1_std": 0.07183550742658158, "nauc_ndcg_at_20_diff1": 0.9212579184919344, "nauc_ndcg_at_20_max": 0.6131029065007272, "nauc_ndcg_at_20_std": 0.07210053998325393, "nauc_ndcg_at_3_diff1": 0.914187763311422, "nauc_ndcg_at_3_max": 0.6349271604621773, "nauc_ndcg_at_3_std": 0.053555195873877144, "nauc_ndcg_at_5_diff1": 0.9199620720983661, "nauc_ndcg_at_5_max": 0.6233361968663719, "nauc_ndcg_at_5_std": 0.05898324798893133, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9346405228757582, "nauc_precision_at_100_max": 0.49603174603179234, "nauc_precision_at_100_std": 0.9346405228757582, "nauc_precision_at_10_diff1": 0.9211841599384868, "nauc_precision_at_10_max": 0.5008787828856862, "nauc_precision_at_10_std": -0.021502718734557884, "nauc_precision_at_1_diff1": 0.9290876344207575, "nauc_precision_at_1_max": 0.6200235118576627, "nauc_precision_at_1_std": 0.07183550742658158, "nauc_precision_at_20_diff1": 0.9183006535947696, "nauc_precision_at_20_max": 0.3355508870214679, "nauc_precision_at_20_std": 0.37219887955182474, "nauc_precision_at_3_diff1": 0.895374816593309, "nauc_precision_at_3_max": 0.6637404961984841, "nauc_precision_at_3_std": 0.022817460317461912, "nauc_precision_at_5_diff1": 0.9137582961112355, "nauc_precision_at_5_max": 0.6175965881848202, "nauc_precision_at_5_std": 0.034017210487790155, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.934640522875857, "nauc_recall_at_100_max": 0.4960317460317535, "nauc_recall_at_100_std": 0.934640522875857, "nauc_recall_at_10_diff1": 0.9211841599384808, "nauc_recall_at_10_max": 0.5008787828857026, "nauc_recall_at_10_std": -0.021502718734554168, "nauc_recall_at_1_diff1": 0.9290876344207575, "nauc_recall_at_1_max": 0.6200235118576627, "nauc_recall_at_1_std": 0.07183550742658158, "nauc_recall_at_20_diff1": 0.9183006535947648, "nauc_recall_at_20_max": 0.33555088702149116, "nauc_recall_at_20_std": 0.37219887955183756, "nauc_recall_at_3_diff1": 0.8953748165933032, "nauc_recall_at_3_max": 0.6637404961984781, "nauc_recall_at_3_std": 0.022817460317458665, "nauc_recall_at_5_diff1": 0.9137582961112365, "nauc_recall_at_5_max": 0.6175965881848232, "nauc_recall_at_5_std": 0.03401721048779963, "ndcg_at_1": 0.85667, "ndcg_at_10": 0.92031, "ndcg_at_100": 0.92405, "ndcg_at_1000": 0.92436, "ndcg_at_20": 0.92293, "ndcg_at_3": 0.9045, "ndcg_at_5": 0.91305, "precision_at_1": 0.85667, "precision_at_10": 0.09811, "precision_at_100": 0.00998, "precision_at_1000": 0.001, "precision_at_20": 0.04956, "precision_at_3": 0.31259, "precision_at_5": 0.19178, "recall_at_1": 0.85667, "recall_at_10": 0.98111, "recall_at_100": 0.99778, "recall_at_1000": 1.0, "recall_at_20": 0.99111, "recall_at_3": 0.93778, "recall_at_5": 0.95889 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.86452, "map_at_1": 0.78556, "map_at_10": 0.83917, "map_at_100": 0.8415, "map_at_1000": 0.84155, "map_at_20": 0.8408, "map_at_3": 0.82722, "map_at_5": 0.83533, "mrr_at_1": 0.7855555555555556, "mrr_at_10": 0.8391706349206354, "mrr_at_100": 0.8415019369013431, "mrr_at_1000": 0.8415523736896204, "mrr_at_20": 0.8407976750011067, "mrr_at_3": 0.8272222222222225, "mrr_at_5": 0.8353333333333338, "nauc_map_at_1000_diff1": 0.8927705645097672, "nauc_map_at_1000_max": 0.7522985353992927, "nauc_map_at_1000_std": 0.18996221933631463, "nauc_map_at_100_diff1": 0.8927729415362704, "nauc_map_at_100_max": 0.7523016927357646, "nauc_map_at_100_std": 0.19022784908812126, "nauc_map_at_10_diff1": 0.892782216866023, "nauc_map_at_10_max": 0.7538520746576524, "nauc_map_at_10_std": 0.19104920288263752, "nauc_map_at_1_diff1": 0.9056590384024021, "nauc_map_at_1_max": 0.7385293491488186, "nauc_map_at_1_std": 0.14492300333008346, "nauc_map_at_20_diff1": 0.8926494720845148, "nauc_map_at_20_max": 0.7525637807150531, "nauc_map_at_20_std": 0.1920752367370665, "nauc_map_at_3_diff1": 0.8872650055085894, "nauc_map_at_3_max": 0.7487353164254025, "nauc_map_at_3_std": 0.16922961302361597, "nauc_map_at_5_diff1": 0.8918257731962219, "nauc_map_at_5_max": 0.7549879563198285, "nauc_map_at_5_std": 0.1938393366249109, "nauc_mrr_at_1000_diff1": 0.8927705645097672, "nauc_mrr_at_1000_max": 0.7522985353992927, "nauc_mrr_at_1000_std": 0.18996221933631463, "nauc_mrr_at_100_diff1": 0.8927729415362704, "nauc_mrr_at_100_max": 0.7523016927357646, "nauc_mrr_at_100_std": 0.19022784908812126, "nauc_mrr_at_10_diff1": 0.892782216866023, "nauc_mrr_at_10_max": 0.7538520746576524, "nauc_mrr_at_10_std": 0.19104920288263752, "nauc_mrr_at_1_diff1": 0.9056590384024021, "nauc_mrr_at_1_max": 0.7385293491488186, "nauc_mrr_at_1_std": 0.14492300333008346, "nauc_mrr_at_20_diff1": 0.8926494720845148, "nauc_mrr_at_20_max": 0.7525637807150531, "nauc_mrr_at_20_std": 0.1920752367370665, "nauc_mrr_at_3_diff1": 0.8872650055085894, "nauc_mrr_at_3_max": 0.7487353164254025, "nauc_mrr_at_3_std": 0.16922961302361597, "nauc_mrr_at_5_diff1": 0.8918257731962219, "nauc_mrr_at_5_max": 0.7549879563198285, "nauc_mrr_at_5_std": 0.1938393366249109, "nauc_ndcg_at_1000_diff1": 0.8913558562559795, "nauc_ndcg_at_1000_max": 0.7544397216354228, "nauc_ndcg_at_1000_std": 0.20394034939099934, "nauc_ndcg_at_100_diff1": 0.8913784112846709, "nauc_ndcg_at_100_max": 0.7545109846083868, "nauc_ndcg_at_100_std": 0.2122690068264438, "nauc_ndcg_at_10_diff1": 0.8910240767840198, "nauc_ndcg_at_10_max": 0.7616020409402215, "nauc_ndcg_at_10_std": 0.2209162854297489, "nauc_ndcg_at_1_diff1": 0.9056590384024021, "nauc_ndcg_at_1_max": 0.7385293491488186, "nauc_ndcg_at_1_std": 0.14492300333008346, "nauc_ndcg_at_20_diff1": 0.8901705655943357, "nauc_ndcg_at_20_max": 0.7563923945801978, "nauc_ndcg_at_20_std": 0.2257500574997457, "nauc_ndcg_at_3_diff1": 0.8796131863096975, "nauc_ndcg_at_3_max": 0.7508780483627049, "nauc_ndcg_at_3_std": 0.17629577865227775, "nauc_ndcg_at_5_diff1": 0.8887001457995483, "nauc_ndcg_at_5_max": 0.7639847302367411, "nauc_ndcg_at_5_std": 0.22769495548405205, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8905488121174426, "nauc_precision_at_100_max": 0.7622159975101201, "nauc_precision_at_100_std": 0.956427015250545, "nauc_precision_at_10_diff1": 0.8838816572380539, "nauc_precision_at_10_max": 0.8229252485307802, "nauc_precision_at_10_std": 0.4592555976638981, "nauc_precision_at_1_diff1": 0.9056590384024021, "nauc_precision_at_1_max": 0.7385293491488186, "nauc_precision_at_1_std": 0.14492300333008346, "nauc_precision_at_20_diff1": 0.8660431914701331, "nauc_precision_at_20_max": 0.7901569229842433, "nauc_precision_at_20_std": 0.6722237282009511, "nauc_precision_at_3_diff1": 0.8490958744299489, "nauc_precision_at_3_max": 0.7585772616396241, "nauc_precision_at_3_std": 0.20325149361897543, "nauc_precision_at_5_diff1": 0.8751474273920078, "nauc_precision_at_5_max": 0.8139773453240927, "nauc_precision_at_5_std": 0.41905621897881695, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8905488121174361, "nauc_recall_at_100_max": 0.7622159975101154, "nauc_recall_at_100_std": 0.9564270152505469, "nauc_recall_at_10_diff1": 0.883881657238058, "nauc_recall_at_10_max": 0.8229252485307852, "nauc_recall_at_10_std": 0.459255597663904, "nauc_recall_at_1_diff1": 0.9056590384024021, "nauc_recall_at_1_max": 0.7385293491488186, "nauc_recall_at_1_std": 0.14492300333008346, "nauc_recall_at_20_diff1": 0.8660431914701362, "nauc_recall_at_20_max": 0.7901569229842438, "nauc_recall_at_20_std": 0.6722237282009564, "nauc_recall_at_3_diff1": 0.8490958744299502, "nauc_recall_at_3_max": 0.7585772616396222, "nauc_recall_at_3_std": 0.20325149361897632, "nauc_recall_at_5_diff1": 0.8751474273920088, "nauc_recall_at_5_max": 0.8139773453240944, "nauc_recall_at_5_std": 0.4190562189788191, "ndcg_at_1": 0.78556, "ndcg_at_10": 0.86452, "ndcg_at_100": 0.87494, "ndcg_at_1000": 0.87623, "ndcg_at_20": 0.87024, "ndcg_at_3": 0.84078, "ndcg_at_5": 0.85541, "precision_at_1": 0.78556, "precision_at_10": 0.09433, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "precision_at_20": 0.04828, "precision_at_3": 0.29333, "precision_at_5": 0.18311, "recall_at_1": 0.78556, "recall_at_10": 0.94333, "recall_at_100": 0.99, "recall_at_1000": 1.0, "recall_at_20": 0.96556, "recall_at_3": 0.88, "recall_at_5": 0.91556 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 103.04216527938843, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.80348, "map_at_1": 0.50664, "map_at_10": 0.76374, "map_at_100": 0.76914, "map_at_1000": 0.76945, "map_at_20": 0.76721, "map_at_3": 0.7275, "map_at_5": 0.75164, "mrr_at_1": 0.768581081081081, "mrr_at_10": 0.8201114060489061, "mrr_at_100": 0.8218358869087574, "mrr_at_1000": 0.8220410843903181, "mrr_at_20": 0.821053386412326, "mrr_at_3": 0.8153153153153152, "mrr_at_5": 0.8185247747747747, "nauc_map_at_1000_diff1": 0.6270232124471452, "nauc_map_at_1000_max": 0.6423349743441891, "nauc_map_at_1000_std": 0.279921772861948, "nauc_map_at_100_diff1": 0.6270244327686766, "nauc_map_at_100_max": 0.6426234475759284, "nauc_map_at_100_std": 0.28050253193435104, "nauc_map_at_10_diff1": 0.6267964661890997, "nauc_map_at_10_max": 0.6451676187166451, "nauc_map_at_10_std": 0.2716152727684029, "nauc_map_at_1_diff1": 0.6764732370312437, "nauc_map_at_1_max": 0.2905536892814776, "nauc_map_at_1_std": -0.011034293163808993, "nauc_map_at_20_diff1": 0.6278280584599769, "nauc_map_at_20_max": 0.6448366164382613, "nauc_map_at_20_std": 0.2819218324018119, "nauc_map_at_3_diff1": 0.6061847436353337, "nauc_map_at_3_max": 0.5780731469434045, "nauc_map_at_3_std": 0.2027741002354236, "nauc_map_at_5_diff1": 0.624348386827946, "nauc_map_at_5_max": 0.6208855054942963, "nauc_map_at_5_std": 0.24142157875599662, "nauc_mrr_at_1000_diff1": 0.717726508368629, "nauc_mrr_at_1000_max": 0.7127501760528012, "nauc_mrr_at_1000_std": 0.38744516297651393, "nauc_mrr_at_100_diff1": 0.7176454571697286, "nauc_mrr_at_100_max": 0.7129788361602002, "nauc_mrr_at_100_std": 0.38805260787614737, "nauc_mrr_at_10_diff1": 0.7180161399152217, "nauc_mrr_at_10_max": 0.7150939227839377, "nauc_mrr_at_10_std": 0.38803096948772753, "nauc_mrr_at_1_diff1": 0.7389675684828687, "nauc_mrr_at_1_max": 0.619783943559237, "nauc_mrr_at_1_std": 0.24520417021488036, "nauc_mrr_at_20_diff1": 0.7178663921394264, "nauc_mrr_at_20_max": 0.7139552353200426, "nauc_mrr_at_20_std": 0.38946641086329103, "nauc_mrr_at_3_diff1": 0.7173884183805199, "nauc_mrr_at_3_max": 0.7146021491140149, "nauc_mrr_at_3_std": 0.37798793931652225, "nauc_mrr_at_5_diff1": 0.7175885111599681, "nauc_mrr_at_5_max": 0.7148505549807599, "nauc_mrr_at_5_std": 0.38756856589650296, "nauc_ndcg_at_1000_diff1": 0.6472404381887801, "nauc_ndcg_at_1000_max": 0.6964562175019898, "nauc_ndcg_at_1000_std": 0.37195544642850276, "nauc_ndcg_at_100_diff1": 0.6470735358009762, "nauc_ndcg_at_100_max": 0.7043670261868529, "nauc_ndcg_at_100_std": 0.3897900357789412, "nauc_ndcg_at_10_diff1": 0.6465969495009757, "nauc_ndcg_at_10_max": 0.7146099910954262, "nauc_ndcg_at_10_std": 0.3679914038774038, "nauc_ndcg_at_1_diff1": 0.7389675684828687, "nauc_ndcg_at_1_max": 0.619783943559237, "nauc_ndcg_at_1_std": 0.24520417021488036, "nauc_ndcg_at_20_diff1": 0.6493928925737483, "nauc_ndcg_at_20_max": 0.7134715845031814, "nauc_ndcg_at_20_std": 0.3939633225716529, "nauc_ndcg_at_3_diff1": 0.6178715093585547, "nauc_ndcg_at_3_max": 0.6319521302138631, "nauc_ndcg_at_3_std": 0.2680060266389483, "nauc_ndcg_at_5_diff1": 0.6405997079687031, "nauc_ndcg_at_5_max": 0.6764242485744868, "nauc_ndcg_at_5_std": 0.31694007367796995, "nauc_precision_at_1000_diff1": -0.3970271186614315, "nauc_precision_at_1000_max": 0.001086350645082997, "nauc_precision_at_1000_std": 0.1934852025821731, "nauc_precision_at_100_diff1": -0.28632625129297085, "nauc_precision_at_100_max": 0.13535950905854507, "nauc_precision_at_100_std": 0.3011672467378491, "nauc_precision_at_10_diff1": -0.16726338149009454, "nauc_precision_at_10_max": 0.2909053191625467, "nauc_precision_at_10_std": 0.3443975592990956, "nauc_precision_at_1_diff1": 0.7389675684828687, "nauc_precision_at_1_max": 0.619783943559237, "nauc_precision_at_1_std": 0.24520417021488036, "nauc_precision_at_20_diff1": -0.2060497574556575, "nauc_precision_at_20_max": 0.23376163020027696, "nauc_precision_at_20_std": 0.35240442591945115, "nauc_precision_at_3_diff1": -0.06277299917404543, "nauc_precision_at_3_max": 0.37286499375085813, "nauc_precision_at_3_std": 0.30817315779367294, "nauc_precision_at_5_diff1": -0.10813798215240919, "nauc_precision_at_5_max": 0.32036525771869867, "nauc_precision_at_5_std": 0.3122024855654004, "nauc_recall_at_1000_diff1": 0.6123484988578439, "nauc_recall_at_1000_max": 0.9389556221977473, "nauc_recall_at_1000_std": 0.9317786678431724, "nauc_recall_at_100_diff1": 0.6237038348199009, "nauc_recall_at_100_max": 0.8941415813611865, "nauc_recall_at_100_std": 0.8389314190049417, "nauc_recall_at_10_diff1": 0.6232544482810121, "nauc_recall_at_10_max": 0.8589984248699257, "nauc_recall_at_10_std": 0.5629988195620854, "nauc_recall_at_1_diff1": 0.6764732370312437, "nauc_recall_at_1_max": 0.2905536892814776, "nauc_recall_at_1_std": -0.011034293163808993, "nauc_recall_at_20_diff1": 0.6390604871621653, "nauc_recall_at_20_max": 0.8854683603368402, "nauc_recall_at_20_std": 0.7169948683694372, "nauc_recall_at_3_diff1": 0.5668709737247727, "nauc_recall_at_3_max": 0.6343809223314516, "nauc_recall_at_3_std": 0.2911839083650689, "nauc_recall_at_5_diff1": 0.6047177057671275, "nauc_recall_at_5_max": 0.7341251106852447, "nauc_recall_at_5_std": 0.40014013976984053, "ndcg_at_1": 0.76858, "ndcg_at_10": 0.80348, "ndcg_at_100": 0.82087, "ndcg_at_1000": 0.82835, "ndcg_at_20": 0.81154, "ndcg_at_3": 0.76483, "ndcg_at_5": 0.78681, "precision_at_1": 0.76858, "precision_at_10": 0.15794, "precision_at_100": 0.0173, "precision_at_1000": 0.00184, "precision_at_20": 0.08201, "precision_at_3": 0.45327, "precision_at_5": 0.29628, "recall_at_1": 0.50664, "recall_at_10": 0.854, "recall_at_100": 0.91833, "recall_at_1000": 0.96585, "recall_at_20": 0.87852, "recall_at_3": 0.7696, "recall_at_5": 0.8158 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 22242.406965255737, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.66486, "map_at_1": 0.39817, "map_at_10": 0.57475, "map_at_100": 0.59592, "map_at_1000": 0.59688, "map_at_20": 0.58615, "map_at_3": 0.51008, "map_at_5": 0.5453, "mrr_at_1": 0.6291079812206573, "mrr_at_10": 0.7366588419405322, "mrr_at_100": 0.7396868370283863, "mrr_at_1000": 0.7397718728043917, "mrr_at_20": 0.7387440806454891, "mrr_at_3": 0.7214397496087638, "mrr_at_5": 0.7320031298904539, "nauc_map_at_1000_diff1": 0.3837911965262718, "nauc_map_at_1000_max": 0.40308399089236396, "nauc_map_at_1000_std": -0.06379339452715582, "nauc_map_at_100_diff1": 0.38278502675420284, "nauc_map_at_100_max": 0.40238736979314654, "nauc_map_at_100_std": -0.06544180548856873, "nauc_map_at_10_diff1": 0.39089373824018875, "nauc_map_at_10_max": 0.4041436760808993, "nauc_map_at_10_std": -0.08672260660681444, "nauc_map_at_1_diff1": 0.440509659668766, "nauc_map_at_1_max": 0.3236573126938934, "nauc_map_at_1_std": -0.18323841041068328, "nauc_map_at_20_diff1": 0.381851663192607, "nauc_map_at_20_max": 0.39906518685264364, "nauc_map_at_20_std": -0.07929991459709138, "nauc_map_at_3_diff1": 0.381696475066722, "nauc_map_at_3_max": 0.3556494320690001, "nauc_map_at_3_std": -0.16600412280283966, "nauc_map_at_5_diff1": 0.3976892335685762, "nauc_map_at_5_max": 0.3685040211177935, "nauc_map_at_5_std": -0.14048277377163843, "nauc_mrr_at_1000_diff1": 0.3872386048717649, "nauc_mrr_at_1000_max": 0.45376262480676327, "nauc_mrr_at_1000_std": 0.1599964264678888, "nauc_mrr_at_100_diff1": 0.3872366940531705, "nauc_mrr_at_100_max": 0.45385927820433125, "nauc_mrr_at_100_std": 0.16019961717007042, "nauc_mrr_at_10_diff1": 0.3887496116894425, "nauc_mrr_at_10_max": 0.458533950697311, "nauc_mrr_at_10_std": 0.1654565986975807, "nauc_mrr_at_1_diff1": 0.4327839855313694, "nauc_mrr_at_1_max": 0.43792362259344914, "nauc_mrr_at_1_std": 0.08901866262287145, "nauc_mrr_at_20_diff1": 0.38712698651768995, "nauc_mrr_at_20_max": 0.4551012845906619, "nauc_mrr_at_20_std": 0.1608626758894431, "nauc_mrr_at_3_diff1": 0.3754946169140696, "nauc_mrr_at_3_max": 0.4402042196432023, "nauc_mrr_at_3_std": 0.16281847393521195, "nauc_mrr_at_5_diff1": 0.39178041592492124, "nauc_mrr_at_5_max": 0.45410519440021624, "nauc_mrr_at_5_std": 0.16606088593627427, "nauc_ndcg_at_1000_diff1": 0.3724402918583468, "nauc_ndcg_at_1000_max": 0.4411204737374578, "nauc_ndcg_at_1000_std": 0.05633736942813012, "nauc_ndcg_at_100_diff1": 0.36306103475892026, "nauc_ndcg_at_100_max": 0.4362276567346318, "nauc_ndcg_at_100_std": 0.04076036137752997, "nauc_ndcg_at_10_diff1": 0.38565581428440926, "nauc_ndcg_at_10_max": 0.4477238632518685, "nauc_ndcg_at_10_std": -0.0022715690211862934, "nauc_ndcg_at_1_diff1": 0.4327839855313694, "nauc_ndcg_at_1_max": 0.43792362259344914, "nauc_ndcg_at_1_std": 0.08901866262287145, "nauc_ndcg_at_20_diff1": 0.36334095594670834, "nauc_ndcg_at_20_max": 0.43121289828585696, "nauc_ndcg_at_20_std": 0.0014560776008887809, "nauc_ndcg_at_3_diff1": 0.3504828896059918, "nauc_ndcg_at_3_max": 0.3903649062281797, "nauc_ndcg_at_3_std": 0.0009487492719896218, "nauc_ndcg_at_5_diff1": 0.3955825272148319, "nauc_ndcg_at_5_max": 0.40013532836706406, "nauc_ndcg_at_5_std": -0.023625493066877553, "nauc_precision_at_1000_diff1": -0.1674893792440607, "nauc_precision_at_1000_max": 0.008713388862239432, "nauc_precision_at_1000_std": 0.47802346731030876, "nauc_precision_at_100_diff1": -0.18625122991805604, "nauc_precision_at_100_max": 0.01634892796986117, "nauc_precision_at_100_std": 0.46547302324700046, "nauc_precision_at_10_diff1": -0.09628418614572895, "nauc_precision_at_10_max": 0.16602575486623067, "nauc_precision_at_10_std": 0.4560172223232804, "nauc_precision_at_1_diff1": 0.4327839855313694, "nauc_precision_at_1_max": 0.43792362259344914, "nauc_precision_at_1_std": 0.08901866262287145, "nauc_precision_at_20_diff1": -0.1642198896134193, "nauc_precision_at_20_max": 0.08357456167411671, "nauc_precision_at_20_std": 0.4430076721462144, "nauc_precision_at_3_diff1": -0.051558683337445156, "nauc_precision_at_3_max": 0.23528208450060462, "nauc_precision_at_3_std": 0.3002556524367771, "nauc_precision_at_5_diff1": -0.04581765133291948, "nauc_precision_at_5_max": 0.15784094877764906, "nauc_precision_at_5_std": 0.3609616652230924, "nauc_recall_at_1000_diff1": 0.18069466870619919, "nauc_recall_at_1000_max": 0.7884234926199413, "nauc_recall_at_1000_std": 0.8396692861367822, "nauc_recall_at_100_diff1": 0.09988559454231889, "nauc_recall_at_100_max": 0.4024750413649136, "nauc_recall_at_100_std": 0.12536806908524142, "nauc_recall_at_10_diff1": 0.3336311951057095, "nauc_recall_at_10_max": 0.462288877527675, "nauc_recall_at_10_std": -0.01382064760940675, "nauc_recall_at_1_diff1": 0.440509659668766, "nauc_recall_at_1_max": 0.3236573126938934, "nauc_recall_at_1_std": -0.18323841041068328, "nauc_recall_at_20_diff1": 0.22916823976098807, "nauc_recall_at_20_max": 0.3843913327854645, "nauc_recall_at_20_std": -0.03841425001610077, "nauc_recall_at_3_diff1": 0.30981968404029536, "nauc_recall_at_3_max": 0.30174085683026775, "nauc_recall_at_3_std": -0.16719772497653437, "nauc_recall_at_5_diff1": 0.3646875535602896, "nauc_recall_at_5_max": 0.3294498583147711, "nauc_recall_at_5_std": -0.1518666939810128, "ndcg_at_1": 0.62911, "ndcg_at_10": 0.66486, "ndcg_at_100": 0.71956, "ndcg_at_1000": 0.72913, "ndcg_at_20": 0.68729, "ndcg_at_3": 0.623, "ndcg_at_5": 0.63547, "precision_at_1": 0.62911, "precision_at_10": 0.16761, "precision_at_100": 0.02324, "precision_at_1000": 0.00253, "precision_at_20": 0.09437, "precision_at_3": 0.37089, "precision_at_5": 0.27512, "recall_at_1": 0.39817, "recall_at_10": 0.75212, "recall_at_100": 0.93362, "recall_at_1000": 0.98208, "recall_at_20": 0.81836, "recall_at_3": 0.57548, "recall_at_5": 0.6547 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 23272.43692088127, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.64211, "map_at_1": 0.46397, "map_at_10": 0.58923, "map_at_100": 0.59711, "map_at_1000": 0.59734, "map_at_20": 0.59392, "map_at_3": 0.5644, "map_at_5": 0.58063, "mrr_at_1": 0.498812351543943, "mrr_at_10": 0.6117925951061345, "mrr_at_100": 0.6172461485002978, "mrr_at_1000": 0.6174050167685023, "mrr_at_20": 0.6147653071304376, "mrr_at_3": 0.5918448139350755, "mrr_at_5": 0.604790182106097, "nauc_map_at_1000_diff1": 0.47931223136256057, "nauc_map_at_1000_max": 0.4007334827100805, "nauc_map_at_1000_std": 0.043487872418719056, "nauc_map_at_100_diff1": 0.47925112843131884, "nauc_map_at_100_max": 0.40077407832527784, "nauc_map_at_100_std": 0.043456957506543874, "nauc_map_at_10_diff1": 0.4806526770216943, "nauc_map_at_10_max": 0.39632760172530024, "nauc_map_at_10_std": 0.03950815662330351, "nauc_map_at_1_diff1": 0.5321911659832964, "nauc_map_at_1_max": 0.3954179793853281, "nauc_map_at_1_std": 0.05094634517180547, "nauc_map_at_20_diff1": 0.4800495551765683, "nauc_map_at_20_max": 0.39902429014320684, "nauc_map_at_20_std": 0.041856276283318505, "nauc_map_at_3_diff1": 0.48229212227224766, "nauc_map_at_3_max": 0.3995713838422883, "nauc_map_at_3_std": 0.03690170493850673, "nauc_map_at_5_diff1": 0.4783003027560702, "nauc_map_at_5_max": 0.3993736834106997, "nauc_map_at_5_std": 0.039477183473525135, "nauc_mrr_at_1000_diff1": 0.495225038039527, "nauc_mrr_at_1000_max": 0.4220848374925641, "nauc_mrr_at_1000_std": 0.046087169293733715, "nauc_mrr_at_100_diff1": 0.49509055545526093, "nauc_mrr_at_100_max": 0.4220145500933536, "nauc_mrr_at_100_std": 0.045971616948838695, "nauc_mrr_at_10_diff1": 0.4963834416827471, "nauc_mrr_at_10_max": 0.4197489896599539, "nauc_mrr_at_10_std": 0.04347046617574166, "nauc_mrr_at_1_diff1": 0.5502772419562014, "nauc_mrr_at_1_max": 0.42432282839509594, "nauc_mrr_at_1_std": 0.056539997728625516, "nauc_mrr_at_20_diff1": 0.495834428313876, "nauc_mrr_at_20_max": 0.4219989014934441, "nauc_mrr_at_20_std": 0.045713852924685404, "nauc_mrr_at_3_diff1": 0.49311673694395813, "nauc_mrr_at_3_max": 0.42217278185884216, "nauc_mrr_at_3_std": 0.045486376174776987, "nauc_mrr_at_5_diff1": 0.49487684702992973, "nauc_mrr_at_5_max": 0.4228996637119372, "nauc_mrr_at_5_std": 0.042717981438324684, "nauc_ndcg_at_1000_diff1": 0.46718225176412104, "nauc_ndcg_at_1000_max": 0.41406232527666265, "nauc_ndcg_at_1000_std": 0.057770566345204215, "nauc_ndcg_at_100_diff1": 0.4637665823255492, "nauc_ndcg_at_100_max": 0.41521149956451847, "nauc_ndcg_at_100_std": 0.05925644569030309, "nauc_ndcg_at_10_diff1": 0.4705926342509073, "nauc_ndcg_at_10_max": 0.3960903500827478, "nauc_ndcg_at_10_std": 0.03738347747999284, "nauc_ndcg_at_1_diff1": 0.5502772419562014, "nauc_ndcg_at_1_max": 0.42432282839509594, "nauc_ndcg_at_1_std": 0.056539997728625516, "nauc_ndcg_at_20_diff1": 0.4693735386172784, "nauc_ndcg_at_20_max": 0.4051958577086621, "nauc_ndcg_at_20_std": 0.04665714880742706, "nauc_ndcg_at_3_diff1": 0.4691674613529208, "nauc_ndcg_at_3_max": 0.40442677313868036, "nauc_ndcg_at_3_std": 0.03658738600868339, "nauc_ndcg_at_5_diff1": 0.4652513608671465, "nauc_ndcg_at_5_max": 0.4032050160398976, "nauc_ndcg_at_5_std": 0.03646578334852213, "nauc_precision_at_1000_diff1": -0.1376901317798828, "nauc_precision_at_1000_max": 0.07074525569946662, "nauc_precision_at_1000_std": 0.06525116901031895, "nauc_precision_at_100_diff1": -0.07291729186781257, "nauc_precision_at_100_max": 0.1543057084796784, "nauc_precision_at_100_std": 0.09579018179279059, "nauc_precision_at_10_diff1": 0.17539177775898368, "nauc_precision_at_10_max": 0.2194703262505171, "nauc_precision_at_10_std": 0.00045100241308365505, "nauc_precision_at_1_diff1": 0.5502772419562014, "nauc_precision_at_1_max": 0.42432282839509594, "nauc_precision_at_1_std": 0.056539997728625516, "nauc_precision_at_20_diff1": 0.1001212988896256, "nauc_precision_at_20_max": 0.19500946854026321, "nauc_precision_at_20_std": 0.033877091949833134, "nauc_precision_at_3_diff1": 0.3108146336952218, "nauc_precision_at_3_max": 0.3399582145797798, "nauc_precision_at_3_std": 0.012185139119488025, "nauc_precision_at_5_diff1": 0.2176647197706826, "nauc_precision_at_5_max": 0.279589656873362, "nauc_precision_at_5_std": -8.576091514749797e-06, "nauc_recall_at_1000_diff1": 0.17922367256077656, "nauc_recall_at_1000_max": 0.6269115941767262, "nauc_recall_at_1000_std": 0.5692952967604661, "nauc_recall_at_100_diff1": 0.2351755934148718, "nauc_recall_at_100_max": 0.527848439581095, "nauc_recall_at_100_std": 0.32990668711326004, "nauc_recall_at_10_diff1": 0.40367269891236246, "nauc_recall_at_10_max": 0.3462384537056759, "nauc_recall_at_10_std": 0.025491997823164098, "nauc_recall_at_1_diff1": 0.5321911659832964, "nauc_recall_at_1_max": 0.3954179793853281, "nauc_recall_at_1_std": 0.05094634517180547, "nauc_recall_at_20_diff1": 0.38960282862441376, "nauc_recall_at_20_max": 0.3859266471606545, "nauc_recall_at_20_std": 0.07772930796862623, "nauc_recall_at_3_diff1": 0.4098250299008375, "nauc_recall_at_3_max": 0.39022839382240526, "nauc_recall_at_3_std": 0.027788026010023104, "nauc_recall_at_5_diff1": 0.3912032247692726, "nauc_recall_at_5_max": 0.37651414158503244, "nauc_recall_at_5_std": 0.023162297288624345, "ndcg_at_1": 0.49881, "ndcg_at_10": 0.64211, "ndcg_at_100": 0.67545, "ndcg_at_1000": 0.68117, "ndcg_at_20": 0.65623, "ndcg_at_3": 0.5974, "ndcg_at_5": 0.62353, "precision_at_1": 0.49881, "precision_at_10": 0.08884, "precision_at_100": 0.01076, "precision_at_1000": 0.00113, "precision_at_20": 0.04786, "precision_at_3": 0.24782, "precision_at_5": 0.16437, "recall_at_1": 0.46397, "recall_at_10": 0.77672, "recall_at_100": 0.9228, "recall_at_1000": 0.96635, "recall_at_20": 0.82779, "recall_at_3": 0.66508, "recall_at_5": 0.72328 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 785.9183006286621, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.24596, "map_at_1": 0.155, "map_at_10": 0.21462, "map_at_100": 0.22193, "map_at_1000": 0.22312, "map_at_20": 0.2179, "map_at_3": 0.19833, "map_at_5": 0.20833, "mrr_at_1": 0.155, "mrr_at_10": 0.21461706349206353, "mrr_at_100": 0.22192636282777436, "mrr_at_1000": 0.22311533240055414, "mrr_at_20": 0.21790329991645788, "mrr_at_3": 0.19833333333333333, "mrr_at_5": 0.20833333333333337, "nauc_map_at_1000_diff1": 0.4563057390364821, "nauc_map_at_1000_max": 0.5455807690019324, "nauc_map_at_1000_std": 0.07110024959719435, "nauc_map_at_100_diff1": 0.4564613211456158, "nauc_map_at_100_max": 0.545782541881912, "nauc_map_at_100_std": 0.07148056047352806, "nauc_map_at_10_diff1": 0.45678409482750154, "nauc_map_at_10_max": 0.5462465519755866, "nauc_map_at_10_std": 0.06327954999500235, "nauc_map_at_1_diff1": 0.5704239896928506, "nauc_map_at_1_max": 0.6534024410633671, "nauc_map_at_1_std": 0.023663168661545685, "nauc_map_at_20_diff1": 0.45630693262164523, "nauc_map_at_20_max": 0.5449016727511192, "nauc_map_at_20_std": 0.06628084219719646, "nauc_map_at_3_diff1": 0.46690651146077067, "nauc_map_at_3_max": 0.5624322695466907, "nauc_map_at_3_std": 0.0561734805035326, "nauc_map_at_5_diff1": 0.44601957495182504, "nauc_map_at_5_max": 0.546357284460649, "nauc_map_at_5_std": 0.05302593819749632, "nauc_mrr_at_1000_diff1": 0.4563057390364821, "nauc_mrr_at_1000_max": 0.5455807690019324, "nauc_mrr_at_1000_std": 0.07110024959719435, "nauc_mrr_at_100_diff1": 0.4564613211456158, "nauc_mrr_at_100_max": 0.545782541881912, "nauc_mrr_at_100_std": 0.07148056047352806, "nauc_mrr_at_10_diff1": 0.45678409482750154, "nauc_mrr_at_10_max": 0.5462465519755866, "nauc_mrr_at_10_std": 0.06327954999500235, "nauc_mrr_at_1_diff1": 0.5704239896928506, "nauc_mrr_at_1_max": 0.6534024410633671, "nauc_mrr_at_1_std": 0.023663168661545685, "nauc_mrr_at_20_diff1": 0.45630693262164523, "nauc_mrr_at_20_max": 0.5449016727511192, "nauc_mrr_at_20_std": 0.06628084219719646, "nauc_mrr_at_3_diff1": 0.46690651146077067, "nauc_mrr_at_3_max": 0.5624322695466907, "nauc_mrr_at_3_std": 0.0561734805035326, "nauc_mrr_at_5_diff1": 0.44601957495182504, "nauc_mrr_at_5_max": 0.546357284460649, "nauc_mrr_at_5_std": 0.05302593819749632, "nauc_ndcg_at_1000_diff1": 0.4223530675214165, "nauc_ndcg_at_1000_max": 0.5155856879665851, "nauc_ndcg_at_1000_std": 0.1227265877897248, "nauc_ndcg_at_100_diff1": 0.4208279282960337, "nauc_ndcg_at_100_max": 0.5117756592677707, "nauc_ndcg_at_100_std": 0.127400673676403, "nauc_ndcg_at_10_diff1": 0.42602576993830543, "nauc_ndcg_at_10_max": 0.51468292285022, "nauc_ndcg_at_10_std": 0.08414452558367269, "nauc_ndcg_at_1_diff1": 0.5704239896928506, "nauc_ndcg_at_1_max": 0.6534024410633671, "nauc_ndcg_at_1_std": 0.023663168661545685, "nauc_ndcg_at_20_diff1": 0.42282432452418245, "nauc_ndcg_at_20_max": 0.5087229428962105, "nauc_ndcg_at_20_std": 0.09178765600668244, "nauc_ndcg_at_3_diff1": 0.4391634377402065, "nauc_ndcg_at_3_max": 0.5392950770971048, "nauc_ndcg_at_3_std": 0.06310962168128884, "nauc_ndcg_at_5_diff1": 0.40367034656058765, "nauc_ndcg_at_5_max": 0.512798841157178, "nauc_ndcg_at_5_std": 0.058583454190707915, "nauc_precision_at_1000_diff1": 0.2302029929333522, "nauc_precision_at_1000_max": 0.40395940141333003, "nauc_precision_at_1000_std": 0.5805736455590976, "nauc_precision_at_100_diff1": 0.321829367933686, "nauc_precision_at_100_max": 0.42366111021878955, "nauc_precision_at_100_std": 0.33775847259369973, "nauc_precision_at_10_diff1": 0.3558943452085937, "nauc_precision_at_10_max": 0.441400183712985, "nauc_precision_at_10_std": 0.14219913694547434, "nauc_precision_at_1_diff1": 0.5704239896928506, "nauc_precision_at_1_max": 0.6534024410633671, "nauc_precision_at_1_std": 0.023663168661545685, "nauc_precision_at_20_diff1": 0.3438542385785166, "nauc_precision_at_20_max": 0.4207157742607733, "nauc_precision_at_20_std": 0.16179440361822794, "nauc_precision_at_3_diff1": 0.3702472349205655, "nauc_precision_at_3_max": 0.4821627769636258, "nauc_precision_at_3_std": 0.07985009879189534, "nauc_precision_at_5_diff1": 0.2995270909336579, "nauc_precision_at_5_max": 0.4312457775976221, "nauc_precision_at_5_std": 0.07129441967301732, "nauc_recall_at_1000_diff1": 0.23020299293335245, "nauc_recall_at_1000_max": 0.40395940141332987, "nauc_recall_at_1000_std": 0.5805736455590972, "nauc_recall_at_100_diff1": 0.32182936793368594, "nauc_recall_at_100_max": 0.4236611102187895, "nauc_recall_at_100_std": 0.33775847259369934, "nauc_recall_at_10_diff1": 0.35589434520859364, "nauc_recall_at_10_max": 0.44140018371298445, "nauc_recall_at_10_std": 0.14219913694547434, "nauc_recall_at_1_diff1": 0.5704239896928506, "nauc_recall_at_1_max": 0.6534024410633671, "nauc_recall_at_1_std": 0.023663168661545685, "nauc_recall_at_20_diff1": 0.34385423857851627, "nauc_recall_at_20_max": 0.42071577426077317, "nauc_recall_at_20_std": 0.16179440361822775, "nauc_recall_at_3_diff1": 0.37024723492056555, "nauc_recall_at_3_max": 0.48216277696362575, "nauc_recall_at_3_std": 0.07985009879189547, "nauc_recall_at_5_diff1": 0.29952709093365765, "nauc_recall_at_5_max": 0.431245777597622, "nauc_recall_at_5_std": 0.07129441967301701, "ndcg_at_1": 0.155, "ndcg_at_10": 0.24596, "ndcg_at_100": 0.2866, "ndcg_at_1000": 0.32479, "ndcg_at_20": 0.25838, "ndcg_at_3": 0.21286, "ndcg_at_5": 0.23114, "precision_at_1": 0.155, "precision_at_10": 0.0345, "precision_at_100": 0.00545, "precision_at_1000": 0.00086, "precision_at_20": 0.01975, "precision_at_3": 0.085, "precision_at_5": 0.06, "recall_at_1": 0.155, "recall_at_10": 0.345, "recall_at_100": 0.545, "recall_at_1000": 0.86, "recall_at_20": 0.395, "recall_at_3": 0.255, "recall_at_5": 0.3 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.2708, "map_at_1": 0.19, "map_at_10": 0.23921, "map_at_100": 0.24591, "map_at_1000": 0.24682, "map_at_20": 0.2428, "map_at_3": 0.2225, "map_at_5": 0.22825, "mrr_at_1": 0.19, "mrr_at_10": 0.23920634920634926, "mrr_at_100": 0.24590719223302934, "mrr_at_1000": 0.24682316999124865, "mrr_at_20": 0.2428042409946435, "mrr_at_3": 0.2225, "mrr_at_5": 0.22825000000000004, "nauc_map_at_1000_diff1": 0.578021990250392, "nauc_map_at_1000_max": 0.523318979040455, "nauc_map_at_1000_std": 0.10476750567563853, "nauc_map_at_100_diff1": 0.5775438826446706, "nauc_map_at_100_max": 0.522536444698086, "nauc_map_at_100_std": 0.1043256942507403, "nauc_map_at_10_diff1": 0.5780895156704443, "nauc_map_at_10_max": 0.5268633688986352, "nauc_map_at_10_std": 0.10257357841351267, "nauc_map_at_1_diff1": 0.6510265162342577, "nauc_map_at_1_max": 0.5271047366031646, "nauc_map_at_1_std": 0.055958165890404005, "nauc_map_at_20_diff1": 0.5787872676341045, "nauc_map_at_20_max": 0.5247364919928256, "nauc_map_at_20_std": 0.10278104143405609, "nauc_map_at_3_diff1": 0.6171579403599334, "nauc_map_at_3_max": 0.5469596217147008, "nauc_map_at_3_std": 0.09000285041565309, "nauc_map_at_5_diff1": 0.5979177152578171, "nauc_map_at_5_max": 0.539514391080624, "nauc_map_at_5_std": 0.09213374119424868, "nauc_mrr_at_1000_diff1": 0.578021990250392, "nauc_mrr_at_1000_max": 0.523318979040455, "nauc_mrr_at_1000_std": 0.10476750567563853, "nauc_mrr_at_100_diff1": 0.5775438826446706, "nauc_mrr_at_100_max": 0.522536444698086, "nauc_mrr_at_100_std": 0.1043256942507403, "nauc_mrr_at_10_diff1": 0.5780895156704443, "nauc_mrr_at_10_max": 0.5268633688986352, "nauc_mrr_at_10_std": 0.10257357841351267, "nauc_mrr_at_1_diff1": 0.6510265162342577, "nauc_mrr_at_1_max": 0.5271047366031646, "nauc_mrr_at_1_std": 0.055958165890404005, "nauc_mrr_at_20_diff1": 0.5787872676341045, "nauc_mrr_at_20_max": 0.5247364919928256, "nauc_mrr_at_20_std": 0.10278104143405609, "nauc_mrr_at_3_diff1": 0.6171579403599334, "nauc_mrr_at_3_max": 0.5469596217147008, "nauc_mrr_at_3_std": 0.09000285041565309, "nauc_mrr_at_5_diff1": 0.5979177152578171, "nauc_mrr_at_5_max": 0.539514391080624, "nauc_mrr_at_5_std": 0.09213374119424868, "nauc_ndcg_at_1000_diff1": 0.5288524441900466, "nauc_ndcg_at_1000_max": 0.5043243755008304, "nauc_ndcg_at_1000_std": 0.14684049480460046, "nauc_ndcg_at_100_diff1": 0.5177677734480506, "nauc_ndcg_at_100_max": 0.49039519277894145, "nauc_ndcg_at_100_std": 0.1449929646229025, "nauc_ndcg_at_10_diff1": 0.5294774420810564, "nauc_ndcg_at_10_max": 0.5150527233958064, "nauc_ndcg_at_10_std": 0.1307503383225075, "nauc_ndcg_at_1_diff1": 0.6510265162342577, "nauc_ndcg_at_1_max": 0.5271047366031646, "nauc_ndcg_at_1_std": 0.055958165890404005, "nauc_ndcg_at_20_diff1": 0.5311272743578979, "nauc_ndcg_at_20_max": 0.5059964177926148, "nauc_ndcg_at_20_std": 0.13017095675485332, "nauc_ndcg_at_3_diff1": 0.606812226397841, "nauc_ndcg_at_3_max": 0.5558310317804566, "nauc_ndcg_at_3_std": 0.10534719216179274, "nauc_ndcg_at_5_diff1": 0.5734308027380774, "nauc_ndcg_at_5_max": 0.5423518753341231, "nauc_ndcg_at_5_std": 0.10834330908441163, "nauc_precision_at_1000_diff1": 0.2733106021961501, "nauc_precision_at_1000_max": 0.4060192767202729, "nauc_precision_at_1000_std": 0.4061535157193871, "nauc_precision_at_100_diff1": 0.3053573361232372, "nauc_precision_at_100_max": 0.3555323063602586, "nauc_precision_at_100_std": 0.29667015404596153, "nauc_precision_at_10_diff1": 0.3921055202735711, "nauc_precision_at_10_max": 0.4774857515062697, "nauc_precision_at_10_std": 0.20782934375508877, "nauc_precision_at_1_diff1": 0.6510265162342577, "nauc_precision_at_1_max": 0.5271047366031646, "nauc_precision_at_1_std": 0.055958165890404005, "nauc_precision_at_20_diff1": 0.396730436592059, "nauc_precision_at_20_max": 0.44284116476570745, "nauc_precision_at_20_std": 0.20331378184703042, "nauc_precision_at_3_diff1": 0.5792535698676661, "nauc_precision_at_3_max": 0.5802423139477403, "nauc_precision_at_3_std": 0.14759513079364642, "nauc_precision_at_5_diff1": 0.5061741961182923, "nauc_precision_at_5_max": 0.5485831627817537, "nauc_precision_at_5_std": 0.1515871598697858, "nauc_recall_at_1000_diff1": 0.2733106021961501, "nauc_recall_at_1000_max": 0.4060192767202733, "nauc_recall_at_1000_std": 0.40615351571938707, "nauc_recall_at_100_diff1": 0.30535733612323657, "nauc_recall_at_100_max": 0.35553230636025834, "nauc_recall_at_100_std": 0.2966701540459615, "nauc_recall_at_10_diff1": 0.392105520273571, "nauc_recall_at_10_max": 0.4774857515062694, "nauc_recall_at_10_std": 0.20782934375508869, "nauc_recall_at_1_diff1": 0.6510265162342577, "nauc_recall_at_1_max": 0.5271047366031646, "nauc_recall_at_1_std": 0.055958165890404005, "nauc_recall_at_20_diff1": 0.3967304365920587, "nauc_recall_at_20_max": 0.4428411647657072, "nauc_recall_at_20_std": 0.20331378184703022, "nauc_recall_at_3_diff1": 0.5792535698676662, "nauc_recall_at_3_max": 0.5802423139477398, "nauc_recall_at_3_std": 0.14759513079364625, "nauc_recall_at_5_diff1": 0.506174196118292, "nauc_recall_at_5_max": 0.5485831627817535, "nauc_recall_at_5_std": 0.15158715986978555, "ndcg_at_1": 0.19, "ndcg_at_10": 0.2708, "ndcg_at_100": 0.31019, "ndcg_at_1000": 0.33916, "ndcg_at_20": 0.28444, "ndcg_at_3": 0.23339, "ndcg_at_5": 0.24372, "precision_at_1": 0.19, "precision_at_10": 0.0375, "precision_at_100": 0.00575, "precision_at_1000": 0.00082, "precision_at_20": 0.0215, "precision_at_3": 0.08833, "precision_at_5": 0.058, "recall_at_1": 0.19, "recall_at_10": 0.375, "recall_at_100": 0.575, "recall_at_1000": 0.815, "recall_at_20": 0.43, "recall_at_3": 0.265, "recall_at_5": 0.29 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 5.381413459777832, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.82534, "map_at_1": 0.68831, "map_at_10": 0.78566, "map_at_100": 0.78832, "map_at_1000": 0.78832, "map_at_20": 0.78739, "map_at_3": 0.77056, "map_at_5": 0.77641, "mrr_at_1": 0.6883116883116883, "mrr_at_10": 0.785657596371882, "mrr_at_100": 0.7883238694927007, "mrr_at_1000": 0.7883238694927007, "mrr_at_20": 0.7873891981034838, "mrr_at_3": 0.7705627705627704, "mrr_at_5": 0.7764069264069264, "nauc_map_at_1000_diff1": 0.8042948615488029, "nauc_map_at_1000_max": 0.22404852716375828, "nauc_map_at_1000_std": -0.27881068564841477, "nauc_map_at_100_diff1": 0.8042948615488029, "nauc_map_at_100_max": 0.22404852716375828, "nauc_map_at_100_std": -0.27881068564841477, "nauc_map_at_10_diff1": 0.8025655422786346, "nauc_map_at_10_max": 0.22230817835873698, "nauc_map_at_10_std": -0.2806074153409453, "nauc_map_at_1_diff1": 0.8302946692246838, "nauc_map_at_1_max": 0.2317241170179748, "nauc_map_at_1_std": -0.192793491783307, "nauc_map_at_20_diff1": 0.8034037360099546, "nauc_map_at_20_max": 0.2245496806184615, "nauc_map_at_20_std": -0.27523588788328135, "nauc_map_at_3_diff1": 0.8022516732629955, "nauc_map_at_3_max": 0.2221615156737509, "nauc_map_at_3_std": -0.30766923209187896, "nauc_map_at_5_diff1": 0.8036926446757969, "nauc_map_at_5_max": 0.2241716349857892, "nauc_map_at_5_std": -0.3102630334004927, "nauc_mrr_at_1000_diff1": 0.8042948615488029, "nauc_mrr_at_1000_max": 0.22404852716375828, "nauc_mrr_at_1000_std": -0.27881068564841477, "nauc_mrr_at_100_diff1": 0.8042948615488029, "nauc_mrr_at_100_max": 0.22404852716375828, "nauc_mrr_at_100_std": -0.27881068564841477, "nauc_mrr_at_10_diff1": 0.8025655422786346, "nauc_mrr_at_10_max": 0.22230817835873698, "nauc_mrr_at_10_std": -0.2806074153409453, "nauc_mrr_at_1_diff1": 0.8302946692246838, "nauc_mrr_at_1_max": 0.2317241170179748, "nauc_mrr_at_1_std": -0.192793491783307, "nauc_mrr_at_20_diff1": 0.8034037360099546, "nauc_mrr_at_20_max": 0.2245496806184615, "nauc_mrr_at_20_std": -0.27523588788328135, "nauc_mrr_at_3_diff1": 0.8022516732629955, "nauc_mrr_at_3_max": 0.2221615156737509, "nauc_mrr_at_3_std": -0.30766923209187896, "nauc_mrr_at_5_diff1": 0.8036926446757969, "nauc_mrr_at_5_max": 0.2241716349857892, "nauc_mrr_at_5_std": -0.3102630334004927, "nauc_ndcg_at_1000_diff1": 0.800535416072367, "nauc_ndcg_at_1000_max": 0.22376165537986312, "nauc_ndcg_at_1000_std": -0.2845177705394917, "nauc_ndcg_at_100_diff1": 0.800535416072367, "nauc_ndcg_at_100_max": 0.22376165537986312, "nauc_ndcg_at_100_std": -0.2845177705394917, "nauc_ndcg_at_10_diff1": 0.791163555666361, "nauc_ndcg_at_10_max": 0.21930105422089738, "nauc_ndcg_at_10_std": -0.2823488334907109, "nauc_ndcg_at_1_diff1": 0.8302946692246838, "nauc_ndcg_at_1_max": 0.2317241170179748, "nauc_ndcg_at_1_std": -0.192793491783307, "nauc_ndcg_at_20_diff1": 0.7946105403995175, "nauc_ndcg_at_20_max": 0.22971240795889006, "nauc_ndcg_at_20_std": -0.2571618122570241, "nauc_ndcg_at_3_diff1": 0.7916549180462429, "nauc_ndcg_at_3_max": 0.2165935406595234, "nauc_ndcg_at_3_std": -0.3575105094880901, "nauc_ndcg_at_5_diff1": 0.7943239308426, "nauc_ndcg_at_5_max": 0.22129779544783112, "nauc_ndcg_at_5_std": -0.3642016640354031, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.6752521641350954, "nauc_precision_at_10_max": 0.1981185792546289, "nauc_precision_at_10_std": -0.18261458357571947, "nauc_precision_at_1_diff1": 0.8302946692246838, "nauc_precision_at_1_max": 0.2317241170179748, "nauc_precision_at_1_std": -0.192793491783307, "nauc_precision_at_20_diff1": 0.6387246499035996, "nauc_precision_at_20_max": 0.4167090338610833, "nauc_precision_at_20_std": 0.49717544202323205, "nauc_precision_at_3_diff1": 0.7474623592532387, "nauc_precision_at_3_max": 0.19238123933939982, "nauc_precision_at_3_std": -0.5686316973323878, "nauc_precision_at_5_diff1": 0.7498540019990121, "nauc_precision_at_5_max": 0.20849567237675368, "nauc_precision_at_5_std": -0.6310025883818912, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.6752521641350948, "nauc_recall_at_10_max": 0.1981185792546231, "nauc_recall_at_10_std": -0.18261458357572427, "nauc_recall_at_1_diff1": 0.8302946692246838, "nauc_recall_at_1_max": 0.2317241170179748, "nauc_recall_at_1_std": -0.192793491783307, "nauc_recall_at_20_diff1": 0.6387246499035979, "nauc_recall_at_20_max": 0.41670903386108427, "nauc_recall_at_20_std": 0.4971754420232289, "nauc_recall_at_3_diff1": 0.7474623592532373, "nauc_recall_at_3_max": 0.19238123933939866, "nauc_recall_at_3_std": -0.5686316973323894, "nauc_recall_at_5_diff1": 0.7498540019990096, "nauc_recall_at_5_max": 0.2084956723767503, "nauc_recall_at_5_std": -0.6310025883818963, "ndcg_at_1": 0.68831, "ndcg_at_10": 0.82534, "ndcg_at_100": 0.83718, "ndcg_at_1000": 0.83718, "ndcg_at_20": 0.83183, "ndcg_at_3": 0.79313, "ndcg_at_5": 0.80375, "precision_at_1": 0.68831, "precision_at_10": 0.09481, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.0487, "precision_at_3": 0.28571, "precision_at_5": 0.17662, "recall_at_1": 0.68831, "recall_at_10": 0.94805, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.97403, "recall_at_3": 0.85714, "recall_at_5": 0.88312 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 32.079678773880005, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.3571, "map_at_1": 0.19353, "map_at_10": 0.30797, "map_at_100": 0.3219, "map_at_1000": 0.32316, "map_at_20": 0.31431, "map_at_3": 0.27544, "map_at_5": 0.29204, "mrr_at_1": 0.2798165137614679, "mrr_at_10": 0.34842240667928726, "mrr_at_100": 0.3587165009528493, "mrr_at_1000": 0.35953917996926965, "mrr_at_20": 0.3533289999823643, "mrr_at_3": 0.3256880733944955, "mrr_at_5": 0.33639143730886856, "nauc_map_at_1000_diff1": 0.5138549298770038, "nauc_map_at_1000_max": 0.15706611767695436, "nauc_map_at_1000_std": -0.21947882754711157, "nauc_map_at_100_diff1": 0.5134957364907854, "nauc_map_at_100_max": 0.1572239942298593, "nauc_map_at_100_std": -0.21899257138513686, "nauc_map_at_10_diff1": 0.517443829870831, "nauc_map_at_10_max": 0.1576616812242588, "nauc_map_at_10_std": -0.22424136385398943, "nauc_map_at_1_diff1": 0.6026428255407352, "nauc_map_at_1_max": 0.1333788664457722, "nauc_map_at_1_std": -0.19186056452745517, "nauc_map_at_20_diff1": 0.5129827770934847, "nauc_map_at_20_max": 0.15655611825932708, "nauc_map_at_20_std": -0.22167463561240425, "nauc_map_at_3_diff1": 0.5443309558055008, "nauc_map_at_3_max": 0.15847786314733434, "nauc_map_at_3_std": -0.22563259495426558, "nauc_map_at_5_diff1": 0.5307359022230224, "nauc_map_at_5_max": 0.16149176531535286, "nauc_map_at_5_std": -0.2313479654688125, "nauc_mrr_at_1000_diff1": 0.5283827722446927, "nauc_mrr_at_1000_max": 0.1506119471141864, "nauc_mrr_at_1000_std": -0.21075871618386696, "nauc_mrr_at_100_diff1": 0.5280079337854601, "nauc_mrr_at_100_max": 0.15075854483196036, "nauc_mrr_at_100_std": -0.21008844904204566, "nauc_mrr_at_10_diff1": 0.5298360528677714, "nauc_mrr_at_10_max": 0.14870326948232312, "nauc_mrr_at_10_std": -0.214380021311685, "nauc_mrr_at_1_diff1": 0.5743959063219708, "nauc_mrr_at_1_max": 0.16265322563646364, "nauc_mrr_at_1_std": -0.18833485079936244, "nauc_mrr_at_20_diff1": 0.5279608493500507, "nauc_mrr_at_20_max": 0.15013039546532891, "nauc_mrr_at_20_std": -0.21149396060210648, "nauc_mrr_at_3_diff1": 0.5421280864610498, "nauc_mrr_at_3_max": 0.15026320651956152, "nauc_mrr_at_3_std": -0.21799080721300038, "nauc_mrr_at_5_diff1": 0.5370787309258472, "nauc_mrr_at_5_max": 0.15248396940948003, "nauc_mrr_at_5_std": -0.21967449720066567, "nauc_ndcg_at_1000_diff1": 0.49811150509547963, "nauc_ndcg_at_1000_max": 0.15640312953034563, "nauc_ndcg_at_1000_std": -0.2042439819629608, "nauc_ndcg_at_100_diff1": 0.4852555831694557, "nauc_ndcg_at_100_max": 0.16115266284830218, "nauc_ndcg_at_100_std": -0.18903752480149202, "nauc_ndcg_at_10_diff1": 0.4959985025680068, "nauc_ndcg_at_10_max": 0.15079190670828926, "nauc_ndcg_at_10_std": -0.2242977737169114, "nauc_ndcg_at_1_diff1": 0.5743959063219708, "nauc_ndcg_at_1_max": 0.16265322563646364, "nauc_ndcg_at_1_std": -0.18833485079936244, "nauc_ndcg_at_20_diff1": 0.4847466273790778, "nauc_ndcg_at_20_max": 0.1510848080108304, "nauc_ndcg_at_20_std": -0.21481695964044425, "nauc_ndcg_at_3_diff1": 0.5355513006467298, "nauc_ndcg_at_3_max": 0.15115082409197186, "nauc_ndcg_at_3_std": -0.2326831413567524, "nauc_ndcg_at_5_diff1": 0.5195025224339292, "nauc_ndcg_at_5_max": 0.1578451311010888, "nauc_ndcg_at_5_std": -0.23854976550429074, "nauc_precision_at_1000_diff1": -0.03589529200880238, "nauc_precision_at_1000_max": 0.0721575870675354, "nauc_precision_at_1000_std": 0.11651396952596191, "nauc_precision_at_100_diff1": 0.08463744093830189, "nauc_precision_at_100_max": 0.12889529870201247, "nauc_precision_at_100_std": 0.07228197543698799, "nauc_precision_at_10_diff1": 0.2548094988167048, "nauc_precision_at_10_max": 0.10758428277045155, "nauc_precision_at_10_std": -0.16332896820174375, "nauc_precision_at_1_diff1": 0.5743959063219708, "nauc_precision_at_1_max": 0.16265322563646364, "nauc_precision_at_1_std": -0.18833485079936244, "nauc_precision_at_20_diff1": 0.1884737665414867, "nauc_precision_at_20_max": 0.10039935632406857, "nauc_precision_at_20_std": -0.10994045270402089, "nauc_precision_at_3_diff1": 0.38339762191899696, "nauc_precision_at_3_max": 0.14601652881659463, "nauc_precision_at_3_std": -0.20711924836563086, "nauc_precision_at_5_diff1": 0.33200959009193537, "nauc_precision_at_5_max": 0.13913915589491396, "nauc_precision_at_5_std": -0.2129621767324046, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.30827274858191983, "nauc_recall_at_100_max": 0.2011530724751607, "nauc_recall_at_100_std": -0.010097304957252298, "nauc_recall_at_10_diff1": 0.40650972514828604, "nauc_recall_at_10_max": 0.1407876095206232, "nauc_recall_at_10_std": -0.22041004043085902, "nauc_recall_at_1_diff1": 0.6026428255407352, "nauc_recall_at_1_max": 0.1333788664457722, "nauc_recall_at_1_std": -0.19186056452745517, "nauc_recall_at_20_diff1": 0.3669244580517796, "nauc_recall_at_20_max": 0.14017332698852716, "nauc_recall_at_20_std": -0.19365573286425647, "nauc_recall_at_3_diff1": 0.5076772126959125, "nauc_recall_at_3_max": 0.14378997670580718, "nauc_recall_at_3_std": -0.2508505427267107, "nauc_recall_at_5_diff1": 0.4695357175880984, "nauc_recall_at_5_max": 0.15627328191656, "nauc_recall_at_5_std": -0.25764388965660323, "ndcg_at_1": 0.27982, "ndcg_at_10": 0.3571, "ndcg_at_100": 0.42603, "ndcg_at_1000": 0.45678, "ndcg_at_20": 0.37717, "ndcg_at_3": 0.30518, "ndcg_at_5": 0.32476, "precision_at_1": 0.27982, "precision_at_10": 0.07477, "precision_at_100": 0.0124, "precision_at_1000": 0.00156, "precision_at_20": 0.04335, "precision_at_3": 0.1738, "precision_at_5": 0.12141, "recall_at_1": 0.19353, "recall_at_10": 0.46376, "recall_at_100": 0.77339, "recall_at_1000": 1.0, "recall_at_20": 0.53236, "recall_at_3": 0.32531, "recall_at_5": 0.37551 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.31094, "map_at_1": 0.11333, "map_at_10": 0.24545, "map_at_100": 0.26562, "map_at_1000": 0.26749, "map_at_20": 0.25597, "map_at_3": 0.19906, "map_at_5": 0.22564, "mrr_at_1": 0.20336391437308868, "mrr_at_10": 0.3066459152468327, "mrr_at_100": 0.3200820113927799, "mrr_at_1000": 0.3208093102193586, "mrr_at_20": 0.31510573567817174, "mrr_at_3": 0.27777777777777796, "mrr_at_5": 0.2939092762487259, "nauc_map_at_1000_diff1": 0.11425958191910945, "nauc_map_at_1000_max": 0.13522112373677148, "nauc_map_at_1000_std": -0.23322284294211604, "nauc_map_at_100_diff1": 0.11413610768701515, "nauc_map_at_100_max": 0.13370690711946526, "nauc_map_at_100_std": -0.23411130471182684, "nauc_map_at_10_diff1": 0.1184495655302733, "nauc_map_at_10_max": 0.13563441812048518, "nauc_map_at_10_std": -0.24270965115117196, "nauc_map_at_1_diff1": 0.1692354811812946, "nauc_map_at_1_max": 0.21218528961951696, "nauc_map_at_1_std": -0.17591883514018467, "nauc_map_at_20_diff1": 0.11725028326714887, "nauc_map_at_20_max": 0.13323610200660557, "nauc_map_at_20_std": -0.2404168732861843, "nauc_map_at_3_diff1": 0.11677334137424876, "nauc_map_at_3_max": 0.16600746017544304, "nauc_map_at_3_std": -0.2255894882300477, "nauc_map_at_5_diff1": 0.12201030040561259, "nauc_map_at_5_max": 0.14770064826124832, "nauc_map_at_5_std": -0.23569636575078878, "nauc_mrr_at_1000_diff1": 0.1039871361198157, "nauc_mrr_at_1000_max": 0.13294927914886112, "nauc_mrr_at_1000_std": -0.22912308332946335, "nauc_mrr_at_100_diff1": 0.10377355588617188, "nauc_mrr_at_100_max": 0.13239832936623644, "nauc_mrr_at_100_std": -0.2290072339056491, "nauc_mrr_at_10_diff1": 0.10754838047109636, "nauc_mrr_at_10_max": 0.13026701471658167, "nauc_mrr_at_10_std": -0.23916824363969363, "nauc_mrr_at_1_diff1": 0.15805964434970776, "nauc_mrr_at_1_max": 0.23059526255686508, "nauc_mrr_at_1_std": -0.1766781943662403, "nauc_mrr_at_20_diff1": 0.10352053698401652, "nauc_mrr_at_20_max": 0.13038796281537302, "nauc_mrr_at_20_std": -0.23313763025758455, "nauc_mrr_at_3_diff1": 0.10803633282547864, "nauc_mrr_at_3_max": 0.14947870220370624, "nauc_mrr_at_3_std": -0.22835391953660417, "nauc_mrr_at_5_diff1": 0.10620245353325078, "nauc_mrr_at_5_max": 0.13468665184581083, "nauc_mrr_at_5_std": -0.2341643874293665, "nauc_ndcg_at_1000_diff1": 0.09251582953701985, "nauc_ndcg_at_1000_max": 0.11518617431466921, "nauc_ndcg_at_1000_std": -0.2210907355602964, "nauc_ndcg_at_100_diff1": 0.08223322787581405, "nauc_ndcg_at_100_max": 0.09834852768094031, "nauc_ndcg_at_100_std": -0.21658736652593918, "nauc_ndcg_at_10_diff1": 0.10156045352650941, "nauc_ndcg_at_10_max": 0.10073501229237586, "nauc_ndcg_at_10_std": -0.2622246009221012, "nauc_ndcg_at_1_diff1": 0.15805964434970776, "nauc_ndcg_at_1_max": 0.23059526255686508, "nauc_ndcg_at_1_std": -0.1766781943662403, "nauc_ndcg_at_20_diff1": 0.09221995427715365, "nauc_ndcg_at_20_max": 0.09270514139435625, "nauc_ndcg_at_20_std": -0.2513232750935288, "nauc_ndcg_at_3_diff1": 0.1060598008256348, "nauc_ndcg_at_3_max": 0.15037561069761582, "nauc_ndcg_at_3_std": -0.24293322987087043, "nauc_ndcg_at_5_diff1": 0.10662261587225039, "nauc_ndcg_at_5_max": 0.11923002425610947, "nauc_ndcg_at_5_std": -0.24917569135736384, "nauc_precision_at_1000_diff1": -0.00545209595105802, "nauc_precision_at_1000_max": 0.009575450938722765, "nauc_precision_at_1000_std": 0.013030685694043514, "nauc_precision_at_100_diff1": -0.012190888503449399, "nauc_precision_at_100_max": -0.01150201764909971, "nauc_precision_at_100_std": -0.028989227689994995, "nauc_precision_at_10_diff1": 0.07100666295380739, "nauc_precision_at_10_max": 0.023002780783397795, "nauc_precision_at_10_std": -0.23243183541419282, "nauc_precision_at_1_diff1": 0.15805964434970776, "nauc_precision_at_1_max": 0.23059526255686508, "nauc_precision_at_1_std": -0.1766781943662403, "nauc_precision_at_20_diff1": 0.04742722206087879, "nauc_precision_at_20_max": -0.0021771031283383964, "nauc_precision_at_20_std": -0.19515025526504237, "nauc_precision_at_3_diff1": 0.08617923357508446, "nauc_precision_at_3_max": 0.13180630249788944, "nauc_precision_at_3_std": -0.24183469944734207, "nauc_precision_at_5_diff1": 0.0874821096786221, "nauc_precision_at_5_max": 0.07866866827452602, "nauc_precision_at_5_std": -0.24266623091805606, "nauc_recall_at_1000_diff1": 0.7222547590617517, "nauc_recall_at_1000_max": 0.869369819235015, "nauc_recall_at_1000_std": 0.7222547590617517, "nauc_recall_at_100_diff1": -0.054201511406694665, "nauc_recall_at_100_max": -0.034624616677202624, "nauc_recall_at_100_std": -0.07300396774406927, "nauc_recall_at_10_diff1": 0.05828772902103733, "nauc_recall_at_10_max": 0.028377440281751782, "nauc_recall_at_10_std": -0.28217277278383585, "nauc_recall_at_1_diff1": 0.1692354811812946, "nauc_recall_at_1_max": 0.21218528961951696, "nauc_recall_at_1_std": -0.17591883514018467, "nauc_recall_at_20_diff1": 0.024863917911959483, "nauc_recall_at_20_max": -0.0017203766062452103, "nauc_recall_at_20_std": -0.24781992086304866, "nauc_recall_at_3_diff1": 0.06276907993627277, "nauc_recall_at_3_max": 0.10532900063127473, "nauc_recall_at_3_std": -0.2467354814385313, "nauc_recall_at_5_diff1": 0.07359713001358666, "nauc_recall_at_5_max": 0.05871797288722992, "nauc_recall_at_5_std": -0.2559901740274049, "ndcg_at_1": 0.20336, "ndcg_at_10": 0.31094, "ndcg_at_100": 0.39884, "ndcg_at_1000": 0.42943, "ndcg_at_20": 0.34327, "ndcg_at_3": 0.24829, "ndcg_at_5": 0.27407, "precision_at_1": 0.20336, "precision_at_10": 0.0867, "precision_at_100": 0.0158, "precision_at_1000": 0.00195, "precision_at_20": 0.05367, "precision_at_3": 0.1682, "precision_at_5": 0.13272, "recall_at_1": 0.11333, "recall_at_10": 0.43379, "recall_at_100": 0.7987, "recall_at_1000": 0.99847, "recall_at_20": 0.54123, "recall_at_3": 0.26741, "recall_at_5": 0.34264 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.34022, "map_at_1": 0.18219, "map_at_10": 0.29275, "map_at_100": 0.30851, "map_at_1000": 0.30996, "map_at_20": 0.30086, "map_at_3": 0.25706, "map_at_5": 0.2773, "mrr_at_1": 0.25732899022801303, "mrr_at_10": 0.32775838891474063, "mrr_at_100": 0.33930894289400076, "mrr_at_1000": 0.34010859395124843, "mrr_at_20": 0.33415785156577893, "mrr_at_3": 0.3056460369163953, "mrr_at_5": 0.3163952225841477, "nauc_map_at_1000_diff1": 0.43378857137081317, "nauc_map_at_1000_max": 0.15535049635339068, "nauc_map_at_1000_std": -0.13077142854953677, "nauc_map_at_100_diff1": 0.4333201230465107, "nauc_map_at_100_max": 0.1554593261211638, "nauc_map_at_100_std": -0.13057160318147662, "nauc_map_at_10_diff1": 0.4346209427682855, "nauc_map_at_10_max": 0.156591838771751, "nauc_map_at_10_std": -0.1320605005830169, "nauc_map_at_1_diff1": 0.5225535776695338, "nauc_map_at_1_max": 0.15584196650847043, "nauc_map_at_1_std": -0.08921387429404619, "nauc_map_at_20_diff1": 0.43355620094562536, "nauc_map_at_20_max": 0.15424228962802783, "nauc_map_at_20_std": -0.13123767883247464, "nauc_map_at_3_diff1": 0.45188009332621976, "nauc_map_at_3_max": 0.15456738468260678, "nauc_map_at_3_std": -0.12090759593450583, "nauc_map_at_5_diff1": 0.4359700028114134, "nauc_map_at_5_max": 0.1624127622487365, "nauc_map_at_5_std": -0.12398811036745316, "nauc_mrr_at_1000_diff1": 0.47154484905734406, "nauc_mrr_at_1000_max": 0.18288056423097326, "nauc_mrr_at_1000_std": -0.12756826523652975, "nauc_mrr_at_100_diff1": 0.4714010642831594, "nauc_mrr_at_100_max": 0.18303134467603407, "nauc_mrr_at_100_std": -0.12749949375579667, "nauc_mrr_at_10_diff1": 0.47336164375825707, "nauc_mrr_at_10_max": 0.17949764221735748, "nauc_mrr_at_10_std": -0.13149193189339925, "nauc_mrr_at_1_diff1": 0.505606121700976, "nauc_mrr_at_1_max": 0.1971297183494224, "nauc_mrr_at_1_std": -0.11120789239003809, "nauc_mrr_at_20_diff1": 0.4709653518631757, "nauc_mrr_at_20_max": 0.1834552198970404, "nauc_mrr_at_20_std": -0.12617556743828223, "nauc_mrr_at_3_diff1": 0.48088063281998156, "nauc_mrr_at_3_max": 0.18655389744378906, "nauc_mrr_at_3_std": -0.13278801543058455, "nauc_mrr_at_5_diff1": 0.47522461248833187, "nauc_mrr_at_5_max": 0.18535019838518463, "nauc_mrr_at_5_std": -0.1299630665463673, "nauc_ndcg_at_1000_diff1": 0.43226978854923204, "nauc_ndcg_at_1000_max": 0.16318872909152207, "nauc_ndcg_at_1000_std": -0.12677086508048077, "nauc_ndcg_at_100_diff1": 0.4197454292188696, "nauc_ndcg_at_100_max": 0.16471287955051728, "nauc_ndcg_at_100_std": -0.1242050691448354, "nauc_ndcg_at_10_diff1": 0.42789163679262, "nauc_ndcg_at_10_max": 0.15174365698596645, "nauc_ndcg_at_10_std": -0.13983430076673511, "nauc_ndcg_at_1_diff1": 0.505606121700976, "nauc_ndcg_at_1_max": 0.1971297183494224, "nauc_ndcg_at_1_std": -0.11120789239003809, "nauc_ndcg_at_20_diff1": 0.4228092278846085, "nauc_ndcg_at_20_max": 0.1521780768329811, "nauc_ndcg_at_20_std": -0.13081564253954298, "nauc_ndcg_at_3_diff1": 0.44538947482989755, "nauc_ndcg_at_3_max": 0.15294276809515184, "nauc_ndcg_at_3_std": -0.13479945201257715, "nauc_ndcg_at_5_diff1": 0.4332480000238585, "nauc_ndcg_at_5_max": 0.16421233739990027, "nauc_ndcg_at_5_std": -0.13024612438239955, "nauc_precision_at_1000_diff1": 0.007813782679524005, "nauc_precision_at_1000_max": 0.12181094948854453, "nauc_precision_at_1000_std": 0.026396427608266396, "nauc_precision_at_100_diff1": 0.08275612190388706, "nauc_precision_at_100_max": 0.1470704819648003, "nauc_precision_at_100_std": -0.014471486575332319, "nauc_precision_at_10_diff1": 0.227080207545651, "nauc_precision_at_10_max": 0.10356769209172943, "nauc_precision_at_10_std": -0.16004221638654986, "nauc_precision_at_1_diff1": 0.505606121700976, "nauc_precision_at_1_max": 0.1971297183494224, "nauc_precision_at_1_std": -0.11120789239003809, "nauc_precision_at_20_diff1": 0.18498933751641064, "nauc_precision_at_20_max": 0.09046124563031495, "nauc_precision_at_20_std": -0.1278407870889799, "nauc_precision_at_3_diff1": 0.30644575638980337, "nauc_precision_at_3_max": 0.1458981532783693, "nauc_precision_at_3_std": -0.14357842661749431, "nauc_precision_at_5_diff1": 0.2517676981179563, "nauc_precision_at_5_max": 0.14297124191463026, "nauc_precision_at_5_std": -0.1488828633233652, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.2597346348531165, "nauc_recall_at_100_max": 0.16776282781560076, "nauc_recall_at_100_std": -0.08738029961423872, "nauc_recall_at_10_diff1": 0.36399932536981683, "nauc_recall_at_10_max": 0.11894029557077423, "nauc_recall_at_10_std": -0.14673953529637915, "nauc_recall_at_1_diff1": 0.5225535776695338, "nauc_recall_at_1_max": 0.15584196650847043, "nauc_recall_at_1_std": -0.08921387429404619, "nauc_recall_at_20_diff1": 0.33968994682381265, "nauc_recall_at_20_max": 0.11705223183609517, "nauc_recall_at_20_std": -0.1150270696095652, "nauc_recall_at_3_diff1": 0.4222884972688852, "nauc_recall_at_3_max": 0.13077819571940505, "nauc_recall_at_3_std": -0.14150932434179694, "nauc_recall_at_5_diff1": 0.3873390941681804, "nauc_recall_at_5_max": 0.14812299412156793, "nauc_recall_at_5_std": -0.13082787279472374, "ndcg_at_1": 0.25733, "ndcg_at_10": 0.34022, "ndcg_at_100": 0.41412, "ndcg_at_1000": 0.44655, "ndcg_at_20": 0.3661, "ndcg_at_3": 0.28539, "ndcg_at_5": 0.30887, "precision_at_1": 0.25733, "precision_at_10": 0.07557, "precision_at_100": 0.01313, "precision_at_1000": 0.00167, "precision_at_20": 0.0456, "precision_at_3": 0.1645, "precision_at_5": 0.12117, "recall_at_1": 0.18219, "recall_at_10": 0.4428, "recall_at_100": 0.76261, "recall_at_1000": 1.0, "recall_at_20": 0.53298, "recall_at_3": 0.30131, "recall_at_5": 0.35681 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/model_meta.json ================================================ {"name": "intfloat/multilingual-e5-large", "revision": "ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb", "release_date": "2024-02-08", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": 560000000, "memory_usage": null, "max_tokens": 514, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/intfloat/multilingual-e5-large", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "sentence_transformers_loader"} ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 9.748866319656372, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.77996, "map_at_1": 0.63158, "map_at_10": 0.73422, "map_at_100": 0.73798, "map_at_1000": 0.73798, "map_at_20": 0.73664, "map_at_3": 0.71491, "map_at_5": 0.72456, "mrr_at_1": 0.631578947368421, "mrr_at_10": 0.7342209690893903, "mrr_at_100": 0.7379801603806115, "mrr_at_1000": 0.7379801603806115, "mrr_at_20": 0.7366366532052129, "mrr_at_3": 0.7149122807017544, "mrr_at_5": 0.7245614035087722, "nauc_map_at_1000_diff1": 0.639926260318653, "nauc_map_at_1000_max": 0.0690961457298233, "nauc_map_at_1000_std": -0.4479754754387847, "nauc_map_at_100_diff1": 0.639926260318653, "nauc_map_at_100_max": 0.0690961457298233, "nauc_map_at_100_std": -0.4479754754387847, "nauc_map_at_10_diff1": 0.6413402133513847, "nauc_map_at_10_max": 0.07409359198407652, "nauc_map_at_10_std": -0.4506537175901882, "nauc_map_at_1_diff1": 0.6755296859994486, "nauc_map_at_1_max": 0.024150951562940618, "nauc_map_at_1_std": -0.37759464704889256, "nauc_map_at_20_diff1": 0.6402194895134982, "nauc_map_at_20_max": 0.07059215552277907, "nauc_map_at_20_std": -0.4524287048567066, "nauc_map_at_3_diff1": 0.6077115138203844, "nauc_map_at_3_max": 0.05784884520381813, "nauc_map_at_3_std": -0.43069080633151685, "nauc_map_at_5_diff1": 0.6327767915615091, "nauc_map_at_5_max": 0.045230434391050546, "nauc_map_at_5_std": -0.47052118506215285, "nauc_mrr_at_1000_diff1": 0.639926260318653, "nauc_mrr_at_1000_max": 0.0690961457298233, "nauc_mrr_at_1000_std": -0.4479754754387847, "nauc_mrr_at_100_diff1": 0.639926260318653, "nauc_mrr_at_100_max": 0.0690961457298233, "nauc_mrr_at_100_std": -0.4479754754387847, "nauc_mrr_at_10_diff1": 0.6413402133513847, "nauc_mrr_at_10_max": 0.07409359198407652, "nauc_mrr_at_10_std": -0.4506537175901882, "nauc_mrr_at_1_diff1": 0.6755296859994486, "nauc_mrr_at_1_max": 0.024150951562940618, "nauc_mrr_at_1_std": -0.37759464704889256, "nauc_mrr_at_20_diff1": 0.6402194895134982, "nauc_mrr_at_20_max": 0.07059215552277907, "nauc_mrr_at_20_std": -0.4524287048567066, "nauc_mrr_at_3_diff1": 0.6077115138203844, "nauc_mrr_at_3_max": 0.05784884520381813, "nauc_mrr_at_3_std": -0.43069080633151685, "nauc_mrr_at_5_diff1": 0.6327767915615091, "nauc_mrr_at_5_max": 0.045230434391050546, "nauc_mrr_at_5_std": -0.47052118506215285, "nauc_ndcg_at_1000_diff1": 0.6386861983810836, "nauc_ndcg_at_1000_max": 0.07844222424411847, "nauc_ndcg_at_1000_std": -0.4626464976834516, "nauc_ndcg_at_100_diff1": 0.6386861983810836, "nauc_ndcg_at_100_max": 0.07844222424411847, "nauc_ndcg_at_100_std": -0.4626464976834516, "nauc_ndcg_at_10_diff1": 0.6468051138280505, "nauc_ndcg_at_10_max": 0.10832580287969389, "nauc_ndcg_at_10_std": -0.48937782027332816, "nauc_ndcg_at_1_diff1": 0.6755296859994486, "nauc_ndcg_at_1_max": 0.024150951562940618, "nauc_ndcg_at_1_std": -0.37759464704889256, "nauc_ndcg_at_20_diff1": 0.6410441362879792, "nauc_ndcg_at_20_max": 0.09238377122473308, "nauc_ndcg_at_20_std": -0.4955340508129813, "nauc_ndcg_at_3_diff1": 0.5783362178814786, "nauc_ndcg_at_3_max": 0.06494448464990901, "nauc_ndcg_at_3_std": -0.44777161440202856, "nauc_ndcg_at_5_diff1": 0.6242306769679065, "nauc_ndcg_at_5_max": 0.03631279046215516, "nauc_ndcg_at_5_std": -0.5313356819910321, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.7260630395460898, "nauc_precision_at_10_max": 0.4180512866096978, "nauc_precision_at_10_std": -0.8142213143329762, "nauc_precision_at_1_diff1": 0.6755296859994486, "nauc_precision_at_1_max": 0.024150951562940618, "nauc_precision_at_1_std": -0.37759464704889256, "nauc_precision_at_20_diff1": 0.6874924403711331, "nauc_precision_at_20_max": 0.39049602606361516, "nauc_precision_at_20_std": -1.142232887868855, "nauc_precision_at_3_diff1": 0.45951758918561936, "nauc_precision_at_3_max": 0.09068385936815057, "nauc_precision_at_3_std": -0.5145244386038328, "nauc_precision_at_5_diff1": 0.5894327798332507, "nauc_precision_at_5_max": -0.022828393192234623, "nauc_precision_at_5_std": -0.8547226023738643, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.726063039546092, "nauc_recall_at_10_max": 0.418051286609699, "nauc_recall_at_10_std": -0.8142213143329806, "nauc_recall_at_1_diff1": 0.6755296859994486, "nauc_recall_at_1_max": 0.024150951562940618, "nauc_recall_at_1_std": -0.37759464704889256, "nauc_recall_at_20_diff1": 0.6874924403711355, "nauc_recall_at_20_max": 0.3904960260636177, "nauc_recall_at_20_std": -1.1422328878688501, "nauc_recall_at_3_diff1": 0.4595175891856197, "nauc_recall_at_3_max": 0.09068385936815089, "nauc_recall_at_3_std": -0.514524438603832, "nauc_recall_at_5_diff1": 0.5894327798332518, "nauc_recall_at_5_max": -0.022828393192234387, "nauc_recall_at_5_std": -0.8547226023738652, "ndcg_at_1": 0.63158, "ndcg_at_10": 0.77996, "ndcg_at_100": 0.7974, "ndcg_at_1000": 0.7974, "ndcg_at_20": 0.78881, "ndcg_at_3": 0.74091, "ndcg_at_5": 0.75865, "precision_at_1": 0.63158, "precision_at_10": 0.09211, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04781, "precision_at_3": 0.27193, "precision_at_5": 0.17193, "recall_at_1": 0.63158, "recall_at_10": 0.92105, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.95614, "recall_at_3": 0.81579, "recall_at_5": 0.85965 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 16.148106575012207, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.936, "map_at_1": 0.89222, "map_at_10": 0.92215, "map_at_100": 0.9229, "map_at_1000": 0.92291, "map_at_20": 0.92262, "map_at_3": 0.91574, "map_at_5": 0.92019, "mrr_at_1": 0.8922222222222222, "mrr_at_10": 0.9221534391534393, "mrr_at_100": 0.9228956584633163, "mrr_at_1000": 0.9229125282281675, "mrr_at_20": 0.9226224118302292, "mrr_at_3": 0.9157407407407407, "mrr_at_5": 0.9201851851851852, "nauc_map_at_1000_diff1": 0.9070917036519953, "nauc_map_at_1000_max": 0.78573770292274, "nauc_map_at_1000_std": 0.15485444735095963, "nauc_map_at_100_diff1": 0.9071028250822603, "nauc_map_at_100_max": 0.7857848927895452, "nauc_map_at_100_std": 0.15503091049967274, "nauc_map_at_10_diff1": 0.9077458924057623, "nauc_map_at_10_max": 0.7871315337230818, "nauc_map_at_10_std": 0.15641366666149223, "nauc_map_at_1_diff1": 0.9075378740027407, "nauc_map_at_1_max": 0.7559884167257802, "nauc_map_at_1_std": 0.11310212133285405, "nauc_map_at_20_diff1": 0.9071830579397084, "nauc_map_at_20_max": 0.7863069884342018, "nauc_map_at_20_std": 0.15618718566694126, "nauc_map_at_3_diff1": 0.9053657881969174, "nauc_map_at_3_max": 0.7903874144498463, "nauc_map_at_3_std": 0.1630929154666399, "nauc_map_at_5_diff1": 0.9067110567499744, "nauc_map_at_5_max": 0.7877141287356025, "nauc_map_at_5_std": 0.15697911965481995, "nauc_mrr_at_1000_diff1": 0.9070917036519953, "nauc_mrr_at_1000_max": 0.78573770292274, "nauc_mrr_at_1000_std": 0.15485444735095963, "nauc_mrr_at_100_diff1": 0.9071028250822603, "nauc_mrr_at_100_max": 0.7857848927895452, "nauc_mrr_at_100_std": 0.15503091049967274, "nauc_mrr_at_10_diff1": 0.9077458924057623, "nauc_mrr_at_10_max": 0.7871315337230818, "nauc_mrr_at_10_std": 0.15641366666149223, "nauc_mrr_at_1_diff1": 0.9075378740027407, "nauc_mrr_at_1_max": 0.7559884167257802, "nauc_mrr_at_1_std": 0.11310212133285405, "nauc_mrr_at_20_diff1": 0.9071830579397084, "nauc_mrr_at_20_max": 0.7863069884342018, "nauc_mrr_at_20_std": 0.15618718566694126, "nauc_mrr_at_3_diff1": 0.9053657881969174, "nauc_mrr_at_3_max": 0.7903874144498463, "nauc_mrr_at_3_std": 0.1630929154666399, "nauc_mrr_at_5_diff1": 0.9067110567499744, "nauc_mrr_at_5_max": 0.7877141287356025, "nauc_mrr_at_5_std": 0.15697911965481995, "nauc_ndcg_at_1000_diff1": 0.9079724895902852, "nauc_ndcg_at_1000_max": 0.7909080294027776, "nauc_ndcg_at_1000_std": 0.1616801687839343, "nauc_ndcg_at_100_diff1": 0.9083275859079337, "nauc_ndcg_at_100_max": 0.7924334487329852, "nauc_ndcg_at_100_std": 0.167477710657392, "nauc_ndcg_at_10_diff1": 0.9115597785525953, "nauc_ndcg_at_10_max": 0.8000791539278294, "nauc_ndcg_at_10_std": 0.17191138233130876, "nauc_ndcg_at_1_diff1": 0.9075378740027407, "nauc_ndcg_at_1_max": 0.7559884167257802, "nauc_ndcg_at_1_std": 0.11310212133285405, "nauc_ndcg_at_20_diff1": 0.9088692422365905, "nauc_ndcg_at_20_max": 0.796455678631563, "nauc_ndcg_at_20_std": 0.17284696896394636, "nauc_ndcg_at_3_diff1": 0.9058569690070104, "nauc_ndcg_at_3_max": 0.8046835489465721, "nauc_ndcg_at_3_std": 0.1804182130630102, "nauc_ndcg_at_5_diff1": 0.9086374936067283, "nauc_ndcg_at_5_max": 0.8012419491051281, "nauc_ndcg_at_5_std": 0.17187022513100597, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9564270152505041, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.9564270152505041, "nauc_precision_at_10_diff1": 0.9578603371172977, "nauc_precision_at_10_max": 0.9324291119956715, "nauc_precision_at_10_std": 0.33301390731730346, "nauc_precision_at_1_diff1": 0.9075378740027407, "nauc_precision_at_1_max": 0.7559884167257802, "nauc_precision_at_1_std": 0.11310212133285405, "nauc_precision_at_20_diff1": 0.9332788671023994, "nauc_precision_at_20_max": 0.941059757236231, "nauc_precision_at_20_std": 0.45992841581075417, "nauc_precision_at_3_diff1": 0.9086777568170122, "nauc_precision_at_3_max": 0.8677185159778196, "nauc_precision_at_3_std": 0.25518778940147957, "nauc_precision_at_5_diff1": 0.9227503501400542, "nauc_precision_at_5_max": 0.8872403127917805, "nauc_precision_at_5_std": 0.2647788281979385, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9564270152505339, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.9564270152505339, "nauc_recall_at_10_diff1": 0.9578603371173053, "nauc_recall_at_10_max": 0.9324291119956791, "nauc_recall_at_10_std": 0.3330139073173117, "nauc_recall_at_1_diff1": 0.9075378740027407, "nauc_recall_at_1_max": 0.7559884167257802, "nauc_recall_at_1_std": 0.11310212133285405, "nauc_recall_at_20_diff1": 0.9332788671023909, "nauc_recall_at_20_max": 0.9410597572362199, "nauc_recall_at_20_std": 0.45992841581076965, "nauc_recall_at_3_diff1": 0.908677756817013, "nauc_recall_at_3_max": 0.8677185159778226, "nauc_recall_at_3_std": 0.2551877894014712, "nauc_recall_at_5_diff1": 0.9227503501400555, "nauc_recall_at_5_max": 0.8872403127917826, "nauc_recall_at_5_std": 0.2647788281979452, "ndcg_at_1": 0.89222, "ndcg_at_10": 0.936, "ndcg_at_100": 0.93977, "ndcg_at_1000": 0.94021, "ndcg_at_20": 0.93788, "ndcg_at_3": 0.9234, "ndcg_at_5": 0.93129, "precision_at_1": 0.89222, "precision_at_10": 0.09789, "precision_at_100": 0.00997, "precision_at_1000": 0.001, "precision_at_20": 0.04933, "precision_at_3": 0.31519, "precision_at_5": 0.19289, "recall_at_1": 0.89222, "recall_at_10": 0.97889, "recall_at_100": 0.99667, "recall_at_1000": 1.0, "recall_at_20": 0.98667, "recall_at_3": 0.94556, "recall_at_5": 0.96444 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.92482, "map_at_1": 0.86333, "map_at_10": 0.9074, "map_at_100": 0.90855, "map_at_1000": 0.90856, "map_at_20": 0.9084, "map_at_3": 0.90074, "map_at_5": 0.90502, "mrr_at_1": 0.8633333333333333, "mrr_at_10": 0.9073955026455032, "mrr_at_100": 0.9085521299329449, "mrr_at_1000": 0.9085648476044211, "mrr_at_20": 0.908397048370733, "mrr_at_3": 0.9007407407407411, "mrr_at_5": 0.9050185185185189, "nauc_map_at_1000_diff1": 0.8995815876496662, "nauc_map_at_1000_max": 0.6350214249075169, "nauc_map_at_1000_std": 0.018349457469257657, "nauc_map_at_100_diff1": 0.8995914231410215, "nauc_map_at_100_max": 0.6350594488184639, "nauc_map_at_100_std": 0.01847406737095244, "nauc_map_at_10_diff1": 0.8984256257105865, "nauc_map_at_10_max": 0.6365719801653463, "nauc_map_at_10_std": 0.019304767126614965, "nauc_map_at_1_diff1": 0.903735259178841, "nauc_map_at_1_max": 0.6099140135155323, "nauc_map_at_1_std": 0.00646668712090879, "nauc_map_at_20_diff1": 0.8996288037046547, "nauc_map_at_20_max": 0.6350660515812196, "nauc_map_at_20_std": 0.0186524974702766, "nauc_map_at_3_diff1": 0.8988737870316825, "nauc_map_at_3_max": 0.6374670650986446, "nauc_map_at_3_std": 0.02077754928632171, "nauc_map_at_5_diff1": 0.8984580242742688, "nauc_map_at_5_max": 0.6373387612313168, "nauc_map_at_5_std": 0.016254466520959136, "nauc_mrr_at_1000_diff1": 0.8995815876496662, "nauc_mrr_at_1000_max": 0.6350214249075169, "nauc_mrr_at_1000_std": 0.018349457469257657, "nauc_mrr_at_100_diff1": 0.8995914231410215, "nauc_mrr_at_100_max": 0.6350594488184639, "nauc_mrr_at_100_std": 0.01847406737095244, "nauc_mrr_at_10_diff1": 0.8984256257105865, "nauc_mrr_at_10_max": 0.6365719801653463, "nauc_mrr_at_10_std": 0.019304767126614965, "nauc_mrr_at_1_diff1": 0.903735259178841, "nauc_mrr_at_1_max": 0.6099140135155323, "nauc_mrr_at_1_std": 0.00646668712090879, "nauc_mrr_at_20_diff1": 0.8996288037046547, "nauc_mrr_at_20_max": 0.6350660515812196, "nauc_mrr_at_20_std": 0.0186524974702766, "nauc_mrr_at_3_diff1": 0.8988737870316825, "nauc_mrr_at_3_max": 0.6374670650986446, "nauc_mrr_at_3_std": 0.02077754928632171, "nauc_mrr_at_5_diff1": 0.8984580242742688, "nauc_mrr_at_5_max": 0.6373387612313168, "nauc_mrr_at_5_std": 0.016254466520959136, "nauc_ndcg_at_1000_diff1": 0.8993256215537422, "nauc_ndcg_at_1000_max": 0.6393593719942897, "nauc_ndcg_at_1000_std": 0.022454698103965824, "nauc_ndcg_at_100_diff1": 0.8996789854457844, "nauc_ndcg_at_100_max": 0.6409812520717987, "nauc_ndcg_at_100_std": 0.027445863539626563, "nauc_ndcg_at_10_diff1": 0.8956806520742397, "nauc_ndcg_at_10_max": 0.6479653772824047, "nauc_ndcg_at_10_std": 0.029215214302291608, "nauc_ndcg_at_1_diff1": 0.903735259178841, "nauc_ndcg_at_1_max": 0.6099140135155323, "nauc_ndcg_at_1_std": 0.00646668712090879, "nauc_ndcg_at_20_diff1": 0.9000089526843404, "nauc_ndcg_at_20_max": 0.6413993358261516, "nauc_ndcg_at_20_std": 0.026218032748382948, "nauc_ndcg_at_3_diff1": 0.89658797647391, "nauc_ndcg_at_3_max": 0.6481676790604936, "nauc_ndcg_at_3_std": 0.028589359037325752, "nauc_ndcg_at_5_diff1": 0.8958100284140829, "nauc_ndcg_at_5_max": 0.6491416873953931, "nauc_ndcg_at_5_std": 0.019473810166820073, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9564270152505041, "nauc_precision_at_100_max": 0.9128540305010695, "nauc_precision_at_100_std": 0.8513849984438455, "nauc_precision_at_10_diff1": 0.86440242763772, "nauc_precision_at_10_max": 0.7783146591970098, "nauc_precision_at_10_std": 0.15574229691876484, "nauc_precision_at_1_diff1": 0.903735259178841, "nauc_precision_at_1_max": 0.6099140135155323, "nauc_precision_at_1_std": 0.00646668712090879, "nauc_precision_at_20_diff1": 0.9346405228758121, "nauc_precision_at_20_max": 0.7672152194210989, "nauc_precision_at_20_std": 0.2435807656395918, "nauc_precision_at_3_diff1": 0.8853554493692909, "nauc_precision_at_3_max": 0.6996393328573267, "nauc_precision_at_3_std": 0.06751066439647656, "nauc_precision_at_5_diff1": 0.8778978257969816, "nauc_precision_at_5_max": 0.7289048952914466, "nauc_precision_at_5_std": 0.03888221955448746, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9564270152505339, "nauc_recall_at_100_max": 0.9128540305010676, "nauc_recall_at_100_std": 0.8513849984438123, "nauc_recall_at_10_diff1": 0.8644024276377222, "nauc_recall_at_10_max": 0.7783146591970082, "nauc_recall_at_10_std": 0.15574229691877126, "nauc_recall_at_1_diff1": 0.903735259178841, "nauc_recall_at_1_max": 0.6099140135155323, "nauc_recall_at_1_std": 0.00646668712090879, "nauc_recall_at_20_diff1": 0.9346405228758294, "nauc_recall_at_20_max": 0.7672152194211065, "nauc_recall_at_20_std": 0.24358076563958803, "nauc_recall_at_3_diff1": 0.88535544936929, "nauc_recall_at_3_max": 0.6996393328573252, "nauc_recall_at_3_std": 0.0675106643964764, "nauc_recall_at_5_diff1": 0.877897825796986, "nauc_recall_at_5_max": 0.7289048952914498, "nauc_recall_at_5_std": 0.038882219554492656, "ndcg_at_1": 0.86333, "ndcg_at_10": 0.92482, "ndcg_at_100": 0.92935, "ndcg_at_1000": 0.92976, "ndcg_at_20": 0.92829, "ndcg_at_3": 0.91177, "ndcg_at_5": 0.91928, "precision_at_1": 0.86333, "precision_at_10": 0.09778, "precision_at_100": 0.00997, "precision_at_1000": 0.001, "precision_at_20": 0.04956, "precision_at_3": 0.31444, "precision_at_5": 0.19222, "recall_at_1": 0.86333, "recall_at_10": 0.97778, "recall_at_100": 0.99667, "recall_at_1000": 1.0, "recall_at_20": 0.99111, "recall_at_3": 0.94333, "recall_at_5": 0.96111 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.86177, "map_at_1": 0.77556, "map_at_10": 0.83543, "map_at_100": 0.83726, "map_at_1000": 0.83736, "map_at_20": 0.83663, "map_at_3": 0.82537, "map_at_5": 0.83059, "mrr_at_1": 0.7755555555555556, "mrr_at_10": 0.8354276895943563, "mrr_at_100": 0.8372601744078978, "mrr_at_1000": 0.8373594614287702, "mrr_at_20": 0.8366342586244548, "mrr_at_3": 0.8253703703703703, "mrr_at_5": 0.8305925925925925, "nauc_map_at_1000_diff1": 0.8563262993363897, "nauc_map_at_1000_max": 0.7092525023180286, "nauc_map_at_1000_std": 0.11447648643872059, "nauc_map_at_100_diff1": 0.8562847147884489, "nauc_map_at_100_max": 0.7093705485013881, "nauc_map_at_100_std": 0.11480965523119875, "nauc_map_at_10_diff1": 0.8557451506594478, "nauc_map_at_10_max": 0.7103867572729406, "nauc_map_at_10_std": 0.11659574990893444, "nauc_map_at_1_diff1": 0.875583807055888, "nauc_map_at_1_max": 0.6760802649127515, "nauc_map_at_1_std": 0.039787623543968066, "nauc_map_at_20_diff1": 0.8562644961141803, "nauc_map_at_20_max": 0.7101451002121112, "nauc_map_at_20_std": 0.11702640931466465, "nauc_map_at_3_diff1": 0.852118141777768, "nauc_map_at_3_max": 0.7078386195869544, "nauc_map_at_3_std": 0.10955607768958464, "nauc_map_at_5_diff1": 0.8549593365575148, "nauc_map_at_5_max": 0.7133833504561998, "nauc_map_at_5_std": 0.12167925414056359, "nauc_mrr_at_1000_diff1": 0.8563262993363897, "nauc_mrr_at_1000_max": 0.7092525023180286, "nauc_mrr_at_1000_std": 0.11447648643872059, "nauc_mrr_at_100_diff1": 0.8562847147884489, "nauc_mrr_at_100_max": 0.7093705485013881, "nauc_mrr_at_100_std": 0.11480965523119875, "nauc_mrr_at_10_diff1": 0.8557451506594478, "nauc_mrr_at_10_max": 0.7103867572729406, "nauc_mrr_at_10_std": 0.11659574990893444, "nauc_mrr_at_1_diff1": 0.875583807055888, "nauc_mrr_at_1_max": 0.6760802649127515, "nauc_mrr_at_1_std": 0.039787623543968066, "nauc_mrr_at_20_diff1": 0.8562644961141803, "nauc_mrr_at_20_max": 0.7101451002121112, "nauc_mrr_at_20_std": 0.11702640931466465, "nauc_mrr_at_3_diff1": 0.852118141777768, "nauc_mrr_at_3_max": 0.7078386195869544, "nauc_mrr_at_3_std": 0.10955607768958464, "nauc_mrr_at_5_diff1": 0.8549593365575148, "nauc_mrr_at_5_max": 0.7133833504561998, "nauc_mrr_at_5_std": 0.12167925414056359, "nauc_ndcg_at_1000_diff1": 0.8536060198678865, "nauc_ndcg_at_1000_max": 0.7161415325431293, "nauc_ndcg_at_1000_std": 0.13118419697584274, "nauc_ndcg_at_100_diff1": 0.8524424740549038, "nauc_ndcg_at_100_max": 0.719221257766969, "nauc_ndcg_at_100_std": 0.14010098262195667, "nauc_ndcg_at_10_diff1": 0.8500411904861312, "nauc_ndcg_at_10_max": 0.7257669827316664, "nauc_ndcg_at_10_std": 0.1531268391377476, "nauc_ndcg_at_1_diff1": 0.875583807055888, "nauc_ndcg_at_1_max": 0.6760802649127515, "nauc_ndcg_at_1_std": 0.039787623543968066, "nauc_ndcg_at_20_diff1": 0.8521602697274657, "nauc_ndcg_at_20_max": 0.7255292199942737, "nauc_ndcg_at_20_std": 0.15622394129449554, "nauc_ndcg_at_3_diff1": 0.8430930583173746, "nauc_ndcg_at_3_max": 0.7200938991115078, "nauc_ndcg_at_3_std": 0.1371129407859106, "nauc_ndcg_at_5_diff1": 0.8485407560921437, "nauc_ndcg_at_5_max": 0.7320543787124469, "nauc_ndcg_at_5_std": 0.16299385867300736, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.7805788982259647, "nauc_precision_at_100_max": 0.9069884364002117, "nauc_precision_at_100_std": 0.6863463334051753, "nauc_precision_at_10_diff1": 0.8101462807345137, "nauc_precision_at_10_max": 0.8443089654162306, "nauc_precision_at_10_std": 0.4376979549989915, "nauc_precision_at_1_diff1": 0.875583807055888, "nauc_precision_at_1_max": 0.6760802649127515, "nauc_precision_at_1_std": 0.039787623543968066, "nauc_precision_at_20_diff1": 0.819937234152921, "nauc_precision_at_20_max": 0.8925718435522375, "nauc_precision_at_20_std": 0.59440813362382, "nauc_precision_at_3_diff1": 0.8058400081111246, "nauc_precision_at_3_max": 0.7706994921331332, "nauc_precision_at_3_std": 0.25120053828357747, "nauc_precision_at_5_diff1": 0.8185249709639927, "nauc_precision_at_5_max": 0.8277424791054608, "nauc_precision_at_5_std": 0.3745872332672881, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.7805788982259576, "nauc_recall_at_100_max": 0.9069884364002062, "nauc_recall_at_100_std": 0.6863463334051649, "nauc_recall_at_10_diff1": 0.8101462807345169, "nauc_recall_at_10_max": 0.8443089654162333, "nauc_recall_at_10_std": 0.4376979549989933, "nauc_recall_at_1_diff1": 0.875583807055888, "nauc_recall_at_1_max": 0.6760802649127515, "nauc_recall_at_1_std": 0.039787623543968066, "nauc_recall_at_20_diff1": 0.8199372341529212, "nauc_recall_at_20_max": 0.892571843552236, "nauc_recall_at_20_std": 0.5944081336238212, "nauc_recall_at_3_diff1": 0.8058400081111227, "nauc_recall_at_3_max": 0.7706994921331336, "nauc_recall_at_3_std": 0.25120053828357475, "nauc_recall_at_5_diff1": 0.8185249709639958, "nauc_recall_at_5_max": 0.8277424791054628, "nauc_recall_at_5_std": 0.37458723326729154, "ndcg_at_1": 0.77556, "ndcg_at_10": 0.86177, "ndcg_at_100": 0.87073, "ndcg_at_1000": 0.87273, "ndcg_at_20": 0.86605, "ndcg_at_3": 0.84088, "ndcg_at_5": 0.8504, "precision_at_1": 0.77556, "precision_at_10": 0.09433, "precision_at_100": 0.00986, "precision_at_1000": 0.001, "precision_at_20": 0.048, "precision_at_3": 0.29519, "precision_at_5": 0.18178, "recall_at_1": 0.77556, "recall_at_10": 0.94333, "recall_at_100": 0.98556, "recall_at_1000": 1.0, "recall_at_20": 0.96, "recall_at_3": 0.88556, "recall_at_5": 0.90889 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 115.93965744972229, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.79793, "map_at_1": 0.49126, "map_at_10": 0.75434, "map_at_100": 0.75957, "map_at_1000": 0.75987, "map_at_20": 0.75755, "map_at_3": 0.71057, "map_at_5": 0.74279, "mrr_at_1": 0.7516891891891891, "mrr_at_10": 0.8103858322608322, "mrr_at_100": 0.8120290293768601, "mrr_at_1000": 0.8121733425939148, "mrr_at_20": 0.8110783812989694, "mrr_at_3": 0.804617117117117, "mrr_at_5": 0.8087556306306305, "nauc_map_at_1000_diff1": 0.5309640525572018, "nauc_map_at_1000_max": 0.6034757137873261, "nauc_map_at_1000_std": 0.2524830874626602, "nauc_map_at_100_diff1": 0.5310950999887473, "nauc_map_at_100_max": 0.6038898477176643, "nauc_map_at_100_std": 0.2529213111665448, "nauc_map_at_10_diff1": 0.5287408922081819, "nauc_map_at_10_max": 0.6048810878714043, "nauc_map_at_10_std": 0.24514186826715573, "nauc_map_at_1_diff1": 0.5839831914566329, "nauc_map_at_1_max": 0.24134429916197642, "nauc_map_at_1_std": -0.005362833464650543, "nauc_map_at_20_diff1": 0.531847319431841, "nauc_map_at_20_max": 0.6053733911651847, "nauc_map_at_20_std": 0.25267089208288807, "nauc_map_at_3_diff1": 0.5175569859756408, "nauc_map_at_3_max": 0.5267407209649851, "nauc_map_at_3_std": 0.18034284700192835, "nauc_map_at_5_diff1": 0.5340789493854458, "nauc_map_at_5_max": 0.5841825317705776, "nauc_map_at_5_std": 0.2178539895872495, "nauc_mrr_at_1000_diff1": 0.6129207681904467, "nauc_mrr_at_1000_max": 0.6810699075110801, "nauc_mrr_at_1000_std": 0.33656350406575714, "nauc_mrr_at_100_diff1": 0.6129612079444606, "nauc_mrr_at_100_max": 0.681293581685122, "nauc_mrr_at_100_std": 0.336959957396225, "nauc_mrr_at_10_diff1": 0.6133305000787206, "nauc_mrr_at_10_max": 0.6834511087649037, "nauc_mrr_at_10_std": 0.33747039829177455, "nauc_mrr_at_1_diff1": 0.6254310300227522, "nauc_mrr_at_1_max": 0.5816815042949867, "nauc_mrr_at_1_std": 0.21266545672024523, "nauc_mrr_at_20_diff1": 0.612875409327525, "nauc_mrr_at_20_max": 0.6823877732414835, "nauc_mrr_at_20_std": 0.3375268307975658, "nauc_mrr_at_3_diff1": 0.6176288728475821, "nauc_mrr_at_3_max": 0.6849689448598965, "nauc_mrr_at_3_std": 0.3393152904117805, "nauc_mrr_at_5_diff1": 0.6155674459330454, "nauc_mrr_at_5_max": 0.684849646628972, "nauc_mrr_at_5_std": 0.33727860792942294, "nauc_ndcg_at_1000_diff1": 0.542637450430811, "nauc_ndcg_at_1000_max": 0.659613284461409, "nauc_ndcg_at_1000_std": 0.33200269100657104, "nauc_ndcg_at_100_diff1": 0.547672333085176, "nauc_ndcg_at_100_max": 0.6700030728702414, "nauc_ndcg_at_100_std": 0.3469653351483253, "nauc_ndcg_at_10_diff1": 0.5436935978103205, "nauc_ndcg_at_10_max": 0.6788066553801092, "nauc_ndcg_at_10_std": 0.32641262078820005, "nauc_ndcg_at_1_diff1": 0.6254310300227522, "nauc_ndcg_at_1_max": 0.5816815042949867, "nauc_ndcg_at_1_std": 0.21266545672024523, "nauc_ndcg_at_20_diff1": 0.5487238316054067, "nauc_ndcg_at_20_max": 0.6788721621002779, "nauc_ndcg_at_20_std": 0.3455664306916214, "nauc_ndcg_at_3_diff1": 0.5290660234984015, "nauc_ndcg_at_3_max": 0.5808423803809595, "nauc_ndcg_at_3_std": 0.23792804741859436, "nauc_ndcg_at_5_diff1": 0.54973349540624, "nauc_ndcg_at_5_max": 0.6465132469014665, "nauc_ndcg_at_5_std": 0.2816044773825405, "nauc_precision_at_1000_diff1": -0.3624723519954685, "nauc_precision_at_1000_max": 0.05033883591861056, "nauc_precision_at_1000_std": 0.1988482841500978, "nauc_precision_at_100_diff1": -0.249229073110791, "nauc_precision_at_100_max": 0.18182050204914146, "nauc_precision_at_100_std": 0.29041396174885464, "nauc_precision_at_10_diff1": -0.150896797806953, "nauc_precision_at_10_max": 0.3288052265753716, "nauc_precision_at_10_std": 0.3191898156600829, "nauc_precision_at_1_diff1": 0.6254310300227522, "nauc_precision_at_1_max": 0.5816815042949867, "nauc_precision_at_1_std": 0.21266545672024523, "nauc_precision_at_20_diff1": -0.17002101345912066, "nauc_precision_at_20_max": 0.2862629611332116, "nauc_precision_at_20_std": 0.3255332095997299, "nauc_precision_at_3_diff1": -0.028585511987222784, "nauc_precision_at_3_max": 0.41995825306109386, "nauc_precision_at_3_std": 0.30634819262268315, "nauc_precision_at_5_diff1": -0.08964145017856276, "nauc_precision_at_5_max": 0.3681815785470059, "nauc_precision_at_5_std": 0.2954266706591043, "nauc_recall_at_1000_diff1": 0.275303091358477, "nauc_recall_at_1000_max": 0.876861475892877, "nauc_recall_at_1000_std": 0.907974334424692, "nauc_recall_at_100_diff1": 0.5072766102427224, "nauc_recall_at_100_max": 0.8821544361345515, "nauc_recall_at_100_std": 0.7751863280665128, "nauc_recall_at_10_diff1": 0.503098333242494, "nauc_recall_at_10_max": 0.8343898880673456, "nauc_recall_at_10_std": 0.49474065180833804, "nauc_recall_at_1_diff1": 0.5839831914566329, "nauc_recall_at_1_max": 0.24134429916197642, "nauc_recall_at_1_std": -0.005362833464650543, "nauc_recall_at_20_diff1": 0.5195860953935559, "nauc_recall_at_20_max": 0.860889980047164, "nauc_recall_at_20_std": 0.6144360815190124, "nauc_recall_at_3_diff1": 0.4913567468163894, "nauc_recall_at_3_max": 0.5795842824553967, "nauc_recall_at_3_std": 0.24546706094062273, "nauc_recall_at_5_diff1": 0.5209857740942301, "nauc_recall_at_5_max": 0.7195409453404005, "nauc_recall_at_5_std": 0.34618966424185427, "ndcg_at_1": 0.75169, "ndcg_at_10": 0.79793, "ndcg_at_100": 0.81554, "ndcg_at_1000": 0.82268, "ndcg_at_20": 0.80494, "ndcg_at_3": 0.75097, "ndcg_at_5": 0.7824, "precision_at_1": 0.75169, "precision_at_10": 0.15895, "precision_at_100": 0.01743, "precision_at_1000": 0.00185, "precision_at_20": 0.08201, "precision_at_3": 0.44538, "precision_at_5": 0.29865, "recall_at_1": 0.49126, "recall_at_10": 0.85782, "recall_at_100": 0.9252, "recall_at_1000": 0.97095, "recall_at_20": 0.87913, "recall_at_3": 0.75884, "recall_at_5": 0.8227 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 15485.132284641266, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.59914, "map_at_1": 0.32696, "map_at_10": 0.5021, "map_at_100": 0.52536, "map_at_1000": 0.52675, "map_at_20": 0.51557, "map_at_3": 0.44841, "map_at_5": 0.48126, "mrr_at_1": 0.5539906103286385, "mrr_at_10": 0.6759538713764065, "mrr_at_100": 0.6785929910661365, "mrr_at_1000": 0.6788143429529052, "mrr_at_20": 0.6780745834211364, "mrr_at_3": 0.6533646322378714, "mrr_at_5": 0.6683881064162753, "nauc_map_at_1000_diff1": 0.37998530220717786, "nauc_map_at_1000_max": 0.35683758932659904, "nauc_map_at_1000_std": 0.09661911895952302, "nauc_map_at_100_diff1": 0.3801636369285897, "nauc_map_at_100_max": 0.3557813147855278, "nauc_map_at_100_std": 0.09699912793782361, "nauc_map_at_10_diff1": 0.385840730245925, "nauc_map_at_10_max": 0.3388006342199373, "nauc_map_at_10_std": 0.07469794277413962, "nauc_map_at_1_diff1": 0.4710151650817007, "nauc_map_at_1_max": 0.27386484996995075, "nauc_map_at_1_std": -0.03826587437569447, "nauc_map_at_20_diff1": 0.38004667201099374, "nauc_map_at_20_max": 0.34736178062506173, "nauc_map_at_20_std": 0.08691126593765994, "nauc_map_at_3_diff1": 0.3959381990501782, "nauc_map_at_3_max": 0.31783964379609697, "nauc_map_at_3_std": 0.021876203870068035, "nauc_map_at_5_diff1": 0.4013815389612924, "nauc_map_at_5_max": 0.32284760433197945, "nauc_map_at_5_std": 0.04644907393464365, "nauc_mrr_at_1000_diff1": 0.35046511606892405, "nauc_mrr_at_1000_max": 0.45610342631191003, "nauc_mrr_at_1000_std": 0.21587784888890976, "nauc_mrr_at_100_diff1": 0.35057490500672683, "nauc_mrr_at_100_max": 0.4564646189984297, "nauc_mrr_at_100_std": 0.21625860865160604, "nauc_mrr_at_10_diff1": 0.3496031909413219, "nauc_mrr_at_10_max": 0.45310738338234763, "nauc_mrr_at_10_std": 0.2168338658348742, "nauc_mrr_at_1_diff1": 0.36316059808304396, "nauc_mrr_at_1_max": 0.42616112853710764, "nauc_mrr_at_1_std": 0.14727968392145732, "nauc_mrr_at_20_diff1": 0.34931996988824954, "nauc_mrr_at_20_max": 0.4564065340592655, "nauc_mrr_at_20_std": 0.21678796014048907, "nauc_mrr_at_3_diff1": 0.35144112092691926, "nauc_mrr_at_3_max": 0.46118986626841046, "nauc_mrr_at_3_std": 0.2187047520665188, "nauc_mrr_at_5_diff1": 0.3513761270515314, "nauc_mrr_at_5_max": 0.45557184060456873, "nauc_mrr_at_5_std": 0.22411204059806153, "nauc_ndcg_at_1000_diff1": 0.3622965432026116, "nauc_ndcg_at_1000_max": 0.4132309696262614, "nauc_ndcg_at_1000_std": 0.18060337350151817, "nauc_ndcg_at_100_diff1": 0.36621225660226114, "nauc_ndcg_at_100_max": 0.41503010576140476, "nauc_ndcg_at_100_std": 0.19534523527606712, "nauc_ndcg_at_10_diff1": 0.3577638445537456, "nauc_ndcg_at_10_max": 0.35826634923716805, "nauc_ndcg_at_10_std": 0.1341988805236967, "nauc_ndcg_at_1_diff1": 0.36316059808304396, "nauc_ndcg_at_1_max": 0.42616112853710764, "nauc_ndcg_at_1_std": 0.14727968392145732, "nauc_ndcg_at_20_diff1": 0.3515739876801084, "nauc_ndcg_at_20_max": 0.3812795972772258, "nauc_ndcg_at_20_std": 0.1566067661903798, "nauc_ndcg_at_3_diff1": 0.3430068358868766, "nauc_ndcg_at_3_max": 0.39775892927028483, "nauc_ndcg_at_3_std": 0.13676734265757323, "nauc_ndcg_at_5_diff1": 0.36733201434120677, "nauc_ndcg_at_5_max": 0.36933935830633907, "nauc_ndcg_at_5_std": 0.12597735688319697, "nauc_precision_at_1000_diff1": -0.19002674680239415, "nauc_precision_at_1000_max": 0.174509429358778, "nauc_precision_at_1000_std": 0.28167232364321076, "nauc_precision_at_100_diff1": -0.17309643394949184, "nauc_precision_at_100_max": 0.20656825090247774, "nauc_precision_at_100_std": 0.3512037726617336, "nauc_precision_at_10_diff1": -0.16528090511957233, "nauc_precision_at_10_max": 0.2619352359421429, "nauc_precision_at_10_std": 0.3941957192775191, "nauc_precision_at_1_diff1": 0.36316059808304396, "nauc_precision_at_1_max": 0.42616112853710764, "nauc_precision_at_1_std": 0.14727968392145732, "nauc_precision_at_20_diff1": -0.1898576802938022, "nauc_precision_at_20_max": 0.23701631812155186, "nauc_precision_at_20_std": 0.3740218709201864, "nauc_precision_at_3_diff1": -0.00568419245193641, "nauc_precision_at_3_max": 0.3415257396991487, "nauc_precision_at_3_std": 0.31916275229293856, "nauc_precision_at_5_diff1": -0.08655168669277556, "nauc_precision_at_5_max": 0.2761786653681346, "nauc_precision_at_5_std": 0.35947192558173247, "nauc_recall_at_1000_diff1": 0.1827229283366921, "nauc_recall_at_1000_max": 0.537327498086783, "nauc_recall_at_1000_std": 0.6550227733454026, "nauc_recall_at_100_diff1": 0.33523387191444975, "nauc_recall_at_100_max": 0.4826282458217185, "nauc_recall_at_100_std": 0.4726203131645689, "nauc_recall_at_10_diff1": 0.2385680261998446, "nauc_recall_at_10_max": 0.20787374315415943, "nauc_recall_at_10_std": 0.10710738932195062, "nauc_recall_at_1_diff1": 0.4710151650817007, "nauc_recall_at_1_max": 0.27386484996995075, "nauc_recall_at_1_std": -0.03826587437569447, "nauc_recall_at_20_diff1": 0.2070919483692747, "nauc_recall_at_20_max": 0.26472298029628866, "nauc_recall_at_20_std": 0.16965356881445673, "nauc_recall_at_3_diff1": 0.32594635843508896, "nauc_recall_at_3_max": 0.2711469727674023, "nauc_recall_at_3_std": 0.040071374394099774, "nauc_recall_at_5_diff1": 0.3328227607824658, "nauc_recall_at_5_max": 0.2458408375382006, "nauc_recall_at_5_std": 0.0837219650224051, "ndcg_at_1": 0.55399, "ndcg_at_10": 0.59914, "ndcg_at_100": 0.6601, "ndcg_at_1000": 0.67563, "ndcg_at_20": 0.62687, "ndcg_at_3": 0.56408, "ndcg_at_5": 0.57624, "precision_at_1": 0.55399, "precision_at_10": 0.1507, "precision_at_100": 0.02258, "precision_at_1000": 0.00254, "precision_at_20": 0.08873, "precision_at_3": 0.34585, "precision_at_5": 0.2507, "recall_at_1": 0.32696, "recall_at_10": 0.70681, "recall_at_100": 0.89928, "recall_at_1000": 0.98818, "recall_at_20": 0.78447, "recall_at_3": 0.53559, "recall_at_5": 0.62288 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 15758.073652267456, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.52877, "map_at_1": 0.34917, "map_at_10": 0.46706, "map_at_100": 0.47604, "map_at_1000": 0.47636, "map_at_20": 0.47197, "map_at_3": 0.4391, "map_at_5": 0.45143, "mrr_at_1": 0.37767220902612825, "mrr_at_10": 0.49347830185122343, "mrr_at_100": 0.5003179443111394, "mrr_at_1000": 0.5005908363684997, "mrr_at_20": 0.49746038430543127, "mrr_at_3": 0.46991290577988926, "mrr_at_5": 0.48131433095803655, "nauc_map_at_1000_diff1": 0.35159452922876405, "nauc_map_at_1000_max": 0.2933510225495975, "nauc_map_at_1000_std": 0.15655234071328542, "nauc_map_at_100_diff1": 0.35135462350016194, "nauc_map_at_100_max": 0.2931871700688513, "nauc_map_at_100_std": 0.15653088767109508, "nauc_map_at_10_diff1": 0.35298293783124035, "nauc_map_at_10_max": 0.2946430155110729, "nauc_map_at_10_std": 0.1554238628320712, "nauc_map_at_1_diff1": 0.39952097513531726, "nauc_map_at_1_max": 0.28310443294341225, "nauc_map_at_1_std": 0.11174270301882441, "nauc_map_at_20_diff1": 0.35065372123850846, "nauc_map_at_20_max": 0.29286040418768033, "nauc_map_at_20_std": 0.15621968993533264, "nauc_map_at_3_diff1": 0.35035525763761394, "nauc_map_at_3_max": 0.2942433675582027, "nauc_map_at_3_std": 0.15340354705473352, "nauc_map_at_5_diff1": 0.3564998458591775, "nauc_map_at_5_max": 0.29450450428040004, "nauc_map_at_5_std": 0.15635827772340125, "nauc_mrr_at_1000_diff1": 0.37081278081751473, "nauc_mrr_at_1000_max": 0.31358131772038333, "nauc_mrr_at_1000_std": 0.173869859041972, "nauc_mrr_at_100_diff1": 0.37048775784895055, "nauc_mrr_at_100_max": 0.3133560463765259, "nauc_mrr_at_100_std": 0.17378936217140808, "nauc_mrr_at_10_diff1": 0.37372919724000586, "nauc_mrr_at_10_max": 0.3168249122844439, "nauc_mrr_at_10_std": 0.1737668241296463, "nauc_mrr_at_1_diff1": 0.4164640284913696, "nauc_mrr_at_1_max": 0.3053029955918928, "nauc_mrr_at_1_std": 0.139614777902049, "nauc_mrr_at_20_diff1": 0.3702719030948478, "nauc_mrr_at_20_max": 0.3139466229592053, "nauc_mrr_at_20_std": 0.17374029768386537, "nauc_mrr_at_3_diff1": 0.3643024059691536, "nauc_mrr_at_3_max": 0.31685793544673635, "nauc_mrr_at_3_std": 0.18021454208347543, "nauc_mrr_at_5_diff1": 0.3743438414059076, "nauc_mrr_at_5_max": 0.3160663712710881, "nauc_mrr_at_5_std": 0.1742876218746978, "nauc_ndcg_at_1000_diff1": 0.3509610363029608, "nauc_ndcg_at_1000_max": 0.3038698080687019, "nauc_ndcg_at_1000_std": 0.17682837434987447, "nauc_ndcg_at_100_diff1": 0.3426920404078895, "nauc_ndcg_at_100_max": 0.29853443276775044, "nauc_ndcg_at_100_std": 0.17719600663478413, "nauc_ndcg_at_10_diff1": 0.3504725576888753, "nauc_ndcg_at_10_max": 0.30796980063919227, "nauc_ndcg_at_10_std": 0.17240955500640734, "nauc_ndcg_at_1_diff1": 0.4164640284913696, "nauc_ndcg_at_1_max": 0.3053029955918928, "nauc_ndcg_at_1_std": 0.139614777902049, "nauc_ndcg_at_20_diff1": 0.3417554990631716, "nauc_ndcg_at_20_max": 0.3011309322625806, "nauc_ndcg_at_20_std": 0.17512377335026105, "nauc_ndcg_at_3_diff1": 0.3426012622485198, "nauc_ndcg_at_3_max": 0.3065766498363075, "nauc_ndcg_at_3_std": 0.17298978991080857, "nauc_ndcg_at_5_diff1": 0.3559184262752043, "nauc_ndcg_at_5_max": 0.3065662492821051, "nauc_ndcg_at_5_std": 0.17306806837418015, "nauc_precision_at_1000_diff1": -0.018564106784191587, "nauc_precision_at_1000_max": 0.05069824786767223, "nauc_precision_at_1000_std": 0.07088435598406208, "nauc_precision_at_100_diff1": 0.021787558199151093, "nauc_precision_at_100_max": 0.08798976218503793, "nauc_precision_at_100_std": 0.11717271772508196, "nauc_precision_at_10_diff1": 0.22242723366799524, "nauc_precision_at_10_max": 0.2650261328739618, "nauc_precision_at_10_std": 0.17830127753303943, "nauc_precision_at_1_diff1": 0.4164640284913696, "nauc_precision_at_1_max": 0.3053029955918928, "nauc_precision_at_1_std": 0.139614777902049, "nauc_precision_at_20_diff1": 0.14379218215003164, "nauc_precision_at_20_max": 0.2007146705369895, "nauc_precision_at_20_std": 0.16873847049962112, "nauc_precision_at_3_diff1": 0.30237337077540904, "nauc_precision_at_3_max": 0.32960480137299625, "nauc_precision_at_3_std": 0.21900953203517579, "nauc_precision_at_5_diff1": 0.31830807697094476, "nauc_precision_at_5_max": 0.32377527941214673, "nauc_precision_at_5_std": 0.22023081005719566, "nauc_recall_at_1000_diff1": 0.4080842443883318, "nauc_recall_at_1000_max": 0.4279897557390411, "nauc_recall_at_1000_std": 0.5205164184089071, "nauc_recall_at_100_diff1": 0.21150817061450922, "nauc_recall_at_100_max": 0.24558122864310847, "nauc_recall_at_100_std": 0.31297332981382747, "nauc_recall_at_10_diff1": 0.3099356523371667, "nauc_recall_at_10_max": 0.3212431729034941, "nauc_recall_at_10_std": 0.2088569211424609, "nauc_recall_at_1_diff1": 0.39952097513531726, "nauc_recall_at_1_max": 0.28310443294341225, "nauc_recall_at_1_std": 0.11174270301882441, "nauc_recall_at_20_diff1": 0.2669571566077889, "nauc_recall_at_20_max": 0.29451938489866986, "nauc_recall_at_20_std": 0.23244804522019819, "nauc_recall_at_3_diff1": 0.29743384251464644, "nauc_recall_at_3_max": 0.31274945290783196, "nauc_recall_at_3_std": 0.20077907888230528, "nauc_recall_at_5_diff1": 0.3294058068556948, "nauc_recall_at_5_max": 0.31355500993302904, "nauc_recall_at_5_std": 0.20196346725135705, "ndcg_at_1": 0.37767, "ndcg_at_10": 0.52877, "ndcg_at_100": 0.57106, "ndcg_at_1000": 0.58064, "ndcg_at_20": 0.54516, "ndcg_at_3": 0.47401, "ndcg_at_5": 0.49453, "precision_at_1": 0.37767, "precision_at_10": 0.07767, "precision_at_100": 0.01033, "precision_at_1000": 0.00112, "precision_at_20": 0.04287, "precision_at_3": 0.19794, "precision_at_5": 0.13017, "recall_at_1": 0.34917, "recall_at_10": 0.6924, "recall_at_100": 0.88282, "recall_at_1000": 0.95962, "recall_at_20": 0.75297, "recall_at_3": 0.54592, "recall_at_5": 0.59184 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 156.01362895965576, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.27973, "map_at_1": 0.195, "map_at_10": 0.24843, "map_at_100": 0.25742, "map_at_1000": 0.2584, "map_at_20": 0.25311, "map_at_3": 0.23167, "map_at_5": 0.24192, "mrr_at_1": 0.195, "mrr_at_10": 0.24843055555555552, "mrr_at_100": 0.25742438448832583, "mrr_at_1000": 0.2584015199014167, "mrr_at_20": 0.2531069340016709, "mrr_at_3": 0.2316666666666667, "mrr_at_5": 0.24191666666666667, "nauc_map_at_1000_diff1": 0.37031347443432666, "nauc_map_at_1000_max": 0.2622894003441827, "nauc_map_at_1000_std": 0.013574527284489435, "nauc_map_at_100_diff1": 0.3700960181079339, "nauc_map_at_100_max": 0.2624294930320296, "nauc_map_at_100_std": 0.014387284984817918, "nauc_map_at_10_diff1": 0.37487007427582286, "nauc_map_at_10_max": 0.2596346953674767, "nauc_map_at_10_std": 0.0011087064876853859, "nauc_map_at_1_diff1": 0.4043965637946514, "nauc_map_at_1_max": 0.2943012692215825, "nauc_map_at_1_std": -0.007264460972054072, "nauc_map_at_20_diff1": 0.37227605205579717, "nauc_map_at_20_max": 0.2617885226011628, "nauc_map_at_20_std": 0.009761427845345803, "nauc_map_at_3_diff1": 0.3926946162680841, "nauc_map_at_3_max": 0.262287901437037, "nauc_map_at_3_std": -0.008466116434885087, "nauc_map_at_5_diff1": 0.37943573073662135, "nauc_map_at_5_max": 0.2675365688661934, "nauc_map_at_5_std": 0.007650126294619136, "nauc_mrr_at_1000_diff1": 0.37031347443432666, "nauc_mrr_at_1000_max": 0.2622894003441827, "nauc_mrr_at_1000_std": 0.013574527284489435, "nauc_mrr_at_100_diff1": 0.3700960181079339, "nauc_mrr_at_100_max": 0.2624294930320296, "nauc_mrr_at_100_std": 0.014387284984817918, "nauc_mrr_at_10_diff1": 0.37487007427582286, "nauc_mrr_at_10_max": 0.2596346953674767, "nauc_mrr_at_10_std": 0.0011087064876853859, "nauc_mrr_at_1_diff1": 0.4043965637946514, "nauc_mrr_at_1_max": 0.2943012692215825, "nauc_mrr_at_1_std": -0.007264460972054072, "nauc_mrr_at_20_diff1": 0.37227605205579717, "nauc_mrr_at_20_max": 0.2617885226011628, "nauc_mrr_at_20_std": 0.009761427845345803, "nauc_mrr_at_3_diff1": 0.3926946162680841, "nauc_mrr_at_3_max": 0.262287901437037, "nauc_mrr_at_3_std": -0.008466116434885087, "nauc_mrr_at_5_diff1": 0.37943573073662135, "nauc_mrr_at_5_max": 0.2675365688661934, "nauc_mrr_at_5_std": 0.007650126294619136, "nauc_ndcg_at_1000_diff1": 0.34473693918340026, "nauc_ndcg_at_1000_max": 0.26152522423937735, "nauc_ndcg_at_1000_std": 0.05348864560421092, "nauc_ndcg_at_100_diff1": 0.33551532170894127, "nauc_ndcg_at_100_max": 0.261071278946091, "nauc_ndcg_at_100_std": 0.07134924854819213, "nauc_ndcg_at_10_diff1": 0.3603914540915809, "nauc_ndcg_at_10_max": 0.24699310480834577, "nauc_ndcg_at_10_std": 0.0051495493067397966, "nauc_ndcg_at_1_diff1": 0.4043965637946514, "nauc_ndcg_at_1_max": 0.2943012692215825, "nauc_ndcg_at_1_std": -0.007264460972054072, "nauc_ndcg_at_20_diff1": 0.3506191349557638, "nauc_ndcg_at_20_max": 0.25338465815346495, "nauc_ndcg_at_20_std": 0.03777487888250446, "nauc_ndcg_at_3_diff1": 0.39259849910667133, "nauc_ndcg_at_3_max": 0.25729722647928605, "nauc_ndcg_at_3_std": -0.006994806000349935, "nauc_ndcg_at_5_diff1": 0.36957000381323657, "nauc_ndcg_at_5_max": 0.26672553727809484, "nauc_ndcg_at_5_std": 0.019961780151738107, "nauc_precision_at_1000_diff1": 0.15944161629016781, "nauc_precision_at_1000_max": 0.3266186764237976, "nauc_precision_at_1000_std": 0.4211541520839966, "nauc_precision_at_100_diff1": 0.18989474005137313, "nauc_precision_at_100_max": 0.27414843100938263, "nauc_precision_at_100_std": 0.33297509889557114, "nauc_precision_at_10_diff1": 0.32048449562030523, "nauc_precision_at_10_max": 0.20979648064876646, "nauc_precision_at_10_std": 0.013567956748889061, "nauc_precision_at_1_diff1": 0.4043965637946514, "nauc_precision_at_1_max": 0.2943012692215825, "nauc_precision_at_1_std": -0.007264460972054072, "nauc_precision_at_20_diff1": 0.2840764748248545, "nauc_precision_at_20_max": 0.23000720225234061, "nauc_precision_at_20_std": 0.13421069861847706, "nauc_precision_at_3_diff1": 0.393299122570101, "nauc_precision_at_3_max": 0.24547261407627474, "nauc_precision_at_3_std": -0.0026177466602957254, "nauc_precision_at_5_diff1": 0.34252639426981407, "nauc_precision_at_5_max": 0.26717125321191304, "nauc_precision_at_5_std": 0.05449268416077547, "nauc_recall_at_1000_diff1": 0.15944161629016776, "nauc_recall_at_1000_max": 0.32661867642379955, "nauc_recall_at_1000_std": 0.4211541520839954, "nauc_recall_at_100_diff1": 0.18989474005137233, "nauc_recall_at_100_max": 0.2741484310093828, "nauc_recall_at_100_std": 0.33297509889557103, "nauc_recall_at_10_diff1": 0.320484495620305, "nauc_recall_at_10_max": 0.20979648064876663, "nauc_recall_at_10_std": 0.013567956748888922, "nauc_recall_at_1_diff1": 0.4043965637946514, "nauc_recall_at_1_max": 0.2943012692215825, "nauc_recall_at_1_std": -0.007264460972054072, "nauc_recall_at_20_diff1": 0.2840764748248544, "nauc_recall_at_20_max": 0.23000720225234056, "nauc_recall_at_20_std": 0.13421069861847698, "nauc_recall_at_3_diff1": 0.3932991225701008, "nauc_recall_at_3_max": 0.24547261407627485, "nauc_recall_at_3_std": -0.002617746660295337, "nauc_recall_at_5_diff1": 0.34252639426981396, "nauc_recall_at_5_max": 0.26717125321191293, "nauc_recall_at_5_std": 0.054492684160775344, "ndcg_at_1": 0.195, "ndcg_at_10": 0.27973, "ndcg_at_100": 0.32804, "ndcg_at_1000": 0.36022, "ndcg_at_20": 0.29721, "ndcg_at_3": 0.24524, "ndcg_at_5": 0.26374, "precision_at_1": 0.195, "precision_at_10": 0.038, "precision_at_100": 0.00615, "precision_at_1000": 0.00088, "precision_at_20": 0.0225, "precision_at_3": 0.095, "precision_at_5": 0.066, "recall_at_1": 0.195, "recall_at_10": 0.38, "recall_at_100": 0.615, "recall_at_1000": 0.88, "recall_at_20": 0.45, "recall_at_3": 0.285, "recall_at_5": 0.33 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.25519, "map_at_1": 0.185, "map_at_10": 0.22778, "map_at_100": 0.23672, "map_at_1000": 0.23741, "map_at_20": 0.23146, "map_at_3": 0.2125, "map_at_5": 0.2195, "mrr_at_1": 0.185, "mrr_at_10": 0.2277757936507937, "mrr_at_100": 0.23671519344932127, "mrr_at_1000": 0.23740747825554465, "mrr_at_20": 0.23145719762263883, "mrr_at_3": 0.2125, "mrr_at_5": 0.2195, "nauc_map_at_1000_diff1": 0.5869585562951857, "nauc_map_at_1000_max": 0.5011097999591789, "nauc_map_at_1000_std": 0.01990522057034222, "nauc_map_at_100_diff1": 0.586789824608114, "nauc_map_at_100_max": 0.501131079804156, "nauc_map_at_100_std": 0.01993246864662988, "nauc_map_at_10_diff1": 0.5924721588279078, "nauc_map_at_10_max": 0.5011676903737305, "nauc_map_at_10_std": 0.012874084824344068, "nauc_map_at_1_diff1": 0.651087320461199, "nauc_map_at_1_max": 0.5621184934623409, "nauc_map_at_1_std": -0.018363825238884145, "nauc_map_at_20_diff1": 0.5913314151914077, "nauc_map_at_20_max": 0.5006391866466549, "nauc_map_at_20_std": 0.01619429521386921, "nauc_map_at_3_diff1": 0.6003973094947634, "nauc_map_at_3_max": 0.5116586338153044, "nauc_map_at_3_std": -0.0058836469811811, "nauc_map_at_5_diff1": 0.5905229761539248, "nauc_map_at_5_max": 0.5048739528519287, "nauc_map_at_5_std": 5.919621017967373e-05, "nauc_mrr_at_1000_diff1": 0.5869585562951857, "nauc_mrr_at_1000_max": 0.5011097999591789, "nauc_mrr_at_1000_std": 0.01990522057034222, "nauc_mrr_at_100_diff1": 0.586789824608114, "nauc_mrr_at_100_max": 0.501131079804156, "nauc_mrr_at_100_std": 0.01993246864662988, "nauc_mrr_at_10_diff1": 0.5924721588279078, "nauc_mrr_at_10_max": 0.5011676903737305, "nauc_mrr_at_10_std": 0.012874084824344068, "nauc_mrr_at_1_diff1": 0.651087320461199, "nauc_mrr_at_1_max": 0.5621184934623409, "nauc_mrr_at_1_std": -0.018363825238884145, "nauc_mrr_at_20_diff1": 0.5913314151914077, "nauc_mrr_at_20_max": 0.5006391866466549, "nauc_mrr_at_20_std": 0.01619429521386921, "nauc_mrr_at_3_diff1": 0.6003973094947634, "nauc_mrr_at_3_max": 0.5116586338153044, "nauc_mrr_at_3_std": -0.0058836469811811, "nauc_mrr_at_5_diff1": 0.5905229761539248, "nauc_mrr_at_5_max": 0.5048739528519287, "nauc_mrr_at_5_std": 5.919621017967373e-05, "nauc_ndcg_at_1000_diff1": 0.5474277213838976, "nauc_ndcg_at_1000_max": 0.48324954770196255, "nauc_ndcg_at_1000_std": 0.07899952050880038, "nauc_ndcg_at_100_diff1": 0.5456123914207058, "nauc_ndcg_at_100_max": 0.48216591461373676, "nauc_ndcg_at_100_std": 0.08457994661231409, "nauc_ndcg_at_10_diff1": 0.575580762992824, "nauc_ndcg_at_10_max": 0.4793335268719782, "nauc_ndcg_at_10_std": 0.040740678566620746, "nauc_ndcg_at_1_diff1": 0.651087320461199, "nauc_ndcg_at_1_max": 0.5621184934623409, "nauc_ndcg_at_1_std": -0.018363825238884145, "nauc_ndcg_at_20_diff1": 0.572879929091265, "nauc_ndcg_at_20_max": 0.4769023482742165, "nauc_ndcg_at_20_std": 0.05178662087884135, "nauc_ndcg_at_3_diff1": 0.5855254292976456, "nauc_ndcg_at_3_max": 0.4983773170371353, "nauc_ndcg_at_3_std": -0.001382877447712404, "nauc_ndcg_at_5_diff1": 0.5691214837490666, "nauc_ndcg_at_5_max": 0.48658320797924554, "nauc_ndcg_at_5_std": 0.009253991821409535, "nauc_precision_at_1000_diff1": 0.27137205030380085, "nauc_precision_at_1000_max": 0.4270736187650138, "nauc_precision_at_1000_std": 0.5035325702981494, "nauc_precision_at_100_diff1": 0.3866719995077383, "nauc_precision_at_100_max": 0.4385902839737872, "nauc_precision_at_100_std": 0.3674583884564506, "nauc_precision_at_10_diff1": 0.5359859373657675, "nauc_precision_at_10_max": 0.42305531875524544, "nauc_precision_at_10_std": 0.12263172154926895, "nauc_precision_at_1_diff1": 0.651087320461199, "nauc_precision_at_1_max": 0.5621184934623409, "nauc_precision_at_1_std": -0.018363825238884145, "nauc_precision_at_20_diff1": 0.5297566941297631, "nauc_precision_at_20_max": 0.41496524201853774, "nauc_precision_at_20_std": 0.15807801235839364, "nauc_precision_at_3_diff1": 0.5460095623390954, "nauc_precision_at_3_max": 0.4635307098197868, "nauc_precision_at_3_std": 0.010820154468554766, "nauc_precision_at_5_diff1": 0.5126333873400752, "nauc_precision_at_5_max": 0.4385789708922289, "nauc_precision_at_5_std": 0.03457595396390869, "nauc_recall_at_1000_diff1": 0.27137205030380174, "nauc_recall_at_1000_max": 0.4270736187650142, "nauc_recall_at_1000_std": 0.5035325702981486, "nauc_recall_at_100_diff1": 0.38667199950773756, "nauc_recall_at_100_max": 0.4385902839737867, "nauc_recall_at_100_std": 0.36745838845644985, "nauc_recall_at_10_diff1": 0.535985937365767, "nauc_recall_at_10_max": 0.42305531875524555, "nauc_recall_at_10_std": 0.122631721549269, "nauc_recall_at_1_diff1": 0.651087320461199, "nauc_recall_at_1_max": 0.5621184934623409, "nauc_recall_at_1_std": -0.018363825238884145, "nauc_recall_at_20_diff1": 0.5297566941297629, "nauc_recall_at_20_max": 0.41496524201853785, "nauc_recall_at_20_std": 0.1580780123583932, "nauc_recall_at_3_diff1": 0.5460095623390951, "nauc_recall_at_3_max": 0.46353070981978667, "nauc_recall_at_3_std": 0.010820154468554456, "nauc_recall_at_5_diff1": 0.5126333873400747, "nauc_recall_at_5_max": 0.43857897089222847, "nauc_recall_at_5_std": 0.03457595396390839, "ndcg_at_1": 0.185, "ndcg_at_10": 0.25519, "ndcg_at_100": 0.30897, "ndcg_at_1000": 0.3333, "ndcg_at_20": 0.26893, "ndcg_at_3": 0.22208, "ndcg_at_5": 0.23456, "precision_at_1": 0.185, "precision_at_10": 0.0345, "precision_at_100": 0.0062, "precision_at_1000": 0.00083, "precision_at_20": 0.02, "precision_at_3": 0.08333, "precision_at_5": 0.056, "recall_at_1": 0.185, "recall_at_10": 0.345, "recall_at_100": 0.62, "recall_at_1000": 0.825, "recall_at_20": 0.4, "recall_at_3": 0.25, "recall_at_5": 0.28 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 1.2370071411132812, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.84967, "map_at_1": 0.72727, "map_at_10": 0.81335, "map_at_100": 0.8154, "map_at_1000": 0.8154, "map_at_20": 0.81453, "map_at_3": 0.80519, "map_at_5": 0.80844, "mrr_at_1": 0.7272727272727273, "mrr_at_10": 0.8133477633477633, "mrr_at_100": 0.8153980669402747, "mrr_at_1000": 0.8153980669402747, "mrr_at_20": 0.8145284008920373, "mrr_at_3": 0.8051948051948052, "mrr_at_5": 0.8084415584415584, "nauc_map_at_1000_diff1": 0.8206571496909262, "nauc_map_at_1000_max": 0.27777799731256525, "nauc_map_at_1000_std": -0.18126304477876914, "nauc_map_at_100_diff1": 0.8206571496909262, "nauc_map_at_100_max": 0.27777799731256525, "nauc_map_at_100_std": -0.18126304477876914, "nauc_map_at_10_diff1": 0.8190794622023716, "nauc_map_at_10_max": 0.27448093578803834, "nauc_map_at_10_std": -0.18103424295415874, "nauc_map_at_1_diff1": 0.8502933860364167, "nauc_map_at_1_max": 0.2578662916557108, "nauc_map_at_1_std": -0.17339023849413804, "nauc_map_at_20_diff1": 0.8187291485858915, "nauc_map_at_20_max": 0.2755314745647117, "nauc_map_at_20_std": -0.17835635325681093, "nauc_map_at_3_diff1": 0.8202919737719437, "nauc_map_at_3_max": 0.2832246049725774, "nauc_map_at_3_std": -0.2237173829465317, "nauc_map_at_5_diff1": 0.822121513194672, "nauc_map_at_5_max": 0.27813762918636437, "nauc_map_at_5_std": -0.20610079524493233, "nauc_mrr_at_1000_diff1": 0.8206571496909262, "nauc_mrr_at_1000_max": 0.27777799731256525, "nauc_mrr_at_1000_std": -0.18126304477876914, "nauc_mrr_at_100_diff1": 0.8206571496909262, "nauc_mrr_at_100_max": 0.27777799731256525, "nauc_mrr_at_100_std": -0.18126304477876914, "nauc_mrr_at_10_diff1": 0.8190794622023716, "nauc_mrr_at_10_max": 0.27448093578803834, "nauc_mrr_at_10_std": -0.18103424295415874, "nauc_mrr_at_1_diff1": 0.8502933860364167, "nauc_mrr_at_1_max": 0.2578662916557108, "nauc_mrr_at_1_std": -0.17339023849413804, "nauc_mrr_at_20_diff1": 0.8187291485858915, "nauc_mrr_at_20_max": 0.2755314745647117, "nauc_mrr_at_20_std": -0.17835635325681093, "nauc_mrr_at_3_diff1": 0.8202919737719437, "nauc_mrr_at_3_max": 0.2832246049725774, "nauc_mrr_at_3_std": -0.2237173829465317, "nauc_mrr_at_5_diff1": 0.822121513194672, "nauc_mrr_at_5_max": 0.27813762918636437, "nauc_mrr_at_5_std": -0.20610079524493233, "nauc_ndcg_at_1000_diff1": 0.8152844323746344, "nauc_ndcg_at_1000_max": 0.27496694113179954, "nauc_ndcg_at_1000_std": -0.18043690881056673, "nauc_ndcg_at_100_diff1": 0.8152844323746344, "nauc_ndcg_at_100_max": 0.27496694113179954, "nauc_ndcg_at_100_std": -0.18043690881056673, "nauc_ndcg_at_10_diff1": 0.8028630815222548, "nauc_ndcg_at_10_max": 0.25653467116330825, "nauc_ndcg_at_10_std": -0.16801671661837783, "nauc_ndcg_at_1_diff1": 0.8502933860364167, "nauc_ndcg_at_1_max": 0.2578662916557108, "nauc_ndcg_at_1_std": -0.17339023849413804, "nauc_ndcg_at_20_diff1": 0.801072318229471, "nauc_ndcg_at_20_max": 0.2600943671088392, "nauc_ndcg_at_20_std": -0.1573447636079357, "nauc_ndcg_at_3_diff1": 0.8088899419035144, "nauc_ndcg_at_3_max": 0.2789511826471411, "nauc_ndcg_at_3_std": -0.27327533046798197, "nauc_ndcg_at_5_diff1": 0.8121263422915855, "nauc_ndcg_at_5_max": 0.2686030984088624, "nauc_ndcg_at_5_std": -0.2398529803942681, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.5938250215675132, "nauc_precision_at_10_max": -0.012909070638074602, "nauc_precision_at_10_std": 0.09099141428707733, "nauc_precision_at_1_diff1": 0.8502933860364167, "nauc_precision_at_1_max": 0.2578662916557108, "nauc_precision_at_1_std": -0.17339023849413804, "nauc_precision_at_20_diff1": 0.4563811811151188, "nauc_precision_at_20_max": -0.08218269674418595, "nauc_precision_at_20_std": 0.4167090338610833, "nauc_precision_at_3_diff1": 0.7466686222564458, "nauc_precision_at_3_max": 0.2460394253480639, "nauc_precision_at_3_std": -0.5646971914957185, "nauc_precision_at_5_diff1": 0.7506796183748052, "nauc_precision_at_5_max": 0.1945715538625944, "nauc_precision_at_5_std": -0.46475249116360745, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.5938250215675056, "nauc_recall_at_10_max": -0.012909070638077955, "nauc_recall_at_10_std": 0.09099141428707062, "nauc_recall_at_1_diff1": 0.8502933860364167, "nauc_recall_at_1_max": 0.2578662916557108, "nauc_recall_at_1_std": -0.17339023849413804, "nauc_recall_at_20_diff1": 0.45638118111511516, "nauc_recall_at_20_max": -0.08218269674420323, "nauc_recall_at_20_std": 0.41670903386108427, "nauc_recall_at_3_diff1": 0.7466686222564456, "nauc_recall_at_3_max": 0.24603942534806478, "nauc_recall_at_3_std": -0.5646971914957211, "nauc_recall_at_5_diff1": 0.7506796183748019, "nauc_recall_at_5_max": 0.19457155386259348, "nauc_recall_at_5_std": -0.4647524911636102, "ndcg_at_1": 0.72727, "ndcg_at_10": 0.84967, "ndcg_at_100": 0.85854, "ndcg_at_1000": 0.85854, "ndcg_at_20": 0.85329, "ndcg_at_3": 0.83178, "ndcg_at_5": 0.83738, "precision_at_1": 0.72727, "precision_at_10": 0.0961, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.0487, "precision_at_3": 0.30303, "precision_at_5": 0.18442, "recall_at_1": 0.72727, "recall_at_10": 0.96104, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.97403, "recall_at_3": 0.90909, "recall_at_5": 0.92208 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 15.38088846206665, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.39712, "map_at_1": 0.21888, "map_at_10": 0.34649, "map_at_100": 0.36244, "map_at_1000": 0.36348, "map_at_20": 0.35434, "map_at_3": 0.3126, "map_at_5": 0.3326, "mrr_at_1": 0.308868501529052, "mrr_at_10": 0.38503592058637925, "mrr_at_100": 0.39587904262412565, "mrr_at_1000": 0.39658384769232136, "mrr_at_20": 0.38983454673818096, "mrr_at_3": 0.3621304791029564, "mrr_at_5": 0.3748980632008156, "nauc_map_at_1000_diff1": 0.4938793438182075, "nauc_map_at_1000_max": 0.14033836376799186, "nauc_map_at_1000_std": -0.3457665113560473, "nauc_map_at_100_diff1": 0.4934107716477553, "nauc_map_at_100_max": 0.14040961141910815, "nauc_map_at_100_std": -0.3453077860764266, "nauc_map_at_10_diff1": 0.4986170384156942, "nauc_map_at_10_max": 0.13992927666625024, "nauc_map_at_10_std": -0.35133889099460003, "nauc_map_at_1_diff1": 0.5758611494040373, "nauc_map_at_1_max": 0.10719703322248628, "nauc_map_at_1_std": -0.28511277812214025, "nauc_map_at_20_diff1": 0.494103847934402, "nauc_map_at_20_max": 0.13794317465299416, "nauc_map_at_20_std": -0.35012909945307236, "nauc_map_at_3_diff1": 0.4976826016047651, "nauc_map_at_3_max": 0.1332308266451902, "nauc_map_at_3_std": -0.33494850589834707, "nauc_map_at_5_diff1": 0.4963791581597089, "nauc_map_at_5_max": 0.1412028729309117, "nauc_map_at_5_std": -0.34317404873296514, "nauc_mrr_at_1000_diff1": 0.5064923389173324, "nauc_mrr_at_1000_max": 0.14160753046392952, "nauc_mrr_at_1000_std": -0.34244539779260885, "nauc_mrr_at_100_diff1": 0.5061203037903881, "nauc_mrr_at_100_max": 0.14207692937458782, "nauc_mrr_at_100_std": -0.3417835774278715, "nauc_mrr_at_10_diff1": 0.5086495218767871, "nauc_mrr_at_10_max": 0.13896416587385885, "nauc_mrr_at_10_std": -0.34874817852780676, "nauc_mrr_at_1_diff1": 0.5412822906602741, "nauc_mrr_at_1_max": 0.1528688773246987, "nauc_mrr_at_1_std": -0.32880812723190067, "nauc_mrr_at_20_diff1": 0.5068209639387778, "nauc_mrr_at_20_max": 0.1412416216860168, "nauc_mrr_at_20_std": -0.3447064065860721, "nauc_mrr_at_3_diff1": 0.5113245114527545, "nauc_mrr_at_3_max": 0.1334620103163377, "nauc_mrr_at_3_std": -0.3452484558664774, "nauc_mrr_at_5_diff1": 0.5050392938404901, "nauc_mrr_at_5_max": 0.13518516881793208, "nauc_mrr_at_5_std": -0.34621161778044895, "nauc_ndcg_at_1000_diff1": 0.48504903786795533, "nauc_ndcg_at_1000_max": 0.14709538742126532, "nauc_ndcg_at_1000_std": -0.33563187850448717, "nauc_ndcg_at_100_diff1": 0.4739085606140208, "nauc_ndcg_at_100_max": 0.15608766027510723, "nauc_ndcg_at_100_std": -0.3196535192612011, "nauc_ndcg_at_10_diff1": 0.49173786614731635, "nauc_ndcg_at_10_max": 0.13803761133061102, "nauc_ndcg_at_10_std": -0.3619033788429885, "nauc_ndcg_at_1_diff1": 0.5364706142040498, "nauc_ndcg_at_1_max": 0.14834968074029853, "nauc_ndcg_at_1_std": -0.3323893090055653, "nauc_ndcg_at_20_diff1": 0.47820599954723897, "nauc_ndcg_at_20_max": 0.13696761061378762, "nauc_ndcg_at_20_std": -0.35283836547606195, "nauc_ndcg_at_3_diff1": 0.4911113875408138, "nauc_ndcg_at_3_max": 0.1252264225375447, "nauc_ndcg_at_3_std": -0.3456584703342219, "nauc_ndcg_at_5_diff1": 0.48601696759888824, "nauc_ndcg_at_5_max": 0.13635614344430816, "nauc_ndcg_at_5_std": -0.34912630424542496, "nauc_precision_at_1000_diff1": -0.04754090125061955, "nauc_precision_at_1000_max": 0.12006674175244639, "nauc_precision_at_1000_std": 0.0032772165349477457, "nauc_precision_at_100_diff1": 0.037625935735913524, "nauc_precision_at_100_max": 0.1609741460344705, "nauc_precision_at_100_std": -0.03399066106569336, "nauc_precision_at_10_diff1": 0.2643227750602363, "nauc_precision_at_10_max": 0.12545697636574826, "nauc_precision_at_10_std": -0.3254097435964989, "nauc_precision_at_1_diff1": 0.5364706142040498, "nauc_precision_at_1_max": 0.14834968074029853, "nauc_precision_at_1_std": -0.3323893090055653, "nauc_precision_at_20_diff1": 0.1849450135640393, "nauc_precision_at_20_max": 0.10621803554038761, "nauc_precision_at_20_std": -0.26546287801724017, "nauc_precision_at_3_diff1": 0.3062650294584825, "nauc_precision_at_3_max": 0.14135329779907302, "nauc_precision_at_3_std": -0.3246514267708076, "nauc_precision_at_5_diff1": 0.280497812612027, "nauc_precision_at_5_max": 0.14083496728899678, "nauc_precision_at_5_std": -0.31740076848026383, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.29660023226530086, "nauc_recall_at_100_max": 0.2946376260903222, "nauc_recall_at_100_std": -0.06881895301140882, "nauc_recall_at_10_diff1": 0.44724745606511856, "nauc_recall_at_10_max": 0.13565799896280048, "nauc_recall_at_10_std": -0.3757092863613126, "nauc_recall_at_1_diff1": 0.5758611494040373, "nauc_recall_at_1_max": 0.10719703322248628, "nauc_recall_at_1_std": -0.28511277812214025, "nauc_recall_at_20_diff1": 0.39856512009507705, "nauc_recall_at_20_max": 0.13265107059328365, "nauc_recall_at_20_std": -0.34315879068741667, "nauc_recall_at_3_diff1": 0.4474915398846289, "nauc_recall_at_3_max": 0.10849069030862127, "nauc_recall_at_3_std": -0.3296936110277511, "nauc_recall_at_5_diff1": 0.43224188656662843, "nauc_recall_at_5_max": 0.1290461802519051, "nauc_recall_at_5_std": -0.34324505890807, "ndcg_at_1": 0.3104, "ndcg_at_10": 0.39712, "ndcg_at_100": 0.47154, "ndcg_at_1000": 0.49429, "ndcg_at_20": 0.42036, "ndcg_at_3": 0.34439, "ndcg_at_5": 0.36926, "precision_at_1": 0.3104, "precision_at_10": 0.08135, "precision_at_100": 0.01338, "precision_at_1000": 0.00156, "precision_at_20": 0.04763, "precision_at_3": 0.1947, "precision_at_5": 0.13731, "recall_at_1": 0.21888, "recall_at_10": 0.50757, "recall_at_100": 0.83588, "recall_at_1000": 1.0, "recall_at_20": 0.58723, "recall_at_3": 0.36993, "recall_at_5": 0.43282 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.34845, "map_at_1": 0.16119, "map_at_10": 0.28906, "map_at_100": 0.30919, "map_at_1000": 0.31078, "map_at_20": 0.29904, "map_at_3": 0.24913, "map_at_5": 0.27061, "mrr_at_1": 0.2874617737003058, "mrr_at_10": 0.3652213484782293, "mrr_at_100": 0.3766134383278123, "mrr_at_1000": 0.3773370430145179, "mrr_at_20": 0.3708954194258964, "mrr_at_3": 0.34225280326197777, "mrr_at_5": 0.3528032619775741, "nauc_map_at_1000_diff1": 0.4132604156431238, "nauc_map_at_1000_max": 0.09075253024474995, "nauc_map_at_1000_std": -0.2796985466621365, "nauc_map_at_100_diff1": 0.4132467642757701, "nauc_map_at_100_max": 0.09036616100866424, "nauc_map_at_100_std": -0.2805355536802595, "nauc_map_at_10_diff1": 0.41451380109993435, "nauc_map_at_10_max": 0.08555211183537512, "nauc_map_at_10_std": -0.28279220949434414, "nauc_map_at_1_diff1": 0.4735449804969548, "nauc_map_at_1_max": 0.11557448945335527, "nauc_map_at_1_std": -0.23419923263440692, "nauc_map_at_20_diff1": 0.4126951196771288, "nauc_map_at_20_max": 0.08846849382878437, "nauc_map_at_20_std": -0.28257251222521484, "nauc_map_at_3_diff1": 0.4296265167051265, "nauc_map_at_3_max": 0.08354629304605858, "nauc_map_at_3_std": -0.28401903390380767, "nauc_map_at_5_diff1": 0.4176921026491939, "nauc_map_at_5_max": 0.08287614745727191, "nauc_map_at_5_std": -0.2837474030167906, "nauc_mrr_at_1000_diff1": 0.395373266028508, "nauc_mrr_at_1000_max": 0.08380487419692623, "nauc_mrr_at_1000_std": -0.27874877237229806, "nauc_mrr_at_100_diff1": 0.39523289730559685, "nauc_mrr_at_100_max": 0.08421407730333434, "nauc_mrr_at_100_std": -0.2782364358582657, "nauc_mrr_at_10_diff1": 0.3931608274769371, "nauc_mrr_at_10_max": 0.07677456596398377, "nauc_mrr_at_10_std": -0.2851697681540037, "nauc_mrr_at_1_diff1": 0.43740382846764175, "nauc_mrr_at_1_max": 0.11677365601832777, "nauc_mrr_at_1_std": -0.2698572961246088, "nauc_mrr_at_20_diff1": 0.3932095949508548, "nauc_mrr_at_20_max": 0.08176630490903718, "nauc_mrr_at_20_std": -0.28086195541525866, "nauc_mrr_at_3_diff1": 0.40540297010919407, "nauc_mrr_at_3_max": 0.07698017878099053, "nauc_mrr_at_3_std": -0.2843971691113085, "nauc_mrr_at_5_diff1": 0.398503783285173, "nauc_mrr_at_5_max": 0.07601327253854948, "nauc_mrr_at_5_std": -0.2857619928392789, "nauc_ndcg_at_1000_diff1": 0.3942960671833233, "nauc_ndcg_at_1000_max": 0.0969243382982282, "nauc_ndcg_at_1000_std": -0.2650061994030209, "nauc_ndcg_at_100_diff1": 0.3904512263504864, "nauc_ndcg_at_100_max": 0.1058727472654733, "nauc_ndcg_at_100_std": -0.2582671262785743, "nauc_ndcg_at_10_diff1": 0.3905532311603214, "nauc_ndcg_at_10_max": 0.0737115809616373, "nauc_ndcg_at_10_std": -0.28794085319068563, "nauc_ndcg_at_1_diff1": 0.43740382846764175, "nauc_ndcg_at_1_max": 0.11677365601832777, "nauc_ndcg_at_1_std": -0.2698572961246088, "nauc_ndcg_at_20_diff1": 0.3878286297478578, "nauc_ndcg_at_20_max": 0.08743825748382349, "nauc_ndcg_at_20_std": -0.2805671934768294, "nauc_ndcg_at_3_diff1": 0.41076728730433737, "nauc_ndcg_at_3_max": 0.06933067063194023, "nauc_ndcg_at_3_std": -0.2975432980702906, "nauc_ndcg_at_5_diff1": 0.4026825956038753, "nauc_ndcg_at_5_max": 0.06965207776580915, "nauc_ndcg_at_5_std": -0.2916394094727, "nauc_precision_at_1000_diff1": 0.0012120654272292776, "nauc_precision_at_1000_max": 0.068882805437944, "nauc_precision_at_1000_std": 0.018982569283596245, "nauc_precision_at_100_diff1": 0.06892898098463235, "nauc_precision_at_100_max": 0.09762536800572956, "nauc_precision_at_100_std": -0.039848205836443336, "nauc_precision_at_10_diff1": 0.2076939403725245, "nauc_precision_at_10_max": 0.051045685159536135, "nauc_precision_at_10_std": -0.2167052980022179, "nauc_precision_at_1_diff1": 0.43740382846764175, "nauc_precision_at_1_max": 0.11677365601832777, "nauc_precision_at_1_std": -0.2698572961246088, "nauc_precision_at_20_diff1": 0.17003887782278732, "nauc_precision_at_20_max": 0.06507220082927256, "nauc_precision_at_20_std": -0.18406382211925656, "nauc_precision_at_3_diff1": 0.29603688125001487, "nauc_precision_at_3_max": 0.04197210075871776, "nauc_precision_at_3_std": -0.27903914892926, "nauc_precision_at_5_diff1": 0.2511492722761761, "nauc_precision_at_5_max": 0.043131129659799046, "nauc_precision_at_5_std": -0.2500670288977752, "nauc_recall_at_1000_diff1": 0.6789912062843171, "nauc_recall_at_1000_max": -0.0754372826685466, "nauc_recall_at_1000_std": -0.4366799748609778, "nauc_recall_at_100_diff1": 0.29981693556672984, "nauc_recall_at_100_max": 0.23338288447757893, "nauc_recall_at_100_std": -0.08434646011183694, "nauc_recall_at_10_diff1": 0.3240730506559147, "nauc_recall_at_10_max": 0.04714641182874439, "nauc_recall_at_10_std": -0.27583519148160984, "nauc_recall_at_1_diff1": 0.4735449804969548, "nauc_recall_at_1_max": 0.11557448945335527, "nauc_recall_at_1_std": -0.23419923263440692, "nauc_recall_at_20_diff1": 0.3106510813655099, "nauc_recall_at_20_max": 0.09511191745520513, "nauc_recall_at_20_std": -0.24451933307554488, "nauc_recall_at_3_diff1": 0.3815838931025316, "nauc_recall_at_3_max": 0.04154397826629648, "nauc_recall_at_3_std": -0.2861390949842987, "nauc_recall_at_5_diff1": 0.3583749177827193, "nauc_recall_at_5_max": 0.0396709372361749, "nauc_recall_at_5_std": -0.2899291033734086, "ndcg_at_1": 0.28746, "ndcg_at_10": 0.34845, "ndcg_at_100": 0.435, "ndcg_at_1000": 0.46333, "ndcg_at_20": 0.37639, "ndcg_at_3": 0.30097, "ndcg_at_5": 0.31384, "precision_at_1": 0.28746, "precision_at_10": 0.09174, "precision_at_100": 0.01619, "precision_at_1000": 0.00195, "precision_at_20": 0.05573, "precision_at_3": 0.19929, "precision_at_5": 0.14495, "recall_at_1": 0.16119, "recall_at_10": 0.44134, "recall_at_100": 0.8066, "recall_at_1000": 0.99694, "recall_at_20": 0.53196, "recall_at_3": 0.29962, "recall_at_5": 0.3542 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.37352, "map_at_1": 0.19463, "map_at_10": 0.31834, "map_at_100": 0.33443, "map_at_1000": 0.33558, "map_at_20": 0.32649, "map_at_3": 0.27894, "map_at_5": 0.30041, "mrr_at_1": 0.2752442996742671, "mrr_at_10": 0.35836435551419243, "mrr_at_100": 0.369324949585393, "mrr_at_1000": 0.37001485561127806, "mrr_at_20": 0.3638920526358, "mrr_at_3": 0.33197611292073836, "mrr_at_5": 0.34630836047774155, "nauc_map_at_1000_diff1": 0.4385039701306293, "nauc_map_at_1000_max": 0.04109853971012122, "nauc_map_at_1000_std": -0.3014883929232645, "nauc_map_at_100_diff1": 0.43787844081868205, "nauc_map_at_100_max": 0.04147338706535819, "nauc_map_at_100_std": -0.3008286696906349, "nauc_map_at_10_diff1": 0.4421977965125799, "nauc_map_at_10_max": 0.034228741506822404, "nauc_map_at_10_std": -0.3125065142395519, "nauc_map_at_1_diff1": 0.5581508907427222, "nauc_map_at_1_max": 0.0333779502484941, "nauc_map_at_1_std": -0.2292104531599672, "nauc_map_at_20_diff1": 0.44129038398015746, "nauc_map_at_20_max": 0.03795484756429189, "nauc_map_at_20_std": -0.30987390225868416, "nauc_map_at_3_diff1": 0.45291200776206947, "nauc_map_at_3_max": 0.04254275361818072, "nauc_map_at_3_std": -0.2798266220542301, "nauc_map_at_5_diff1": 0.4464396026810503, "nauc_map_at_5_max": 0.034295293776766, "nauc_map_at_5_std": -0.2993714371137663, "nauc_mrr_at_1000_diff1": 0.46507435682818815, "nauc_mrr_at_1000_max": 0.05139897729335335, "nauc_mrr_at_1000_std": -0.2930435001253733, "nauc_mrr_at_100_diff1": 0.46470644818935436, "nauc_mrr_at_100_max": 0.051808627157288, "nauc_mrr_at_100_std": -0.29242180585091254, "nauc_mrr_at_10_diff1": 0.46679584828865944, "nauc_mrr_at_10_max": 0.04669594755620027, "nauc_mrr_at_10_std": -0.30011366716993615, "nauc_mrr_at_1_diff1": 0.523765306788138, "nauc_mrr_at_1_max": 0.07101697112154846, "nauc_mrr_at_1_std": -0.2720158381402068, "nauc_mrr_at_20_diff1": 0.4664255834677069, "nauc_mrr_at_20_max": 0.0499587264407141, "nauc_mrr_at_20_std": -0.2969349667469947, "nauc_mrr_at_3_diff1": 0.4729610660750291, "nauc_mrr_at_3_max": 0.050329267957550605, "nauc_mrr_at_3_std": -0.29555486383633967, "nauc_mrr_at_5_diff1": 0.46488455928143696, "nauc_mrr_at_5_max": 0.040193067387784505, "nauc_mrr_at_5_std": -0.29932901376821935, "nauc_ndcg_at_1000_diff1": 0.42690516618900637, "nauc_ndcg_at_1000_max": 0.05619517165396485, "nauc_ndcg_at_1000_std": -0.28541751450799574, "nauc_ndcg_at_100_diff1": 0.40892391546541423, "nauc_ndcg_at_100_max": 0.06798408580650303, "nauc_ndcg_at_100_std": -0.2647801256659749, "nauc_ndcg_at_10_diff1": 0.4307636665886242, "nauc_ndcg_at_10_max": 0.03143040834478728, "nauc_ndcg_at_10_std": -0.32725097959553556, "nauc_ndcg_at_1_diff1": 0.523765306788138, "nauc_ndcg_at_1_max": 0.07101697112154846, "nauc_ndcg_at_1_std": -0.2720158381402068, "nauc_ndcg_at_20_diff1": 0.4283767677265797, "nauc_ndcg_at_20_max": 0.043394654891873476, "nauc_ndcg_at_20_std": -0.3167498438789006, "nauc_ndcg_at_3_diff1": 0.444387646756086, "nauc_ndcg_at_3_max": 0.039069892868760496, "nauc_ndcg_at_3_std": -0.29060607655626514, "nauc_ndcg_at_5_diff1": 0.4334668500292305, "nauc_ndcg_at_5_max": 0.022543720407902403, "nauc_ndcg_at_5_std": -0.30996646120190485, "nauc_precision_at_1000_diff1": -0.07809179169050465, "nauc_precision_at_1000_max": 0.15769782136238175, "nauc_precision_at_1000_std": 0.0632262670206087, "nauc_precision_at_100_diff1": -0.03266071890007152, "nauc_precision_at_100_max": 0.17990596259954394, "nauc_precision_at_100_std": 0.042071449268137315, "nauc_precision_at_10_diff1": 0.1969497261484465, "nauc_precision_at_10_max": 0.041005225200070675, "nauc_precision_at_10_std": -0.315577687963074, "nauc_precision_at_1_diff1": 0.523765306788138, "nauc_precision_at_1_max": 0.07101697112154846, "nauc_precision_at_1_std": -0.2720158381402068, "nauc_precision_at_20_diff1": 0.157261472122755, "nauc_precision_at_20_max": 0.07202715790374091, "nauc_precision_at_20_std": -0.25184817016670064, "nauc_precision_at_3_diff1": 0.27458187398964146, "nauc_precision_at_3_max": 0.06826016185057171, "nauc_precision_at_3_std": -0.2858467909073881, "nauc_precision_at_5_diff1": 0.2344017397441233, "nauc_precision_at_5_max": 0.024898959077200182, "nauc_precision_at_5_std": -0.31566360786536113, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.1511433306946275, "nauc_recall_at_100_max": 0.22028163178567373, "nauc_recall_at_100_std": 0.028221514320504418, "nauc_recall_at_10_diff1": 0.35766545994972215, "nauc_recall_at_10_max": 0.015348344799469206, "nauc_recall_at_10_std": -0.3570225338139556, "nauc_recall_at_1_diff1": 0.5581508907427222, "nauc_recall_at_1_max": 0.0333779502484941, "nauc_recall_at_1_std": -0.2292104531599672, "nauc_recall_at_20_diff1": 0.34682430009895354, "nauc_recall_at_20_max": 0.05228464701534023, "nauc_recall_at_20_std": -0.3241105035389205, "nauc_recall_at_3_diff1": 0.4024982484586635, "nauc_recall_at_3_max": 0.022559669627852694, "nauc_recall_at_3_std": -0.2846336396017293, "nauc_recall_at_5_diff1": 0.37282768111220693, "nauc_recall_at_5_max": -0.009473898791724066, "nauc_recall_at_5_std": -0.31952254883598674, "ndcg_at_1": 0.27524, "ndcg_at_10": 0.37352, "ndcg_at_100": 0.44766, "ndcg_at_1000": 0.47325, "ndcg_at_20": 0.3975, "ndcg_at_3": 0.3106, "ndcg_at_5": 0.33838, "precision_at_1": 0.27524, "precision_at_10": 0.08274, "precision_at_100": 0.01391, "precision_at_1000": 0.00167, "precision_at_20": 0.04886, "precision_at_3": 0.17644, "precision_at_5": 0.13127, "recall_at_1": 0.19463, "recall_at_10": 0.49377, "recall_at_100": 0.81522, "recall_at_1000": 1.0, "recall_at_20": 0.57375, "recall_at_3": 0.33201, "recall_at_5": 0.39823 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/model_meta.json ================================================ {"name": "intfloat/multilingual-e5-large-instruct", "revision": "baa7be480a7de1539afce709c8f13f833a510e0a", "release_date": "2024-02-08", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": 560000000, "memory_usage": null, "max_tokens": 514, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["GritLM", "PyTorch"], "reference": "https://huggingface.co/intfloat/multilingual-e5-large-instruct", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"} ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 1.926253318786621, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.58332, "map_at_1": 0.4386, "map_at_10": 0.53697, "map_at_100": 0.54586, "map_at_1000": 0.54614, "map_at_20": 0.54221, "map_at_3": 0.51608, "map_at_5": 0.53056, "mrr_at_1": 0.43859649122807015, "mrr_at_10": 0.5369743803954331, "mrr_at_100": 0.5458626227782769, "mrr_at_1000": 0.5461427541033771, "mrr_at_20": 0.5422120953763044, "mrr_at_3": 0.5160818713450293, "mrr_at_5": 0.5305555555555557, "nauc_map_at_1000_diff1": 0.5517346332846484, "nauc_map_at_1000_max": 0.3738554420856829, "nauc_map_at_1000_std": -0.041467320778462574, "nauc_map_at_100_diff1": 0.5517907504543731, "nauc_map_at_100_max": 0.37428072978767696, "nauc_map_at_100_std": -0.04094065029665378, "nauc_map_at_10_diff1": 0.5437860017629315, "nauc_map_at_10_max": 0.37427453041429154, "nauc_map_at_10_std": -0.043857670246423446, "nauc_map_at_1_diff1": 0.6202229707603372, "nauc_map_at_1_max": 0.34550605993013117, "nauc_map_at_1_std": -0.06910356233222696, "nauc_map_at_20_diff1": 0.5507431969965568, "nauc_map_at_20_max": 0.37486396950221434, "nauc_map_at_20_std": -0.044013189768391754, "nauc_map_at_3_diff1": 0.550991292328967, "nauc_map_at_3_max": 0.36829768923991496, "nauc_map_at_3_std": -0.029483143290972333, "nauc_map_at_5_diff1": 0.5436380533907215, "nauc_map_at_5_max": 0.3612882940446798, "nauc_map_at_5_std": -0.04342390129159417, "nauc_mrr_at_1000_diff1": 0.5517346332846484, "nauc_mrr_at_1000_max": 0.3738554420856829, "nauc_mrr_at_1000_std": -0.041467320778462574, "nauc_mrr_at_100_diff1": 0.5517907504543731, "nauc_mrr_at_100_max": 0.37428072978767696, "nauc_mrr_at_100_std": -0.04094065029665378, "nauc_mrr_at_10_diff1": 0.5437860017629315, "nauc_mrr_at_10_max": 0.37427453041429154, "nauc_mrr_at_10_std": -0.043857670246423446, "nauc_mrr_at_1_diff1": 0.6202229707603372, "nauc_mrr_at_1_max": 0.34550605993013117, "nauc_mrr_at_1_std": -0.06910356233222696, "nauc_mrr_at_20_diff1": 0.5507431969965568, "nauc_mrr_at_20_max": 0.37486396950221434, "nauc_mrr_at_20_std": -0.044013189768391754, "nauc_mrr_at_3_diff1": 0.550991292328967, "nauc_mrr_at_3_max": 0.36829768923991496, "nauc_mrr_at_3_std": -0.029483143290972333, "nauc_mrr_at_5_diff1": 0.5436380533907215, "nauc_mrr_at_5_max": 0.3612882940446798, "nauc_mrr_at_5_std": -0.04342390129159417, "nauc_ndcg_at_1000_diff1": 0.541609472661349, "nauc_ndcg_at_1000_max": 0.385806640077638, "nauc_ndcg_at_1000_std": -0.039003526996751414, "nauc_ndcg_at_100_diff1": 0.5430909554997655, "nauc_ndcg_at_100_max": 0.39736728259191517, "nauc_ndcg_at_100_std": -0.023662216602588205, "nauc_ndcg_at_10_diff1": 0.5082666270294448, "nauc_ndcg_at_10_max": 0.4006411092235243, "nauc_ndcg_at_10_std": -0.05061806791160763, "nauc_ndcg_at_1_diff1": 0.6202229707603372, "nauc_ndcg_at_1_max": 0.34550605993013117, "nauc_ndcg_at_1_std": -0.06910356233222696, "nauc_ndcg_at_20_diff1": 0.5347847565766581, "nauc_ndcg_at_20_max": 0.40239235073804375, "nauc_ndcg_at_20_std": -0.05250049718351136, "nauc_ndcg_at_3_diff1": 0.5252751824223435, "nauc_ndcg_at_3_max": 0.37436810249463026, "nauc_ndcg_at_3_std": -0.025330820273204355, "nauc_ndcg_at_5_diff1": 0.5100034314502537, "nauc_ndcg_at_5_max": 0.3635626974773759, "nauc_ndcg_at_5_std": -0.05091061686797668, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 0.5949706437682594, "nauc_precision_at_100_max": 0.8249538540540309, "nauc_precision_at_100_std": 0.5525122873546001, "nauc_precision_at_10_diff1": 0.3623729064886154, "nauc_precision_at_10_max": 0.527440761323421, "nauc_precision_at_10_std": -0.08346085656775253, "nauc_precision_at_1_diff1": 0.6202229707603372, "nauc_precision_at_1_max": 0.34550605993013117, "nauc_precision_at_1_std": -0.06910356233222696, "nauc_precision_at_20_diff1": 0.4845052106706362, "nauc_precision_at_20_max": 0.5732796252269216, "nauc_precision_at_20_std": -0.10933275275973593, "nauc_precision_at_3_diff1": 0.4431620932912698, "nauc_precision_at_3_max": 0.3930005010668853, "nauc_precision_at_3_std": -0.015383329296425807, "nauc_precision_at_5_diff1": 0.3893931298353917, "nauc_precision_at_5_max": 0.3706176726143006, "nauc_precision_at_5_std": -0.08465041790219158, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.5949706437682629, "nauc_recall_at_100_max": 0.8249538540540327, "nauc_recall_at_100_std": 0.5525122873545996, "nauc_recall_at_10_diff1": 0.3623729064886153, "nauc_recall_at_10_max": 0.5274407613234211, "nauc_recall_at_10_std": -0.08346085656775215, "nauc_recall_at_1_diff1": 0.6202229707603372, "nauc_recall_at_1_max": 0.34550605993013117, "nauc_recall_at_1_std": -0.06910356233222696, "nauc_recall_at_20_diff1": 0.4845052106706359, "nauc_recall_at_20_max": 0.5732796252269218, "nauc_recall_at_20_std": -0.1093327527597355, "nauc_recall_at_3_diff1": 0.44316209329126927, "nauc_recall_at_3_max": 0.3930005010668845, "nauc_recall_at_3_std": -0.015383329296426449, "nauc_recall_at_5_diff1": 0.38939312983539126, "nauc_recall_at_5_max": 0.3706176726142997, "nauc_recall_at_5_std": -0.08465041790219185, "ndcg_at_1": 0.4386, "ndcg_at_10": 0.58332, "ndcg_at_100": 0.62923, "ndcg_at_1000": 0.63615, "ndcg_at_20": 0.60298, "ndcg_at_3": 0.54125, "ndcg_at_5": 0.56692, "precision_at_1": 0.4386, "precision_at_10": 0.07281, "precision_at_100": 0.00947, "precision_at_1000": 0.001, "precision_at_20": 0.04035, "precision_at_3": 0.20468, "precision_at_5": 0.13509, "recall_at_1": 0.4386, "recall_at_10": 0.72807, "recall_at_100": 0.94737, "recall_at_1000": 1.0, "recall_at_20": 0.80702, "recall_at_3": 0.61404, "recall_at_5": 0.67544 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 14.881865978240967, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.81636, "map_at_1": 0.71111, "map_at_10": 0.78502, "map_at_100": 0.78852, "map_at_1000": 0.78858, "map_at_20": 0.78742, "map_at_3": 0.77278, "map_at_5": 0.78217, "mrr_at_1": 0.7111111111111111, "mrr_at_10": 0.785019400352734, "mrr_at_100": 0.7885154478476163, "mrr_at_1000": 0.7885840708869507, "mrr_at_20": 0.7874219215310551, "mrr_at_3": 0.7727777777777781, "mrr_at_5": 0.782166666666667, "nauc_map_at_1000_diff1": 0.7460104277733405, "nauc_map_at_1000_max": 0.6691811197505108, "nauc_map_at_1000_std": 0.1255105150080002, "nauc_map_at_100_diff1": 0.7459543272981454, "nauc_map_at_100_max": 0.6691892038435562, "nauc_map_at_100_std": 0.12558031284678517, "nauc_map_at_10_diff1": 0.7430599248125045, "nauc_map_at_10_max": 0.668437348080883, "nauc_map_at_10_std": 0.12674252500632405, "nauc_map_at_1_diff1": 0.7834916962736507, "nauc_map_at_1_max": 0.6615425927456003, "nauc_map_at_1_std": 0.09969635627530353, "nauc_map_at_20_diff1": 0.7456399948953604, "nauc_map_at_20_max": 0.6699521349766611, "nauc_map_at_20_std": 0.12686929393460583, "nauc_map_at_3_diff1": 0.7412164858825709, "nauc_map_at_3_max": 0.6673170148889498, "nauc_map_at_3_std": 0.12768602289395178, "nauc_map_at_5_diff1": 0.7415733714008201, "nauc_map_at_5_max": 0.6662297485125583, "nauc_map_at_5_std": 0.12559583058273427, "nauc_mrr_at_1000_diff1": 0.7460104277733405, "nauc_mrr_at_1000_max": 0.6691811197505108, "nauc_mrr_at_1000_std": 0.1255105150080002, "nauc_mrr_at_100_diff1": 0.7459543272981454, "nauc_mrr_at_100_max": 0.6691892038435562, "nauc_mrr_at_100_std": 0.12558031284678517, "nauc_mrr_at_10_diff1": 0.7430599248125045, "nauc_mrr_at_10_max": 0.668437348080883, "nauc_mrr_at_10_std": 0.12674252500632405, "nauc_mrr_at_1_diff1": 0.7834916962736507, "nauc_mrr_at_1_max": 0.6615425927456003, "nauc_mrr_at_1_std": 0.09969635627530353, "nauc_mrr_at_20_diff1": 0.7456399948953604, "nauc_mrr_at_20_max": 0.6699521349766611, "nauc_mrr_at_20_std": 0.12686929393460583, "nauc_mrr_at_3_diff1": 0.7412164858825709, "nauc_mrr_at_3_max": 0.6673170148889498, "nauc_mrr_at_3_std": 0.12768602289395178, "nauc_mrr_at_5_diff1": 0.7415733714008201, "nauc_mrr_at_5_max": 0.6662297485125583, "nauc_mrr_at_5_std": 0.12559583058273427, "nauc_ndcg_at_1000_diff1": 0.7403877052308159, "nauc_ndcg_at_1000_max": 0.6711504381026011, "nauc_ndcg_at_1000_std": 0.13147530447063688, "nauc_ndcg_at_100_diff1": 0.7385280349845585, "nauc_ndcg_at_100_max": 0.6712436908129258, "nauc_ndcg_at_100_std": 0.1322414189580357, "nauc_ndcg_at_10_diff1": 0.7273219314921056, "nauc_ndcg_at_10_max": 0.6717930792030385, "nauc_ndcg_at_10_std": 0.1420570858544101, "nauc_ndcg_at_1_diff1": 0.7834916962736507, "nauc_ndcg_at_1_max": 0.6615425927456003, "nauc_ndcg_at_1_std": 0.09969635627530353, "nauc_ndcg_at_20_diff1": 0.7370348876719133, "nauc_ndcg_at_20_max": 0.6779629792122771, "nauc_ndcg_at_20_std": 0.14419940200337705, "nauc_ndcg_at_3_diff1": 0.7242474640949236, "nauc_ndcg_at_3_max": 0.667543486664276, "nauc_ndcg_at_3_std": 0.14052838438895446, "nauc_ndcg_at_5_diff1": 0.7238051330078697, "nauc_ndcg_at_5_max": 0.665008106581145, "nauc_ndcg_at_5_std": 0.13686426047249073, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.5729458450046719, "nauc_precision_at_100_max": 0.6769374416433434, "nauc_precision_at_100_std": 0.16938997821350255, "nauc_precision_at_10_diff1": 0.631185807656393, "nauc_precision_at_10_max": 0.697094871703954, "nauc_precision_at_10_std": 0.24538760651939961, "nauc_precision_at_1_diff1": 0.7834916962736507, "nauc_precision_at_1_max": 0.6615425927456003, "nauc_precision_at_1_std": 0.09969635627530353, "nauc_precision_at_20_diff1": 0.6812324929971952, "nauc_precision_at_20_max": 0.7779271708683456, "nauc_precision_at_20_std": 0.33409897292250024, "nauc_precision_at_3_diff1": 0.6554203539823014, "nauc_precision_at_3_max": 0.6675597632456034, "nauc_precision_at_3_std": 0.19331111366509504, "nauc_precision_at_5_diff1": 0.6320571924901426, "nauc_precision_at_5_max": 0.6564250875736125, "nauc_precision_at_5_std": 0.19454705113153592, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.5729458450046645, "nauc_recall_at_100_max": 0.6769374416433231, "nauc_recall_at_100_std": 0.16938997821351892, "nauc_recall_at_10_diff1": 0.631185807656395, "nauc_recall_at_10_max": 0.6970948717039557, "nauc_recall_at_10_std": 0.24538760651940017, "nauc_recall_at_1_diff1": 0.7834916962736507, "nauc_recall_at_1_max": 0.6615425927456003, "nauc_recall_at_1_std": 0.09969635627530353, "nauc_recall_at_20_diff1": 0.6812324929972007, "nauc_recall_at_20_max": 0.7779271708683464, "nauc_recall_at_20_std": 0.33409897292250357, "nauc_recall_at_3_diff1": 0.6554203539822998, "nauc_recall_at_3_max": 0.6675597632456036, "nauc_recall_at_3_std": 0.1933111136650955, "nauc_recall_at_5_diff1": 0.6320571924901449, "nauc_recall_at_5_max": 0.6564250875736146, "nauc_recall_at_5_std": 0.19454705113153728, "ndcg_at_1": 0.71111, "ndcg_at_10": 0.81636, "ndcg_at_100": 0.83256, "ndcg_at_1000": 0.83428, "ndcg_at_20": 0.82472, "ndcg_at_3": 0.79237, "ndcg_at_5": 0.80929, "precision_at_1": 0.71111, "precision_at_10": 0.09122, "precision_at_100": 0.00987, "precision_at_1000": 0.001, "precision_at_20": 0.04722, "precision_at_3": 0.28296, "precision_at_5": 0.178, "recall_at_1": 0.71111, "recall_at_10": 0.91222, "recall_at_100": 0.98667, "recall_at_1000": 1.0, "recall_at_20": 0.94444, "recall_at_3": 0.84889, "recall_at_5": 0.89 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.38503, "map_at_1": 0.25111, "map_at_10": 0.33623, "map_at_100": 0.34861, "map_at_1000": 0.34944, "map_at_20": 0.34335, "map_at_3": 0.3087, "map_at_5": 0.32426, "mrr_at_1": 0.2511111111111111, "mrr_at_10": 0.33623368606701926, "mrr_at_100": 0.3486067336694853, "mrr_at_1000": 0.3494351883870817, "mrr_at_20": 0.34335314480077095, "mrr_at_3": 0.3087037037037038, "mrr_at_5": 0.3242592592592592, "nauc_map_at_1000_diff1": 0.4154567226238162, "nauc_map_at_1000_max": 0.4158309147498473, "nauc_map_at_1000_std": 0.17888546818703868, "nauc_map_at_100_diff1": 0.4152326250614072, "nauc_map_at_100_max": 0.41600557985805253, "nauc_map_at_100_std": 0.17935211886089916, "nauc_map_at_10_diff1": 0.41510374037744324, "nauc_map_at_10_max": 0.4157099886442739, "nauc_map_at_10_std": 0.1765358033571632, "nauc_map_at_1_diff1": 0.4554212032327214, "nauc_map_at_1_max": 0.4215474516380579, "nauc_map_at_1_std": 0.16045332489371625, "nauc_map_at_20_diff1": 0.41400658612862834, "nauc_map_at_20_max": 0.4159250228040855, "nauc_map_at_20_std": 0.1805880886003614, "nauc_map_at_3_diff1": 0.42562381034249075, "nauc_map_at_3_max": 0.41471323056725135, "nauc_map_at_3_std": 0.16381294055280307, "nauc_map_at_5_diff1": 0.4212331205561075, "nauc_map_at_5_max": 0.41586765905095485, "nauc_map_at_5_std": 0.17257668694334635, "nauc_mrr_at_1000_diff1": 0.4154567226238162, "nauc_mrr_at_1000_max": 0.4158309147498473, "nauc_mrr_at_1000_std": 0.17888546818703868, "nauc_mrr_at_100_diff1": 0.4152326250614072, "nauc_mrr_at_100_max": 0.41600557985805253, "nauc_mrr_at_100_std": 0.17935211886089916, "nauc_mrr_at_10_diff1": 0.41510374037744324, "nauc_mrr_at_10_max": 0.4157099886442739, "nauc_mrr_at_10_std": 0.1765358033571632, "nauc_mrr_at_1_diff1": 0.4554212032327214, "nauc_mrr_at_1_max": 0.4215474516380579, "nauc_mrr_at_1_std": 0.16045332489371625, "nauc_mrr_at_20_diff1": 0.41400658612862834, "nauc_mrr_at_20_max": 0.4159250228040855, "nauc_mrr_at_20_std": 0.1805880886003614, "nauc_mrr_at_3_diff1": 0.42562381034249075, "nauc_mrr_at_3_max": 0.41471323056725135, "nauc_mrr_at_3_std": 0.16381294055280307, "nauc_mrr_at_5_diff1": 0.4212331205561075, "nauc_mrr_at_5_max": 0.41586765905095485, "nauc_mrr_at_5_std": 0.17257668694334635, "nauc_ndcg_at_1000_diff1": 0.4059833739794044, "nauc_ndcg_at_1000_max": 0.4159984209283889, "nauc_ndcg_at_1000_std": 0.18892348858073468, "nauc_ndcg_at_100_diff1": 0.4004914557560742, "nauc_ndcg_at_100_max": 0.42053802814260877, "nauc_ndcg_at_100_std": 0.20103630157858776, "nauc_ndcg_at_10_diff1": 0.39660555776825546, "nauc_ndcg_at_10_max": 0.4147810458283918, "nauc_ndcg_at_10_std": 0.19015467037702996, "nauc_ndcg_at_1_diff1": 0.4554212032327214, "nauc_ndcg_at_1_max": 0.4215474516380579, "nauc_ndcg_at_1_std": 0.16045332489371625, "nauc_ndcg_at_20_diff1": 0.39183960842200916, "nauc_ndcg_at_20_max": 0.4164531901222306, "nauc_ndcg_at_20_std": 0.20561997894391823, "nauc_ndcg_at_3_diff1": 0.4183527595068305, "nauc_ndcg_at_3_max": 0.41205328463590385, "nauc_ndcg_at_3_std": 0.16473954554940762, "nauc_ndcg_at_5_diff1": 0.41106019452382037, "nauc_ndcg_at_5_max": 0.414430081220993, "nauc_ndcg_at_5_std": 0.1798219427802208, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.3173059058716875, "nauc_precision_at_100_max": 0.48874488150355705, "nauc_precision_at_100_std": 0.3851970481788937, "nauc_precision_at_10_diff1": 0.3379013591127392, "nauc_precision_at_10_max": 0.4135948408179432, "nauc_precision_at_10_std": 0.2362789028073475, "nauc_precision_at_1_diff1": 0.4554212032327214, "nauc_precision_at_1_max": 0.4215474516380579, "nauc_precision_at_1_std": 0.16045332489371625, "nauc_precision_at_20_diff1": 0.3074462375210219, "nauc_precision_at_20_max": 0.4227394711418037, "nauc_precision_at_20_std": 0.31148602712276796, "nauc_precision_at_3_diff1": 0.3991111406019204, "nauc_precision_at_3_max": 0.4046560538220088, "nauc_precision_at_3_std": 0.16723019756277682, "nauc_precision_at_5_diff1": 0.38311471951228393, "nauc_precision_at_5_max": 0.410868853532377, "nauc_precision_at_5_std": 0.2013298704322076, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.3173059058716864, "nauc_recall_at_100_max": 0.4887448815035555, "nauc_recall_at_100_std": 0.3851970481788933, "nauc_recall_at_10_diff1": 0.33790135911273933, "nauc_recall_at_10_max": 0.4135948408179429, "nauc_recall_at_10_std": 0.23627890280734765, "nauc_recall_at_1_diff1": 0.4554212032327214, "nauc_recall_at_1_max": 0.4215474516380579, "nauc_recall_at_1_std": 0.16045332489371625, "nauc_recall_at_20_diff1": 0.3074462375210217, "nauc_recall_at_20_max": 0.4227394711418042, "nauc_recall_at_20_std": 0.311486027122768, "nauc_recall_at_3_diff1": 0.3991111406019202, "nauc_recall_at_3_max": 0.4046560538220091, "nauc_recall_at_3_std": 0.1672301975627768, "nauc_recall_at_5_diff1": 0.3831147195122837, "nauc_recall_at_5_max": 0.41086885353237734, "nauc_recall_at_5_std": 0.2013298704322081, "ndcg_at_1": 0.25111, "ndcg_at_10": 0.38503, "ndcg_at_100": 0.44957, "ndcg_at_1000": 0.46889, "ndcg_at_20": 0.41091, "ndcg_at_3": 0.32758, "ndcg_at_5": 0.35555, "precision_at_1": 0.25111, "precision_at_10": 0.05422, "precision_at_100": 0.00854, "precision_at_1000": 0.001, "precision_at_20": 0.03222, "precision_at_3": 0.12741, "precision_at_5": 0.09, "recall_at_1": 0.25111, "recall_at_10": 0.54222, "recall_at_100": 0.85444, "recall_at_1000": 1.0, "recall_at_20": 0.64444, "recall_at_3": 0.38222, "recall_at_5": 0.45 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.57379, "map_at_1": 0.41556, "map_at_10": 0.5192, "map_at_100": 0.52712, "map_at_1000": 0.52746, "map_at_20": 0.52356, "map_at_3": 0.48759, "map_at_5": 0.5087, "mrr_at_1": 0.41555555555555557, "mrr_at_10": 0.5191979717813047, "mrr_at_100": 0.5271212900559807, "mrr_at_1000": 0.5274581721766388, "mrr_at_20": 0.5235614299223684, "mrr_at_3": 0.4875925925925923, "mrr_at_5": 0.5087037037037031, "nauc_map_at_1000_diff1": 0.5585832359996836, "nauc_map_at_1000_max": 0.3777517058797167, "nauc_map_at_1000_std": 0.10864198894559263, "nauc_map_at_100_diff1": 0.5583640887580703, "nauc_map_at_100_max": 0.3778017746569433, "nauc_map_at_100_std": 0.10892037540374716, "nauc_map_at_10_diff1": 0.5568130166322439, "nauc_map_at_10_max": 0.37526350602597447, "nauc_map_at_10_std": 0.10705131891830384, "nauc_map_at_1_diff1": 0.6112996413770887, "nauc_map_at_1_max": 0.3748486565301294, "nauc_map_at_1_std": 0.07824565238022668, "nauc_map_at_20_diff1": 0.5571447630522545, "nauc_map_at_20_max": 0.37715294322911946, "nauc_map_at_20_std": 0.10884717375853395, "nauc_map_at_3_diff1": 0.5601493741691581, "nauc_map_at_3_max": 0.3785858586871427, "nauc_map_at_3_std": 0.10364894814143376, "nauc_map_at_5_diff1": 0.5586379573886199, "nauc_map_at_5_max": 0.3782769675070838, "nauc_map_at_5_std": 0.10784039363976371, "nauc_mrr_at_1000_diff1": 0.5585832359996836, "nauc_mrr_at_1000_max": 0.3777517058797167, "nauc_mrr_at_1000_std": 0.10864198894559263, "nauc_mrr_at_100_diff1": 0.5583640887580703, "nauc_mrr_at_100_max": 0.3778017746569433, "nauc_mrr_at_100_std": 0.10892037540374716, "nauc_mrr_at_10_diff1": 0.5568130166322439, "nauc_mrr_at_10_max": 0.37526350602597447, "nauc_mrr_at_10_std": 0.10705131891830384, "nauc_mrr_at_1_diff1": 0.6112996413770887, "nauc_mrr_at_1_max": 0.3748486565301294, "nauc_mrr_at_1_std": 0.07824565238022668, "nauc_mrr_at_20_diff1": 0.5571447630522545, "nauc_mrr_at_20_max": 0.37715294322911946, "nauc_mrr_at_20_std": 0.10884717375853395, "nauc_mrr_at_3_diff1": 0.5601493741691581, "nauc_mrr_at_3_max": 0.3785858586871427, "nauc_mrr_at_3_std": 0.10364894814143376, "nauc_mrr_at_5_diff1": 0.5586379573886199, "nauc_mrr_at_5_max": 0.3782769675070838, "nauc_mrr_at_5_std": 0.10784039363976371, "nauc_ndcg_at_1000_diff1": 0.5477809675932058, "nauc_ndcg_at_1000_max": 0.37861192244266095, "nauc_ndcg_at_1000_std": 0.11744180418719868, "nauc_ndcg_at_100_diff1": 0.5425101873218164, "nauc_ndcg_at_100_max": 0.3798599781503716, "nauc_ndcg_at_100_std": 0.12566306929003634, "nauc_ndcg_at_10_diff1": 0.5334566852145959, "nauc_ndcg_at_10_max": 0.37091057534300936, "nauc_ndcg_at_10_std": 0.11898284694817188, "nauc_ndcg_at_1_diff1": 0.6112996413770887, "nauc_ndcg_at_1_max": 0.3748486565301294, "nauc_ndcg_at_1_std": 0.07824565238022668, "nauc_ndcg_at_20_diff1": 0.5334288621204861, "nauc_ndcg_at_20_max": 0.3768878936587119, "nauc_ndcg_at_20_std": 0.12506099443640833, "nauc_ndcg_at_3_diff1": 0.5429609926684678, "nauc_ndcg_at_3_max": 0.3794746016162753, "nauc_ndcg_at_3_std": 0.11178884413936593, "nauc_ndcg_at_5_diff1": 0.5394538018456475, "nauc_ndcg_at_5_max": 0.37921389986002785, "nauc_ndcg_at_5_std": 0.12042972274581841, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.3670186023127206, "nauc_precision_at_100_max": 0.4221432162608671, "nauc_precision_at_100_std": 0.41216871363930435, "nauc_precision_at_10_diff1": 0.43143142452187516, "nauc_precision_at_10_max": 0.3472467649603327, "nauc_precision_at_10_std": 0.1698648922267003, "nauc_precision_at_1_diff1": 0.6112996413770887, "nauc_precision_at_1_max": 0.3748486565301294, "nauc_precision_at_1_std": 0.07824565238022668, "nauc_precision_at_20_diff1": 0.40364263714887605, "nauc_precision_at_20_max": 0.3747221128684894, "nauc_precision_at_20_std": 0.21668828011022712, "nauc_precision_at_3_diff1": 0.4900076623182607, "nauc_precision_at_3_max": 0.38209105789789044, "nauc_precision_at_3_std": 0.13673463159027632, "nauc_precision_at_5_diff1": 0.47218228840184173, "nauc_precision_at_5_max": 0.38251136393494295, "nauc_precision_at_5_std": 0.1659566162311902, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.3670186023127214, "nauc_recall_at_100_max": 0.4221432162608645, "nauc_recall_at_100_std": 0.41216871363930163, "nauc_recall_at_10_diff1": 0.431431424521877, "nauc_recall_at_10_max": 0.3472467649603337, "nauc_recall_at_10_std": 0.1698648922267015, "nauc_recall_at_1_diff1": 0.6112996413770887, "nauc_recall_at_1_max": 0.3748486565301294, "nauc_recall_at_1_std": 0.07824565238022668, "nauc_recall_at_20_diff1": 0.4036426371488774, "nauc_recall_at_20_max": 0.37472211286848983, "nauc_recall_at_20_std": 0.21668828011022812, "nauc_recall_at_3_diff1": 0.49000766231826093, "nauc_recall_at_3_max": 0.38209105789789033, "nauc_recall_at_3_std": 0.13673463159027632, "nauc_recall_at_5_diff1": 0.47218228840184223, "nauc_recall_at_5_max": 0.38251136393494284, "nauc_recall_at_5_std": 0.16595661623118982, "ndcg_at_1": 0.41556, "ndcg_at_10": 0.57379, "ndcg_at_100": 0.61458, "ndcg_at_1000": 0.62228, "ndcg_at_20": 0.58958, "ndcg_at_3": 0.51051, "ndcg_at_5": 0.54853, "precision_at_1": 0.41556, "precision_at_10": 0.07467, "precision_at_100": 0.00942, "precision_at_1000": 0.001, "precision_at_20": 0.04044, "precision_at_3": 0.19222, "precision_at_5": 0.13378, "recall_at_1": 0.41556, "recall_at_10": 0.74667, "recall_at_100": 0.94222, "recall_at_1000": 1.0, "recall_at_20": 0.80889, "recall_at_3": 0.57667, "recall_at_5": 0.66889 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 20.74926519393921, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.65097, "map_at_1": 0.37409, "map_at_10": 0.58031, "map_at_100": 0.59031, "map_at_1000": 0.5909, "map_at_20": 0.5873, "map_at_3": 0.53023, "map_at_5": 0.56176, "mrr_at_1": 0.5844594594594594, "mrr_at_10": 0.6746159105534107, "mrr_at_100": 0.677604599211867, "mrr_at_1000": 0.6777892991738226, "mrr_at_20": 0.6764324160197807, "mrr_at_3": 0.6573761261261261, "mrr_at_5": 0.6688626126126125, "nauc_map_at_1000_diff1": 0.30437588062635906, "nauc_map_at_1000_max": 0.4390854193045797, "nauc_map_at_1000_std": 0.14446283271908086, "nauc_map_at_100_diff1": 0.30444645347431776, "nauc_map_at_100_max": 0.43931312052110805, "nauc_map_at_100_std": 0.14435130630753498, "nauc_map_at_10_diff1": 0.30418203912088493, "nauc_map_at_10_max": 0.4366218414760577, "nauc_map_at_10_std": 0.1378511395654776, "nauc_map_at_1_diff1": 0.4087465742894733, "nauc_map_at_1_max": 0.25766176850386774, "nauc_map_at_1_std": 0.055245934886855, "nauc_map_at_20_diff1": 0.30456349353261336, "nauc_map_at_20_max": 0.4391665415776432, "nauc_map_at_20_std": 0.1421171019566587, "nauc_map_at_3_diff1": 0.31526050997633975, "nauc_map_at_3_max": 0.37018839340712634, "nauc_map_at_3_std": 0.08429713272536982, "nauc_map_at_5_diff1": 0.3069323555381713, "nauc_map_at_5_max": 0.41802740453050763, "nauc_map_at_5_std": 0.11909808743170387, "nauc_mrr_at_1000_diff1": 0.36515183653248784, "nauc_mrr_at_1000_max": 0.48358232224139225, "nauc_mrr_at_1000_std": 0.21146982434985648, "nauc_mrr_at_100_diff1": 0.36507238696029265, "nauc_mrr_at_100_max": 0.4838076218430697, "nauc_mrr_at_100_std": 0.2116090849112434, "nauc_mrr_at_10_diff1": 0.3665187269649373, "nauc_mrr_at_10_max": 0.48677214917328243, "nauc_mrr_at_10_std": 0.21174175553184899, "nauc_mrr_at_1_diff1": 0.40917227698428543, "nauc_mrr_at_1_max": 0.41704874435036265, "nauc_mrr_at_1_std": 0.16145322996342915, "nauc_mrr_at_20_diff1": 0.36586886928481055, "nauc_mrr_at_20_max": 0.48521303884222944, "nauc_mrr_at_20_std": 0.21170653563898306, "nauc_mrr_at_3_diff1": 0.3675254317772204, "nauc_mrr_at_3_max": 0.4654506632776554, "nauc_mrr_at_3_std": 0.19152422350584644, "nauc_mrr_at_5_diff1": 0.36640859610853055, "nauc_mrr_at_5_max": 0.4833872713977703, "nauc_mrr_at_5_std": 0.20526088902278128, "nauc_ndcg_at_1000_diff1": 0.2932746108713316, "nauc_ndcg_at_1000_max": 0.4890279378297942, "nauc_ndcg_at_1000_std": 0.21070421679328963, "nauc_ndcg_at_100_diff1": 0.29457269198991803, "nauc_ndcg_at_100_max": 0.4970625190600618, "nauc_ndcg_at_100_std": 0.21366100392466914, "nauc_ndcg_at_10_diff1": 0.29879033994280735, "nauc_ndcg_at_10_max": 0.4960716816141179, "nauc_ndcg_at_10_std": 0.19260925022449377, "nauc_ndcg_at_1_diff1": 0.40917227698428543, "nauc_ndcg_at_1_max": 0.41704874435036265, "nauc_ndcg_at_1_std": 0.16145322996342915, "nauc_ndcg_at_20_diff1": 0.29662820381530053, "nauc_ndcg_at_20_max": 0.49791452080604626, "nauc_ndcg_at_20_std": 0.20205453789312205, "nauc_ndcg_at_3_diff1": 0.31592108768743415, "nauc_ndcg_at_3_max": 0.4039815215837218, "nauc_ndcg_at_3_std": 0.11043780715322889, "nauc_ndcg_at_5_diff1": 0.3049782661130862, "nauc_ndcg_at_5_max": 0.4628313056055861, "nauc_ndcg_at_5_std": 0.1576020257597286, "nauc_precision_at_1000_diff1": -0.21147644159242057, "nauc_precision_at_1000_max": 0.1475822618299784, "nauc_precision_at_1000_std": 0.21746299221018425, "nauc_precision_at_100_diff1": -0.15702369326136553, "nauc_precision_at_100_max": 0.28589498077169495, "nauc_precision_at_100_std": 0.2743788996380555, "nauc_precision_at_10_diff1": -0.07291814728827116, "nauc_precision_at_10_max": 0.40126112733846653, "nauc_precision_at_10_std": 0.2539708185779428, "nauc_precision_at_1_diff1": 0.40917227698428543, "nauc_precision_at_1_max": 0.41704874435036265, "nauc_precision_at_1_std": 0.16145322996342915, "nauc_precision_at_20_diff1": -0.1024753137898999, "nauc_precision_at_20_max": 0.3563278273399836, "nauc_precision_at_20_std": 0.25837994338439596, "nauc_precision_at_3_diff1": 0.04041036837743305, "nauc_precision_at_3_max": 0.3779530001169553, "nauc_precision_at_3_std": 0.1542555866037716, "nauc_precision_at_5_diff1": -0.0149201805828455, "nauc_precision_at_5_max": 0.41480676647959547, "nauc_precision_at_5_std": 0.21427245371226203, "nauc_recall_at_1000_diff1": -0.15379458116866024, "nauc_recall_at_1000_max": 0.7442018036133632, "nauc_recall_at_1000_std": 0.7578319183569155, "nauc_recall_at_100_diff1": 0.11383562452978276, "nauc_recall_at_100_max": 0.6856710794648846, "nauc_recall_at_100_std": 0.4913330834868765, "nauc_recall_at_10_diff1": 0.2037742079617328, "nauc_recall_at_10_max": 0.596721438173084, "nauc_recall_at_10_std": 0.2821691194081661, "nauc_recall_at_1_diff1": 0.4087465742894733, "nauc_recall_at_1_max": 0.25766176850386774, "nauc_recall_at_1_std": 0.055245934886855, "nauc_recall_at_20_diff1": 0.17475890305806785, "nauc_recall_at_20_max": 0.6203287851827303, "nauc_recall_at_20_std": 0.3347291769139317, "nauc_recall_at_3_diff1": 0.26487208047481864, "nauc_recall_at_3_max": 0.3756905143946326, "nauc_recall_at_3_std": 0.08518989523236246, "nauc_recall_at_5_diff1": 0.2342458831468749, "nauc_recall_at_5_max": 0.5001635759627852, "nauc_recall_at_5_std": 0.1804476515557575, "ndcg_at_1": 0.58446, "ndcg_at_10": 0.65097, "ndcg_at_100": 0.68332, "ndcg_at_1000": 0.69489, "ndcg_at_20": 0.66748, "ndcg_at_3": 0.58339, "ndcg_at_5": 0.6209, "precision_at_1": 0.58446, "precision_at_10": 0.13801, "precision_at_100": 0.01645, "precision_at_1000": 0.00182, "precision_at_20": 0.07441, "precision_at_3": 0.3491, "precision_at_5": 0.24291, "recall_at_1": 0.37409, "recall_at_10": 0.7541, "recall_at_100": 0.87558, "recall_at_1000": 0.94941, "recall_at_20": 0.80756, "recall_at_3": 0.60173, "recall_at_5": 0.68135 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 5007.700176239014, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.36698, "map_at_1": 0.19445, "map_at_10": 0.28928, "map_at_100": 0.30966, "map_at_1000": 0.31195, "map_at_20": 0.29975, "map_at_3": 0.24441, "map_at_5": 0.27145, "mrr_at_1": 0.28169014084507044, "mrr_at_10": 0.39602615694165005, "mrr_at_100": 0.4042702928182023, "mrr_at_1000": 0.40451989571053387, "mrr_at_20": 0.40173810805814913, "mrr_at_3": 0.3482003129890454, "mrr_at_5": 0.3805946791862286, "nauc_map_at_1000_diff1": 0.4084812687318048, "nauc_map_at_1000_max": 0.29140202294213974, "nauc_map_at_1000_std": -0.10630821534195593, "nauc_map_at_100_diff1": 0.40742723296453154, "nauc_map_at_100_max": 0.28975427407538223, "nauc_map_at_100_std": -0.1106351903100884, "nauc_map_at_10_diff1": 0.4121116125647296, "nauc_map_at_10_max": 0.2894274577639449, "nauc_map_at_10_std": -0.12285121600162507, "nauc_map_at_1_diff1": 0.4761051522790686, "nauc_map_at_1_max": 0.3119569418707627, "nauc_map_at_1_std": -0.14522459005957955, "nauc_map_at_20_diff1": 0.40590849052549394, "nauc_map_at_20_max": 0.28201228278198365, "nauc_map_at_20_std": -0.1205107948403296, "nauc_map_at_3_diff1": 0.4408380154587492, "nauc_map_at_3_max": 0.30494179600376997, "nauc_map_at_3_std": -0.1382146965430355, "nauc_map_at_5_diff1": 0.41801699399093833, "nauc_map_at_5_max": 0.3025665385521116, "nauc_map_at_5_std": -0.13301925778263518, "nauc_mrr_at_1000_diff1": 0.41134115398046606, "nauc_mrr_at_1000_max": 0.3125498108612434, "nauc_mrr_at_1000_std": 0.044174739506616, "nauc_mrr_at_100_diff1": 0.4111987547500878, "nauc_mrr_at_100_max": 0.31258283685850924, "nauc_mrr_at_100_std": 0.044483320664040475, "nauc_mrr_at_10_diff1": 0.41212585672574054, "nauc_mrr_at_10_max": 0.3120882974896019, "nauc_mrr_at_10_std": 0.045639875989178576, "nauc_mrr_at_1_diff1": 0.45527743554327094, "nauc_mrr_at_1_max": 0.32212490304705027, "nauc_mrr_at_1_std": -0.03182439808400793, "nauc_mrr_at_20_diff1": 0.41136406408372556, "nauc_mrr_at_20_max": 0.3134288175021971, "nauc_mrr_at_20_std": 0.04672064188547081, "nauc_mrr_at_3_diff1": 0.44480535457371284, "nauc_mrr_at_3_max": 0.2981907381271467, "nauc_mrr_at_3_std": 0.008714004445038616, "nauc_mrr_at_5_diff1": 0.4163770936930242, "nauc_mrr_at_5_max": 0.3126931142367139, "nauc_mrr_at_5_std": 0.026265454008010135, "nauc_ndcg_at_1000_diff1": 0.38205007930521095, "nauc_ndcg_at_1000_max": 0.3092002379525429, "nauc_ndcg_at_1000_std": 0.016782053477124366, "nauc_ndcg_at_100_diff1": 0.36866426092238114, "nauc_ndcg_at_100_max": 0.2853766989148958, "nauc_ndcg_at_100_std": -0.021193940154560918, "nauc_ndcg_at_10_diff1": 0.3805717938979542, "nauc_ndcg_at_10_max": 0.272095190874917, "nauc_ndcg_at_10_std": -0.07059390730970473, "nauc_ndcg_at_1_diff1": 0.45527743554327094, "nauc_ndcg_at_1_max": 0.32212490304705027, "nauc_ndcg_at_1_std": -0.03182439808400793, "nauc_ndcg_at_20_diff1": 0.36175602830379566, "nauc_ndcg_at_20_max": 0.25454157242306663, "nauc_ndcg_at_20_std": -0.06349740896234377, "nauc_ndcg_at_3_diff1": 0.43204065984406287, "nauc_ndcg_at_3_max": 0.2867720765234763, "nauc_ndcg_at_3_std": -0.07158473283164735, "nauc_ndcg_at_5_diff1": 0.39748832233752823, "nauc_ndcg_at_5_max": 0.29677453559710093, "nauc_ndcg_at_5_std": -0.07779659140585396, "nauc_precision_at_1000_diff1": 0.021860807014730633, "nauc_precision_at_1000_max": 0.1700264921174884, "nauc_precision_at_1000_std": 0.5077164288944549, "nauc_precision_at_100_diff1": 0.0026157449960968254, "nauc_precision_at_100_max": 0.13205715256460054, "nauc_precision_at_100_std": 0.4236211729103046, "nauc_precision_at_10_diff1": 0.15964955193684693, "nauc_precision_at_10_max": 0.14733342867957075, "nauc_precision_at_10_std": 0.2156510659448347, "nauc_precision_at_1_diff1": 0.45527743554327094, "nauc_precision_at_1_max": 0.32212490304705027, "nauc_precision_at_1_std": -0.03182439808400793, "nauc_precision_at_20_diff1": 0.0701739835664581, "nauc_precision_at_20_max": 0.05933737286117831, "nauc_precision_at_20_std": 0.265477884150881, "nauc_precision_at_3_diff1": 0.37724174833700597, "nauc_precision_at_3_max": 0.24736180718912662, "nauc_precision_at_3_std": 0.03344389979387605, "nauc_precision_at_5_diff1": 0.22976789382336985, "nauc_precision_at_5_max": 0.21889460837802935, "nauc_precision_at_5_std": 0.10690416667822458, "nauc_recall_at_1000_diff1": 0.04980217844861664, "nauc_recall_at_1000_max": 0.29036332846210466, "nauc_recall_at_1000_std": 0.48176575107864594, "nauc_recall_at_100_diff1": 0.12291797709788775, "nauc_recall_at_100_max": 0.1316840073623182, "nauc_recall_at_100_std": 0.03737203628432888, "nauc_recall_at_10_diff1": 0.2600474672179701, "nauc_recall_at_10_max": 0.17919696888471337, "nauc_recall_at_10_std": -0.08346349816900192, "nauc_recall_at_1_diff1": 0.4761051522790686, "nauc_recall_at_1_max": 0.3119569418707627, "nauc_recall_at_1_std": -0.14522459005957955, "nauc_recall_at_20_diff1": 0.16428844908711895, "nauc_recall_at_20_max": 0.09292750326518946, "nauc_recall_at_20_std": -0.0829316444707036, "nauc_recall_at_3_diff1": 0.3765871397580306, "nauc_recall_at_3_max": 0.25291263723238044, "nauc_recall_at_3_std": -0.1234198085587421, "nauc_recall_at_5_diff1": 0.29894607365177606, "nauc_recall_at_5_max": 0.25233533039098743, "nauc_recall_at_5_std": -0.11015172084505716, "ndcg_at_1": 0.28169, "ndcg_at_10": 0.36698, "ndcg_at_100": 0.44518, "ndcg_at_1000": 0.47453, "ndcg_at_20": 0.39691, "ndcg_at_3": 0.29756, "ndcg_at_5": 0.33325, "precision_at_1": 0.28169, "precision_at_10": 0.09484, "precision_at_100": 0.0184, "precision_at_1000": 0.00243, "precision_at_20": 0.06009, "precision_at_3": 0.16432, "precision_at_5": 0.14085, "recall_at_1": 0.19445, "recall_at_10": 0.48427, "recall_at_100": 0.7644, "recall_at_1000": 0.92632, "recall_at_20": 0.57682, "recall_at_3": 0.29714, "recall_at_5": 0.39071 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 5739.107554197311, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.29475, "map_at_1": 0.15994, "map_at_10": 0.24016, "map_at_100": 0.25386, "map_at_1000": 0.2545, "map_at_20": 0.24889, "map_at_3": 0.21002, "map_at_5": 0.22369, "mrr_at_1": 0.171021377672209, "mrr_at_10": 0.2549004637484446, "mrr_at_100": 0.26794031250774775, "mrr_at_1000": 0.26847337014089206, "mrr_at_20": 0.2639186069456764, "mrr_at_3": 0.22525732383214553, "mrr_at_5": 0.23867775138558972, "nauc_map_at_1000_diff1": 0.2832741558778889, "nauc_map_at_1000_max": 0.30161766318839106, "nauc_map_at_1000_std": 0.04624985557103702, "nauc_map_at_100_diff1": 0.28351451499813074, "nauc_map_at_100_max": 0.3017353312134317, "nauc_map_at_100_std": 0.046188252226174134, "nauc_map_at_10_diff1": 0.2873674844384835, "nauc_map_at_10_max": 0.2961876664066545, "nauc_map_at_10_std": 0.03522190540459761, "nauc_map_at_1_diff1": 0.3532324565822577, "nauc_map_at_1_max": 0.31141505949053216, "nauc_map_at_1_std": -0.0017967262011835588, "nauc_map_at_20_diff1": 0.2834184172144873, "nauc_map_at_20_max": 0.3021113046991481, "nauc_map_at_20_std": 0.04309878962922, "nauc_map_at_3_diff1": 0.29965140104816035, "nauc_map_at_3_max": 0.2857684495528775, "nauc_map_at_3_std": 0.02136024520165048, "nauc_map_at_5_diff1": 0.2900758061449452, "nauc_map_at_5_max": 0.2943250232991835, "nauc_map_at_5_std": 0.029069130820721243, "nauc_mrr_at_1000_diff1": 0.28276618529454395, "nauc_mrr_at_1000_max": 0.2912140553907691, "nauc_mrr_at_1000_std": 0.03867367679906866, "nauc_mrr_at_100_diff1": 0.28295833248122726, "nauc_mrr_at_100_max": 0.29122806177171007, "nauc_mrr_at_100_std": 0.03849294096824469, "nauc_mrr_at_10_diff1": 0.2859160120511703, "nauc_mrr_at_10_max": 0.28592697013598267, "nauc_mrr_at_10_std": 0.03010924502414894, "nauc_mrr_at_1_diff1": 0.35540723505482436, "nauc_mrr_at_1_max": 0.29163834385425874, "nauc_mrr_at_1_std": 0.0005053015896857002, "nauc_mrr_at_20_diff1": 0.28269824716550424, "nauc_mrr_at_20_max": 0.2917184843577354, "nauc_mrr_at_20_std": 0.03713132895915048, "nauc_mrr_at_3_diff1": 0.2977194535633791, "nauc_mrr_at_3_max": 0.2784285287605545, "nauc_mrr_at_3_std": 0.018894261124117226, "nauc_mrr_at_5_diff1": 0.28836334301901656, "nauc_mrr_at_5_max": 0.28161635076441177, "nauc_mrr_at_5_std": 0.024303594069320034, "nauc_ndcg_at_1000_diff1": 0.24307593673364616, "nauc_ndcg_at_1000_max": 0.30427955135745827, "nauc_ndcg_at_1000_std": 0.09256710826288128, "nauc_ndcg_at_100_diff1": 0.2459229848753632, "nauc_ndcg_at_100_max": 0.3048337087510945, "nauc_ndcg_at_100_std": 0.09015904011893983, "nauc_ndcg_at_10_diff1": 0.2614472292832507, "nauc_ndcg_at_10_max": 0.293958878411953, "nauc_ndcg_at_10_std": 0.050346231843163544, "nauc_ndcg_at_1_diff1": 0.35540723505482436, "nauc_ndcg_at_1_max": 0.29163834385425874, "nauc_ndcg_at_1_std": 0.0005053015896857002, "nauc_ndcg_at_20_diff1": 0.2485736870889863, "nauc_ndcg_at_20_max": 0.3115124284024338, "nauc_ndcg_at_20_std": 0.07518431151250228, "nauc_ndcg_at_3_diff1": 0.28398424015189655, "nauc_ndcg_at_3_max": 0.27606340962541187, "nauc_ndcg_at_3_std": 0.02569983191968628, "nauc_ndcg_at_5_diff1": 0.2678088098509297, "nauc_ndcg_at_5_max": 0.2866300761568848, "nauc_ndcg_at_5_std": 0.03716130547179914, "nauc_precision_at_1000_diff1": -0.07783157126497497, "nauc_precision_at_1000_max": 0.13358263047313396, "nauc_precision_at_1000_std": 0.24496327497755185, "nauc_precision_at_100_diff1": 0.017689799956194612, "nauc_precision_at_100_max": 0.20403447718635603, "nauc_precision_at_100_std": 0.23161778135054684, "nauc_precision_at_10_diff1": 0.17140859062943214, "nauc_precision_at_10_max": 0.27080895548185246, "nauc_precision_at_10_std": 0.1111213686879615, "nauc_precision_at_1_diff1": 0.35540723505482436, "nauc_precision_at_1_max": 0.29163834385425874, "nauc_precision_at_1_std": 0.0005053015896857002, "nauc_precision_at_20_diff1": 0.10519977097057016, "nauc_precision_at_20_max": 0.30075017277433147, "nauc_precision_at_20_std": 0.1781342399350624, "nauc_precision_at_3_diff1": 0.24146327962265618, "nauc_precision_at_3_max": 0.2445917921216271, "nauc_precision_at_3_std": 0.04325814748725382, "nauc_precision_at_5_diff1": 0.19315517422178646, "nauc_precision_at_5_max": 0.2584422143296543, "nauc_precision_at_5_std": 0.06843086536044221, "nauc_recall_at_1000_diff1": -0.08690101389912284, "nauc_recall_at_1000_max": 0.29308317670098727, "nauc_recall_at_1000_std": 0.5138372988334909, "nauc_recall_at_100_diff1": 0.08061134719287578, "nauc_recall_at_100_max": 0.2971842687439878, "nauc_recall_at_100_std": 0.2959580996054577, "nauc_recall_at_10_diff1": 0.19210787059840054, "nauc_recall_at_10_max": 0.2878230698778371, "nauc_recall_at_10_std": 0.08857407115755006, "nauc_recall_at_1_diff1": 0.3532324565822577, "nauc_recall_at_1_max": 0.31141505949053216, "nauc_recall_at_1_std": -0.0017967262011835588, "nauc_recall_at_20_diff1": 0.14136043973459164, "nauc_recall_at_20_max": 0.34190086037125317, "nauc_recall_at_20_std": 0.17748696144241338, "nauc_recall_at_3_diff1": 0.24367381453168466, "nauc_recall_at_3_max": 0.25134463133508317, "nauc_recall_at_3_std": 0.03721642499267232, "nauc_recall_at_5_diff1": 0.2099661026221608, "nauc_recall_at_5_max": 0.26844222350013386, "nauc_recall_at_5_std": 0.05834148443622969, "ndcg_at_1": 0.17102, "ndcg_at_10": 0.29475, "ndcg_at_100": 0.35874, "ndcg_at_1000": 0.3762, "ndcg_at_20": 0.32582, "ndcg_at_3": 0.23167, "ndcg_at_5": 0.25587, "precision_at_1": 0.17102, "precision_at_10": 0.05131, "precision_at_100": 0.00853, "precision_at_1000": 0.00101, "precision_at_20": 0.03242, "precision_at_3": 0.10214, "precision_at_5": 0.07506, "recall_at_1": 0.15994, "recall_at_10": 0.4521, "recall_at_100": 0.73872, "recall_at_1000": 0.8749, "recall_at_20": 0.56968, "recall_at_3": 0.28029, "recall_at_5": 0.3369 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 85.13093566894531, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.20416, "map_at_1": 0.135, "map_at_10": 0.17894, "map_at_100": 0.18859, "map_at_1000": 0.18957, "map_at_20": 0.18362, "map_at_3": 0.1625, "map_at_5": 0.1735, "mrr_at_1": 0.135, "mrr_at_10": 0.17894047619047626, "mrr_at_100": 0.18858983326557305, "mrr_at_1000": 0.18956992828534505, "mrr_at_20": 0.1836249715197084, "mrr_at_3": 0.1625, "mrr_at_5": 0.1735, "nauc_map_at_1000_diff1": 0.28100437217058716, "nauc_map_at_1000_max": 0.2448855574076613, "nauc_map_at_1000_std": 0.022598480125543625, "nauc_map_at_100_diff1": 0.2818682896804877, "nauc_map_at_100_max": 0.24413728054766345, "nauc_map_at_100_std": 0.022166492634078164, "nauc_map_at_10_diff1": 0.28819146637250814, "nauc_map_at_10_max": 0.2467101224318175, "nauc_map_at_10_std": 0.014691870202771925, "nauc_map_at_1_diff1": 0.3201864740929537, "nauc_map_at_1_max": 0.25431057957280867, "nauc_map_at_1_std": 0.00884444880131574, "nauc_map_at_20_diff1": 0.28311700608141116, "nauc_map_at_20_max": 0.24345349652236448, "nauc_map_at_20_std": 0.018089583525167617, "nauc_map_at_3_diff1": 0.27856041375056345, "nauc_map_at_3_max": 0.2593722568859576, "nauc_map_at_3_std": 0.017636592251666545, "nauc_map_at_5_diff1": 0.28811910521569867, "nauc_map_at_5_max": 0.25953710224194465, "nauc_map_at_5_std": 0.024596692690859713, "nauc_mrr_at_1000_diff1": 0.28100437217058716, "nauc_mrr_at_1000_max": 0.2448855574076613, "nauc_mrr_at_1000_std": 0.022598480125543625, "nauc_mrr_at_100_diff1": 0.2818682896804877, "nauc_mrr_at_100_max": 0.24413728054766345, "nauc_mrr_at_100_std": 0.022166492634078164, "nauc_mrr_at_10_diff1": 0.28819146637250814, "nauc_mrr_at_10_max": 0.2467101224318175, "nauc_mrr_at_10_std": 0.014691870202771925, "nauc_mrr_at_1_diff1": 0.3201864740929537, "nauc_mrr_at_1_max": 0.25431057957280867, "nauc_mrr_at_1_std": 0.00884444880131574, "nauc_mrr_at_20_diff1": 0.28311700608141116, "nauc_mrr_at_20_max": 0.24345349652236448, "nauc_mrr_at_20_std": 0.018089583525167617, "nauc_mrr_at_3_diff1": 0.27856041375056345, "nauc_mrr_at_3_max": 0.2593722568859576, "nauc_mrr_at_3_std": 0.017636592251666545, "nauc_mrr_at_5_diff1": 0.28811910521569867, "nauc_mrr_at_5_max": 0.25953710224194465, "nauc_mrr_at_5_std": 0.024596692690859713, "nauc_ndcg_at_1000_diff1": 0.24243284227693745, "nauc_ndcg_at_1000_max": 0.2608444179252126, "nauc_ndcg_at_1000_std": 0.07399856069567344, "nauc_ndcg_at_100_diff1": 0.26198693575850857, "nauc_ndcg_at_100_max": 0.23388406296093836, "nauc_ndcg_at_100_std": 0.060943683503428735, "nauc_ndcg_at_10_diff1": 0.2857293233276634, "nauc_ndcg_at_10_max": 0.2343710688976858, "nauc_ndcg_at_10_std": 0.013418260645036978, "nauc_ndcg_at_1_diff1": 0.3201864740929537, "nauc_ndcg_at_1_max": 0.25431057957280867, "nauc_ndcg_at_1_std": 0.00884444880131574, "nauc_ndcg_at_20_diff1": 0.27044040348149756, "nauc_ndcg_at_20_max": 0.22410170866757315, "nauc_ndcg_at_20_std": 0.02671959107582927, "nauc_ndcg_at_3_diff1": 0.26546702841368613, "nauc_ndcg_at_3_max": 0.2592203101653777, "nauc_ndcg_at_3_std": 0.02261190025854227, "nauc_ndcg_at_5_diff1": 0.2839466403374545, "nauc_ndcg_at_5_max": 0.26065311707716504, "nauc_ndcg_at_5_std": 0.03478102079103265, "nauc_precision_at_1000_diff1": -0.08150677387377493, "nauc_precision_at_1000_max": 0.5487801519991193, "nauc_precision_at_1000_std": 0.516989756581121, "nauc_precision_at_100_diff1": 0.20105868139695682, "nauc_precision_at_100_max": 0.22047249593217744, "nauc_precision_at_100_std": 0.21731600522357344, "nauc_precision_at_10_diff1": 0.28463493514498056, "nauc_precision_at_10_max": 0.201947381907077, "nauc_precision_at_10_std": 0.007271518500820812, "nauc_precision_at_1_diff1": 0.3201864740929537, "nauc_precision_at_1_max": 0.25431057957280867, "nauc_precision_at_1_std": 0.00884444880131574, "nauc_precision_at_20_diff1": 0.2394694094977542, "nauc_precision_at_20_max": 0.17202101843387688, "nauc_precision_at_20_std": 0.05227644896321772, "nauc_precision_at_3_diff1": 0.23210035169215806, "nauc_precision_at_3_max": 0.25841549092766, "nauc_precision_at_3_std": 0.03581988749145475, "nauc_precision_at_5_diff1": 0.2768752365614542, "nauc_precision_at_5_max": 0.26370943202986424, "nauc_precision_at_5_std": 0.060849172127680186, "nauc_recall_at_1000_diff1": -0.08150677387377446, "nauc_recall_at_1000_max": 0.5487801519991184, "nauc_recall_at_1000_std": 0.5169897565811207, "nauc_recall_at_100_diff1": 0.20105868139695665, "nauc_recall_at_100_max": 0.22047249593217721, "nauc_recall_at_100_std": 0.21731600522357278, "nauc_recall_at_10_diff1": 0.2846349351449803, "nauc_recall_at_10_max": 0.20194738190707712, "nauc_recall_at_10_std": 0.007271518500820617, "nauc_recall_at_1_diff1": 0.3201864740929537, "nauc_recall_at_1_max": 0.25431057957280867, "nauc_recall_at_1_std": 0.00884444880131574, "nauc_recall_at_20_diff1": 0.23946940949775433, "nauc_recall_at_20_max": 0.17202101843387663, "nauc_recall_at_20_std": 0.05227644896321764, "nauc_recall_at_3_diff1": 0.23210035169215812, "nauc_recall_at_3_max": 0.2584154909276601, "nauc_recall_at_3_std": 0.03581988749145466, "nauc_recall_at_5_diff1": 0.27687523656145396, "nauc_recall_at_5_max": 0.26370943202986435, "nauc_recall_at_5_std": 0.0608491721276802, "ndcg_at_1": 0.135, "ndcg_at_10": 0.20416, "ndcg_at_100": 0.25717, "ndcg_at_1000": 0.29012, "ndcg_at_20": 0.22164, "ndcg_at_3": 0.17089, "ndcg_at_5": 0.19111, "precision_at_1": 0.135, "precision_at_10": 0.0285, "precision_at_100": 0.00545, "precision_at_1000": 0.00082, "precision_at_20": 0.01775, "precision_at_3": 0.065, "precision_at_5": 0.049, "recall_at_1": 0.135, "recall_at_10": 0.285, "recall_at_100": 0.545, "recall_at_1000": 0.82, "recall_at_20": 0.355, "recall_at_3": 0.195, "recall_at_5": 0.245 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.21749, "map_at_1": 0.15, "map_at_10": 0.19059, "map_at_100": 0.19693, "map_at_1000": 0.19789, "map_at_20": 0.1941, "map_at_3": 0.17167, "map_at_5": 0.18367, "mrr_at_1": 0.15, "mrr_at_10": 0.19058928571428577, "mrr_at_100": 0.19692929063561926, "mrr_at_1000": 0.1978932938205996, "mrr_at_20": 0.19410260521212538, "mrr_at_3": 0.17166666666666663, "mrr_at_5": 0.18366666666666664, "nauc_map_at_1000_diff1": 0.5218625299201293, "nauc_map_at_1000_max": 0.37079832709704313, "nauc_map_at_1000_std": 0.009330179985214605, "nauc_map_at_100_diff1": 0.5222179195043319, "nauc_map_at_100_max": 0.3704980486474256, "nauc_map_at_100_std": 0.009062073587896, "nauc_map_at_10_diff1": 0.5320962166651869, "nauc_map_at_10_max": 0.3681883848210749, "nauc_map_at_10_std": 0.0030555217218802477, "nauc_map_at_1_diff1": 0.6160870879951618, "nauc_map_at_1_max": 0.3586936800725733, "nauc_map_at_1_std": -0.047001310351778965, "nauc_map_at_20_diff1": 0.5246292934312703, "nauc_map_at_20_max": 0.37074115444677463, "nauc_map_at_20_std": 0.0036769872659049526, "nauc_map_at_3_diff1": 0.5500650589473496, "nauc_map_at_3_max": 0.3712530444251005, "nauc_map_at_3_std": -0.02759014986305762, "nauc_map_at_5_diff1": 0.5332310454698315, "nauc_map_at_5_max": 0.3669794799749638, "nauc_map_at_5_std": -0.01690970869827071, "nauc_mrr_at_1000_diff1": 0.5218625299201293, "nauc_mrr_at_1000_max": 0.37079832709704313, "nauc_mrr_at_1000_std": 0.009330179985214605, "nauc_mrr_at_100_diff1": 0.5222179195043319, "nauc_mrr_at_100_max": 0.3704980486474256, "nauc_mrr_at_100_std": 0.009062073587896, "nauc_mrr_at_10_diff1": 0.5320962166651869, "nauc_mrr_at_10_max": 0.3681883848210749, "nauc_mrr_at_10_std": 0.0030555217218802477, "nauc_mrr_at_1_diff1": 0.6160870879951618, "nauc_mrr_at_1_max": 0.3586936800725733, "nauc_mrr_at_1_std": -0.047001310351778965, "nauc_mrr_at_20_diff1": 0.5246292934312703, "nauc_mrr_at_20_max": 0.37074115444677463, "nauc_mrr_at_20_std": 0.0036769872659049526, "nauc_mrr_at_3_diff1": 0.5500650589473496, "nauc_mrr_at_3_max": 0.3712530444251005, "nauc_mrr_at_3_std": -0.02759014986305762, "nauc_mrr_at_5_diff1": 0.5332310454698315, "nauc_mrr_at_5_max": 0.3669794799749638, "nauc_mrr_at_5_std": -0.01690970869827071, "nauc_ndcg_at_1000_diff1": 0.45623846273278934, "nauc_ndcg_at_1000_max": 0.3816000840131088, "nauc_ndcg_at_1000_std": 0.07607811884916021, "nauc_ndcg_at_100_diff1": 0.4625424763168331, "nauc_ndcg_at_100_max": 0.3701486410068145, "nauc_ndcg_at_100_std": 0.06926544745696855, "nauc_ndcg_at_10_diff1": 0.5033392272393276, "nauc_ndcg_at_10_max": 0.3673632410841749, "nauc_ndcg_at_10_std": 0.0382763210269878, "nauc_ndcg_at_1_diff1": 0.6160870879951618, "nauc_ndcg_at_1_max": 0.3586936800725733, "nauc_ndcg_at_1_std": -0.047001310351778965, "nauc_ndcg_at_20_diff1": 0.4768549616277589, "nauc_ndcg_at_20_max": 0.3755630641369249, "nauc_ndcg_at_20_std": 0.04045437434263005, "nauc_ndcg_at_3_diff1": 0.5310771757487783, "nauc_ndcg_at_3_max": 0.37335629078608895, "nauc_ndcg_at_3_std": -0.019784525871146607, "nauc_ndcg_at_5_diff1": 0.5016003002896496, "nauc_ndcg_at_5_max": 0.36466226068816154, "nauc_ndcg_at_5_std": -0.0034444712035437866, "nauc_precision_at_1000_diff1": 0.13158213158213095, "nauc_precision_at_1000_max": 0.479479479479479, "nauc_precision_at_1000_std": 0.4335434335434332, "nauc_precision_at_100_diff1": 0.28989340439706884, "nauc_precision_at_100_max": 0.35896735509660244, "nauc_precision_at_100_std": 0.25167221852098587, "nauc_precision_at_10_diff1": 0.4385222425145065, "nauc_precision_at_10_max": 0.36427719409382264, "nauc_precision_at_10_std": 0.12881182313814574, "nauc_precision_at_1_diff1": 0.6160870879951618, "nauc_precision_at_1_max": 0.3586936800725733, "nauc_precision_at_1_std": -0.047001310351778965, "nauc_precision_at_20_diff1": 0.35300215189209067, "nauc_precision_at_20_max": 0.38977457618222877, "nauc_precision_at_20_std": 0.13397496457250815, "nauc_precision_at_3_diff1": 0.48228931223764915, "nauc_precision_at_3_max": 0.37834194381659647, "nauc_precision_at_3_std": 0.0008879560865352576, "nauc_precision_at_5_diff1": 0.4219908571928522, "nauc_precision_at_5_max": 0.35699964274517537, "nauc_precision_at_5_std": 0.030096896257573547, "nauc_recall_at_1000_diff1": 0.13158213158213183, "nauc_recall_at_1000_max": 0.47947947947947983, "nauc_recall_at_1000_std": 0.43354343354343294, "nauc_recall_at_100_diff1": 0.28989340439706857, "nauc_recall_at_100_max": 0.3589673550966022, "nauc_recall_at_100_std": 0.25167221852098587, "nauc_recall_at_10_diff1": 0.438522242514506, "nauc_recall_at_10_max": 0.3642771940938223, "nauc_recall_at_10_std": 0.12881182313814532, "nauc_recall_at_1_diff1": 0.6160870879951618, "nauc_recall_at_1_max": 0.3586936800725733, "nauc_recall_at_1_std": -0.047001310351778965, "nauc_recall_at_20_diff1": 0.35300215189209033, "nauc_recall_at_20_max": 0.3897745761822284, "nauc_recall_at_20_std": 0.13397496457250832, "nauc_recall_at_3_diff1": 0.4822893122376495, "nauc_recall_at_3_max": 0.3783419438165969, "nauc_recall_at_3_std": 0.0008879560865352862, "nauc_recall_at_5_diff1": 0.4219908571928522, "nauc_recall_at_5_max": 0.35699964274517537, "nauc_recall_at_5_std": 0.030096896257573457, "ndcg_at_1": 0.15, "ndcg_at_10": 0.21749, "ndcg_at_100": 0.25537, "ndcg_at_1000": 0.28733, "ndcg_at_20": 0.23103, "ndcg_at_3": 0.17893, "ndcg_at_5": 0.20108, "precision_at_1": 0.15, "precision_at_10": 0.0305, "precision_at_100": 0.005, "precision_at_1000": 0.00077, "precision_at_20": 0.018, "precision_at_3": 0.06667, "precision_at_5": 0.051, "recall_at_1": 0.15, "recall_at_10": 0.305, "recall_at_100": 0.5, "recall_at_1000": 0.765, "recall_at_20": 0.36, "recall_at_3": 0.2, "recall_at_5": 0.255 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 0.5960137844085693, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.69212, "map_at_1": 0.51948, "map_at_10": 0.62536, "map_at_100": 0.63083, "map_at_1000": 0.63083, "map_at_20": 0.6294, "map_at_3": 0.58658, "map_at_5": 0.60087, "mrr_at_1": 0.5194805194805194, "mrr_at_10": 0.6253555967841682, "mrr_at_100": 0.6308311676381538, "mrr_at_1000": 0.6308311676381538, "mrr_at_20": 0.629400037192245, "mrr_at_3": 0.5865800865800868, "mrr_at_5": 0.600865800865801, "nauc_map_at_1000_diff1": 0.5904540290551432, "nauc_map_at_1000_max": 0.1705997040097517, "nauc_map_at_1000_std": -0.07632464300606626, "nauc_map_at_100_diff1": 0.5904540290551432, "nauc_map_at_100_max": 0.1705997040097517, "nauc_map_at_100_std": -0.07632464300606626, "nauc_map_at_10_diff1": 0.5810613919569926, "nauc_map_at_10_max": 0.165288215922278, "nauc_map_at_10_std": -0.07756012340475889, "nauc_map_at_1_diff1": 0.6191580005022813, "nauc_map_at_1_max": 0.17833218944439605, "nauc_map_at_1_std": -0.09107595840831198, "nauc_map_at_20_diff1": 0.5884560790280374, "nauc_map_at_20_max": 0.17054182222192502, "nauc_map_at_20_std": -0.07343464303680558, "nauc_map_at_3_diff1": 0.6090085648305908, "nauc_map_at_3_max": 0.19453931071063257, "nauc_map_at_3_std": -0.06795354806260433, "nauc_map_at_5_diff1": 0.5957441004664911, "nauc_map_at_5_max": 0.17021323168919492, "nauc_map_at_5_std": -0.08837204248824297, "nauc_mrr_at_1000_diff1": 0.5904540290551432, "nauc_mrr_at_1000_max": 0.1705997040097517, "nauc_mrr_at_1000_std": -0.07632464300606626, "nauc_mrr_at_100_diff1": 0.5904540290551432, "nauc_mrr_at_100_max": 0.1705997040097517, "nauc_mrr_at_100_std": -0.07632464300606626, "nauc_mrr_at_10_diff1": 0.5810613919569926, "nauc_mrr_at_10_max": 0.165288215922278, "nauc_mrr_at_10_std": -0.07756012340475889, "nauc_mrr_at_1_diff1": 0.6191580005022813, "nauc_mrr_at_1_max": 0.17833218944439605, "nauc_mrr_at_1_std": -0.09107595840831198, "nauc_mrr_at_20_diff1": 0.5884560790280374, "nauc_mrr_at_20_max": 0.17054182222192502, "nauc_mrr_at_20_std": -0.07343464303680558, "nauc_mrr_at_3_diff1": 0.6090085648305908, "nauc_mrr_at_3_max": 0.19453931071063257, "nauc_mrr_at_3_std": -0.06795354806260433, "nauc_mrr_at_5_diff1": 0.5957441004664911, "nauc_mrr_at_5_max": 0.17021323168919492, "nauc_mrr_at_5_std": -0.08837204248824297, "nauc_ndcg_at_1000_diff1": 0.5832707848906321, "nauc_ndcg_at_1000_max": 0.165379178631264, "nauc_ndcg_at_1000_std": -0.07407865099638525, "nauc_ndcg_at_100_diff1": 0.5832707848906321, "nauc_ndcg_at_100_max": 0.165379178631264, "nauc_ndcg_at_100_std": -0.07407865099638525, "nauc_ndcg_at_10_diff1": 0.5414290115530259, "nauc_ndcg_at_10_max": 0.1416500440311118, "nauc_ndcg_at_10_std": -0.07114074202061504, "nauc_ndcg_at_1_diff1": 0.6191580005022813, "nauc_ndcg_at_1_max": 0.17833218944439605, "nauc_ndcg_at_1_std": -0.09107595840831198, "nauc_ndcg_at_20_diff1": 0.5701308148180418, "nauc_ndcg_at_20_max": 0.16441559243571785, "nauc_ndcg_at_20_std": -0.05461644829084728, "nauc_ndcg_at_3_diff1": 0.606960998871192, "nauc_ndcg_at_3_max": 0.19845868240059197, "nauc_ndcg_at_3_std": -0.06157170762240057, "nauc_ndcg_at_5_diff1": 0.5820742198586897, "nauc_ndcg_at_5_max": 0.15190511865290174, "nauc_ndcg_at_5_std": -0.10189164153807106, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.13376374524588608, "nauc_precision_at_10_max": -0.08900787547969854, "nauc_precision_at_10_std": 0.007388603659539997, "nauc_precision_at_1_diff1": 0.6191580005022813, "nauc_precision_at_1_max": 0.17833218944439605, "nauc_precision_at_1_std": -0.09107595840831198, "nauc_precision_at_20_diff1": 0.22227871181478126, "nauc_precision_at_20_max": 0.1301604202575047, "nauc_precision_at_20_std": 0.4627549073276905, "nauc_precision_at_3_diff1": 0.6007076905121849, "nauc_precision_at_3_max": 0.21083305884213124, "nauc_precision_at_3_std": -0.041012324527044994, "nauc_precision_at_5_diff1": 0.5262303207900256, "nauc_precision_at_5_max": 0.07201028555000641, "nauc_precision_at_5_std": -0.16349277608293908, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.13376374524588425, "nauc_recall_at_10_max": -0.08900787547969922, "nauc_recall_at_10_std": 0.007388603659539275, "nauc_recall_at_1_diff1": 0.6191580005022813, "nauc_recall_at_1_max": 0.17833218944439605, "nauc_recall_at_1_std": -0.09107595840831198, "nauc_recall_at_20_diff1": 0.22227871181477624, "nauc_recall_at_20_max": 0.13016042025750219, "nauc_recall_at_20_std": 0.4627549073276888, "nauc_recall_at_3_diff1": 0.6007076905121846, "nauc_recall_at_3_max": 0.21083305884213052, "nauc_recall_at_3_std": -0.04101232452704562, "nauc_recall_at_5_diff1": 0.5262303207900242, "nauc_recall_at_5_max": 0.0720102855500049, "nauc_recall_at_5_std": -0.16349277608294063, "ndcg_at_1": 0.51948, "ndcg_at_10": 0.69212, "ndcg_at_100": 0.71389, "ndcg_at_1000": 0.71389, "ndcg_at_20": 0.70581, "ndcg_at_3": 0.60931, "ndcg_at_5": 0.63556, "precision_at_1": 0.51948, "precision_at_10": 0.09091, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04805, "precision_at_3": 0.22511, "precision_at_5": 0.14805, "recall_at_1": 0.51948, "recall_at_10": 0.90909, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.96104, "recall_at_3": 0.67532, "recall_at_5": 0.74026 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 13.350108623504639, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.27961, "map_at_1": 0.16284, "map_at_10": 0.23808, "map_at_100": 0.25039, "map_at_1000": 0.25208, "map_at_20": 0.24277, "map_at_3": 0.21111, "map_at_5": 0.22575, "mrr_at_1": 0.22935779816513763, "mrr_at_10": 0.28000218435998236, "mrr_at_100": 0.2899681942422444, "mrr_at_1000": 0.2908827646075394, "mrr_at_20": 0.2841006872311016, "mrr_at_3": 0.2599388379204893, "mrr_at_5": 0.2701070336391437, "nauc_map_at_1000_diff1": 0.4214155757637042, "nauc_map_at_1000_max": 0.23550404240073378, "nauc_map_at_1000_std": -0.19082976954284206, "nauc_map_at_100_diff1": 0.4211832097906042, "nauc_map_at_100_max": 0.23510982131035243, "nauc_map_at_100_std": -0.19112693116109628, "nauc_map_at_10_diff1": 0.42491758063652063, "nauc_map_at_10_max": 0.23377930427998797, "nauc_map_at_10_std": -0.19215120573331193, "nauc_map_at_1_diff1": 0.49508894472645903, "nauc_map_at_1_max": 0.21200110623130414, "nauc_map_at_1_std": -0.16844296221187263, "nauc_map_at_20_diff1": 0.42293177758553196, "nauc_map_at_20_max": 0.23276599466098036, "nauc_map_at_20_std": -0.19372732595358427, "nauc_map_at_3_diff1": 0.4391343104593348, "nauc_map_at_3_max": 0.2417851004872019, "nauc_map_at_3_std": -0.18125653720105458, "nauc_map_at_5_diff1": 0.4290256686549028, "nauc_map_at_5_max": 0.23842288309676038, "nauc_map_at_5_std": -0.18596085451344127, "nauc_mrr_at_1000_diff1": 0.4339308328759577, "nauc_mrr_at_1000_max": 0.23052512484010262, "nauc_mrr_at_1000_std": -0.212731603813038, "nauc_mrr_at_100_diff1": 0.4336604877516317, "nauc_mrr_at_100_max": 0.2306537175917452, "nauc_mrr_at_100_std": -0.21254894869650906, "nauc_mrr_at_10_diff1": 0.43651305267929, "nauc_mrr_at_10_max": 0.227822365310929, "nauc_mrr_at_10_std": -0.2157135899647547, "nauc_mrr_at_1_diff1": 0.4779620643287542, "nauc_mrr_at_1_max": 0.251649276278888, "nauc_mrr_at_1_std": -0.2048346984958598, "nauc_mrr_at_20_diff1": 0.4345827687718731, "nauc_mrr_at_20_max": 0.22710692119239329, "nauc_mrr_at_20_std": -0.21598034826399987, "nauc_mrr_at_3_diff1": 0.4432033544402822, "nauc_mrr_at_3_max": 0.23478668550178164, "nauc_mrr_at_3_std": -0.21090263477523583, "nauc_mrr_at_5_diff1": 0.4377288835397934, "nauc_mrr_at_5_max": 0.23140937321857954, "nauc_mrr_at_5_std": -0.21287979163279244, "nauc_ndcg_at_1000_diff1": 0.4068622705032811, "nauc_ndcg_at_1000_max": 0.2362348940380059, "nauc_ndcg_at_1000_std": -0.19296527934696367, "nauc_ndcg_at_100_diff1": 0.3954240766125709, "nauc_ndcg_at_100_max": 0.23784433171873276, "nauc_ndcg_at_100_std": -0.18906422912246504, "nauc_ndcg_at_10_diff1": 0.41014774605731696, "nauc_ndcg_at_10_max": 0.2198857724678368, "nauc_ndcg_at_10_std": -0.20708819629924666, "nauc_ndcg_at_1_diff1": 0.47163133066509216, "nauc_ndcg_at_1_max": 0.24733983446418387, "nauc_ndcg_at_1_std": -0.20786115681188994, "nauc_ndcg_at_20_diff1": 0.40431506990372906, "nauc_ndcg_at_20_max": 0.21529530278570344, "nauc_ndcg_at_20_std": -0.21132627060527578, "nauc_ndcg_at_3_diff1": 0.4303012919013767, "nauc_ndcg_at_3_max": 0.23437231251585022, "nauc_ndcg_at_3_std": -0.19030054330589352, "nauc_ndcg_at_5_diff1": 0.4187378333666249, "nauc_ndcg_at_5_max": 0.22940131285810786, "nauc_ndcg_at_5_std": -0.19693896936170513, "nauc_precision_at_1000_diff1": -0.001251120709928727, "nauc_precision_at_1000_max": 0.1408970362643088, "nauc_precision_at_1000_std": 0.006519872959041239, "nauc_precision_at_100_diff1": 0.11767955785983104, "nauc_precision_at_100_max": 0.21887911815573843, "nauc_precision_at_100_std": -0.06463301427250882, "nauc_precision_at_10_diff1": 0.2566620319642375, "nauc_precision_at_10_max": 0.19484544050157246, "nauc_precision_at_10_std": -0.20866446979622968, "nauc_precision_at_1_diff1": 0.47163133066509216, "nauc_precision_at_1_max": 0.24733983446418387, "nauc_precision_at_1_std": -0.20786115681188994, "nauc_precision_at_20_diff1": 0.22333251838170926, "nauc_precision_at_20_max": 0.16681753275124725, "nauc_precision_at_20_std": -0.20802685122153342, "nauc_precision_at_3_diff1": 0.3222914812111711, "nauc_precision_at_3_max": 0.25677046804837805, "nauc_precision_at_3_std": -0.1971581914636415, "nauc_precision_at_5_diff1": 0.29348323841539453, "nauc_precision_at_5_max": 0.2282958661223689, "nauc_precision_at_5_std": -0.20241263344179536, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.2608078592841538, "nauc_recall_at_100_max": 0.2544835738790891, "nauc_recall_at_100_std": -0.12233805846621212, "nauc_recall_at_10_diff1": 0.34864154401985253, "nauc_recall_at_10_max": 0.179329618645416, "nauc_recall_at_10_std": -0.214286105443704, "nauc_recall_at_1_diff1": 0.49508894472645903, "nauc_recall_at_1_max": 0.21200110623130414, "nauc_recall_at_1_std": -0.16844296221187263, "nauc_recall_at_20_diff1": 0.3301717345342511, "nauc_recall_at_20_max": 0.16383958678364147, "nauc_recall_at_20_std": -0.22609209261541532, "nauc_recall_at_3_diff1": 0.40434566144584727, "nauc_recall_at_3_max": 0.2192093599150267, "nauc_recall_at_3_std": -0.18206254394044408, "nauc_recall_at_5_diff1": 0.3756229824887185, "nauc_recall_at_5_max": 0.20583583054847093, "nauc_recall_at_5_std": -0.19328068106011442, "ndcg_at_1": 0.23089, "ndcg_at_10": 0.27961, "ndcg_at_100": 0.34801, "ndcg_at_1000": 0.39067, "ndcg_at_20": 0.2953, "ndcg_at_3": 0.23296, "ndcg_at_5": 0.25242, "precision_at_1": 0.23089, "precision_at_10": 0.05749, "precision_at_100": 0.01095, "precision_at_1000": 0.00156, "precision_at_20": 0.03364, "precision_at_3": 0.12538, "precision_at_5": 0.09113, "recall_at_1": 0.16284, "recall_at_10": 0.36318, "recall_at_100": 0.67806, "recall_at_1000": 1.0, "recall_at_20": 0.41723, "recall_at_3": 0.2395, "recall_at_5": 0.28619 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.1836, "map_at_1": 0.08759, "map_at_10": 0.14392, "map_at_100": 0.15878, "map_at_1000": 0.16133, "map_at_20": 0.15078, "map_at_3": 0.12174, "map_at_5": 0.13335, "mrr_at_1": 0.15137614678899083, "mrr_at_10": 0.20600759671860588, "mrr_at_100": 0.2157491629989898, "mrr_at_1000": 0.21708576403816518, "mrr_at_20": 0.21068637974846255, "mrr_at_3": 0.18603465851172268, "mrr_at_5": 0.19589704383282366, "nauc_map_at_1000_diff1": 0.3323837585455884, "nauc_map_at_1000_max": 0.14303424039470392, "nauc_map_at_1000_std": -0.019450779897357603, "nauc_map_at_100_diff1": 0.33238455031441677, "nauc_map_at_100_max": 0.1420330658354713, "nauc_map_at_100_std": -0.020605907819890888, "nauc_map_at_10_diff1": 0.33895092286545214, "nauc_map_at_10_max": 0.14064565959195846, "nauc_map_at_10_std": -0.03210504177476584, "nauc_map_at_1_diff1": 0.4190187012199572, "nauc_map_at_1_max": 0.15134792821012716, "nauc_map_at_1_std": -0.04697241262575727, "nauc_map_at_20_diff1": 0.3338336698929536, "nauc_map_at_20_max": 0.13920479083694737, "nauc_map_at_20_std": -0.027631683392214904, "nauc_map_at_3_diff1": 0.3629507638191468, "nauc_map_at_3_max": 0.16126202951274413, "nauc_map_at_3_std": -0.02681843523976257, "nauc_map_at_5_diff1": 0.3465080154020162, "nauc_map_at_5_max": 0.1419158062834336, "nauc_map_at_5_std": -0.03513882062880755, "nauc_mrr_at_1000_diff1": 0.3165662315528989, "nauc_mrr_at_1000_max": 0.10440424338855178, "nauc_mrr_at_1000_std": -0.03650895468546071, "nauc_mrr_at_100_diff1": 0.3162235285389014, "nauc_mrr_at_100_max": 0.10451230472237277, "nauc_mrr_at_100_std": -0.03624916629178117, "nauc_mrr_at_10_diff1": 0.3181664664556047, "nauc_mrr_at_10_max": 0.0959551433749828, "nauc_mrr_at_10_std": -0.04665592310685699, "nauc_mrr_at_1_diff1": 0.401585102884664, "nauc_mrr_at_1_max": 0.1326972078174377, "nauc_mrr_at_1_std": -0.05207300122030268, "nauc_mrr_at_20_diff1": 0.3163325090017384, "nauc_mrr_at_20_max": 0.10160977429351967, "nauc_mrr_at_20_std": -0.0400209730294806, "nauc_mrr_at_3_diff1": 0.3359875539979813, "nauc_mrr_at_3_max": 0.11955416883057066, "nauc_mrr_at_3_std": -0.03476235016257288, "nauc_mrr_at_5_diff1": 0.32252659352575813, "nauc_mrr_at_5_max": 0.10046345533893442, "nauc_mrr_at_5_std": -0.045529883380858664, "nauc_ndcg_at_1000_diff1": 0.2931526330921674, "nauc_ndcg_at_1000_max": 0.14343734872583952, "nauc_ndcg_at_1000_std": 0.01531556734099745, "nauc_ndcg_at_100_diff1": 0.28793660618185096, "nauc_ndcg_at_100_max": 0.14080051473350946, "nauc_ndcg_at_100_std": 0.01939473783704219, "nauc_ndcg_at_10_diff1": 0.30274490180366037, "nauc_ndcg_at_10_max": 0.11207325709525649, "nauc_ndcg_at_10_std": -0.037138795222154526, "nauc_ndcg_at_1_diff1": 0.401585102884664, "nauc_ndcg_at_1_max": 0.1326972078174377, "nauc_ndcg_at_1_std": -0.05207300122030268, "nauc_ndcg_at_20_diff1": 0.2924643035488461, "nauc_ndcg_at_20_max": 0.11576477873988932, "nauc_ndcg_at_20_std": -0.021628226067522374, "nauc_ndcg_at_3_diff1": 0.3381531011543047, "nauc_ndcg_at_3_max": 0.1483085845798095, "nauc_ndcg_at_3_std": -0.02020927671457784, "nauc_ndcg_at_5_diff1": 0.31836406385775057, "nauc_ndcg_at_5_max": 0.11859396452974193, "nauc_ndcg_at_5_std": -0.0398300926768745, "nauc_precision_at_1000_diff1": 0.01897509499484508, "nauc_precision_at_1000_max": 0.1272058228791155, "nauc_precision_at_1000_std": 0.15277529372849502, "nauc_precision_at_100_diff1": 0.11516779002559407, "nauc_precision_at_100_max": 0.14853048761591342, "nauc_precision_at_100_std": 0.14492486506295213, "nauc_precision_at_10_diff1": 0.21251133142026507, "nauc_precision_at_10_max": 0.0749955278240198, "nauc_precision_at_10_std": -0.021005872981786113, "nauc_precision_at_1_diff1": 0.401585102884664, "nauc_precision_at_1_max": 0.1326972078174377, "nauc_precision_at_1_std": -0.05207300122030268, "nauc_precision_at_20_diff1": 0.17961078306722833, "nauc_precision_at_20_max": 0.07124907605202703, "nauc_precision_at_20_std": 0.018615517877496685, "nauc_precision_at_3_diff1": 0.2861280133449851, "nauc_precision_at_3_max": 0.13997648217997413, "nauc_precision_at_3_std": -0.0045893982638058025, "nauc_precision_at_5_diff1": 0.23365603097222445, "nauc_precision_at_5_max": 0.07201622385482957, "nauc_precision_at_5_std": -0.037993705245915846, "nauc_recall_at_1000_diff1": 0.15524576455683287, "nauc_recall_at_1000_max": 0.3777839196641194, "nauc_recall_at_1000_std": 0.37249201078606803, "nauc_recall_at_100_diff1": 0.18838026703717084, "nauc_recall_at_100_max": 0.16705539370590955, "nauc_recall_at_100_std": 0.13071431855660154, "nauc_recall_at_10_diff1": 0.2259821503446232, "nauc_recall_at_10_max": 0.07734043989687381, "nauc_recall_at_10_std": -0.03896354343756332, "nauc_recall_at_1_diff1": 0.4190187012199572, "nauc_recall_at_1_max": 0.15134792821012716, "nauc_recall_at_1_std": -0.04697241262575727, "nauc_recall_at_20_diff1": 0.20419825567192415, "nauc_recall_at_20_max": 0.09223973482857481, "nauc_recall_at_20_std": -0.0010681016439537414, "nauc_recall_at_3_diff1": 0.3084967319100879, "nauc_recall_at_3_max": 0.15205899480484425, "nauc_recall_at_3_std": -0.016393168209809336, "nauc_recall_at_5_diff1": 0.27003956265333967, "nauc_recall_at_5_max": 0.09709950457509019, "nauc_recall_at_5_std": -0.04449121016608349, "ndcg_at_1": 0.15138, "ndcg_at_10": 0.1836, "ndcg_at_100": 0.25502, "ndcg_at_1000": 0.31552, "ndcg_at_20": 0.20472, "ndcg_at_3": 0.15104, "ndcg_at_5": 0.16047, "precision_at_1": 0.15138, "precision_at_10": 0.05, "precision_at_100": 0.01125, "precision_at_1000": 0.0019, "precision_at_20": 0.03287, "precision_at_3": 0.09837, "precision_at_5": 0.07615, "recall_at_1": 0.08759, "recall_at_10": 0.23626, "recall_at_100": 0.53598, "recall_at_1000": 0.96376, "recall_at_20": 0.30341, "recall_at_3": 0.14518, "recall_at_5": 0.1764 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.13208, "map_at_1": 0.05869, "map_at_10": 0.10211, "map_at_100": 0.11408, "map_at_1000": 0.11686, "map_at_20": 0.10762, "map_at_3": 0.08392, "map_at_5": 0.09327, "mrr_at_1": 0.08957654723127036, "mrr_at_10": 0.1315637764334833, "mrr_at_100": 0.14213263563289533, "mrr_at_1000": 0.14378913837763863, "mrr_at_20": 0.13754496155982124, "mrr_at_3": 0.11427795874049947, "mrr_at_5": 0.122828447339848, "nauc_map_at_1000_diff1": 0.21560922855620007, "nauc_map_at_1000_max": 0.19561583483776016, "nauc_map_at_1000_std": -0.08618144873074722, "nauc_map_at_100_diff1": 0.21524058773854432, "nauc_map_at_100_max": 0.19431477445319648, "nauc_map_at_100_std": -0.08676346645432348, "nauc_map_at_10_diff1": 0.21422950284520126, "nauc_map_at_10_max": 0.19885627844058584, "nauc_map_at_10_std": -0.09151289850316742, "nauc_map_at_1_diff1": 0.3538976408299393, "nauc_map_at_1_max": 0.23850048629684373, "nauc_map_at_1_std": -0.06908623232628476, "nauc_map_at_20_diff1": 0.21693053893599712, "nauc_map_at_20_max": 0.19072627724433514, "nauc_map_at_20_std": -0.09462057211855156, "nauc_map_at_3_diff1": 0.24575719829487122, "nauc_map_at_3_max": 0.2063557681660242, "nauc_map_at_3_std": -0.08734603877301897, "nauc_map_at_5_diff1": 0.22781099520034429, "nauc_map_at_5_max": 0.20325106012746608, "nauc_map_at_5_std": -0.09807300505119161, "nauc_mrr_at_1000_diff1": 0.24945463442366622, "nauc_mrr_at_1000_max": 0.19118543371597183, "nauc_mrr_at_1000_std": -0.09983062611158049, "nauc_mrr_at_100_diff1": 0.24891535170964876, "nauc_mrr_at_100_max": 0.19110971717545924, "nauc_mrr_at_100_std": -0.09915074234087376, "nauc_mrr_at_10_diff1": 0.24941942040095283, "nauc_mrr_at_10_max": 0.1919868661708655, "nauc_mrr_at_10_std": -0.10432343680920428, "nauc_mrr_at_1_diff1": 0.3376404249877776, "nauc_mrr_at_1_max": 0.21146912532087958, "nauc_mrr_at_1_std": -0.12859947809219066, "nauc_mrr_at_20_diff1": 0.24938581241814445, "nauc_mrr_at_20_max": 0.18738807630665177, "nauc_mrr_at_20_std": -0.10443144645359773, "nauc_mrr_at_3_diff1": 0.2957268343270363, "nauc_mrr_at_3_max": 0.21399609341337603, "nauc_mrr_at_3_std": -0.10290389227513133, "nauc_mrr_at_5_diff1": 0.26434737424521426, "nauc_mrr_at_5_max": 0.2040101668123861, "nauc_mrr_at_5_std": -0.1039001693931301, "nauc_ndcg_at_1000_diff1": 0.1987438343299619, "nauc_ndcg_at_1000_max": 0.1924612516252132, "nauc_ndcg_at_1000_std": -0.06685289419687576, "nauc_ndcg_at_100_diff1": 0.1838404743726864, "nauc_ndcg_at_100_max": 0.1848659194171432, "nauc_ndcg_at_100_std": -0.05906364514428352, "nauc_ndcg_at_10_diff1": 0.18242977368934568, "nauc_ndcg_at_10_max": 0.18123187262386636, "nauc_ndcg_at_10_std": -0.0917795439461141, "nauc_ndcg_at_1_diff1": 0.3376404249877776, "nauc_ndcg_at_1_max": 0.21146912532087958, "nauc_ndcg_at_1_std": -0.12859947809219066, "nauc_ndcg_at_20_diff1": 0.18990385592671677, "nauc_ndcg_at_20_max": 0.16079696958979647, "nauc_ndcg_at_20_std": -0.09728697788201912, "nauc_ndcg_at_3_diff1": 0.2517920264760545, "nauc_ndcg_at_3_max": 0.20648883858241032, "nauc_ndcg_at_3_std": -0.08518847040847785, "nauc_ndcg_at_5_diff1": 0.21041924820244198, "nauc_ndcg_at_5_max": 0.19530185319315377, "nauc_ndcg_at_5_std": -0.09941341807558633, "nauc_precision_at_1000_diff1": 0.03513800086318182, "nauc_precision_at_1000_max": 0.1255489280293357, "nauc_precision_at_1000_std": 0.09356806987379775, "nauc_precision_at_100_diff1": 0.09880744710648004, "nauc_precision_at_100_max": 0.19393587366376208, "nauc_precision_at_100_std": 0.049293428739512556, "nauc_precision_at_10_diff1": 0.10686341522204414, "nauc_precision_at_10_max": 0.15110569691521045, "nauc_precision_at_10_std": -0.1006454762936947, "nauc_precision_at_1_diff1": 0.3376404249877776, "nauc_precision_at_1_max": 0.21146912532087958, "nauc_precision_at_1_std": -0.12859947809219066, "nauc_precision_at_20_diff1": 0.1380424489755519, "nauc_precision_at_20_max": 0.11046126212037552, "nauc_precision_at_20_std": -0.10974323144810633, "nauc_precision_at_3_diff1": 0.21406006876255954, "nauc_precision_at_3_max": 0.19917630927345578, "nauc_precision_at_3_std": -0.10243766390060259, "nauc_precision_at_5_diff1": 0.1586259842576393, "nauc_precision_at_5_max": 0.1825289897633917, "nauc_precision_at_5_std": -0.1173922587116099, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.09876684110940794, "nauc_recall_at_100_max": 0.152107667117311, "nauc_recall_at_100_std": 0.0035501001547093764, "nauc_recall_at_10_diff1": 0.09242510783637202, "nauc_recall_at_10_max": 0.1455731287806074, "nauc_recall_at_10_std": -0.07703531519035574, "nauc_recall_at_1_diff1": 0.3538976408299393, "nauc_recall_at_1_max": 0.23850048629684373, "nauc_recall_at_1_std": -0.06908623232628476, "nauc_recall_at_20_diff1": 0.11102806918077908, "nauc_recall_at_20_max": 0.09446327110879274, "nauc_recall_at_20_std": -0.08584496849591045, "nauc_recall_at_3_diff1": 0.20353526483578888, "nauc_recall_at_3_max": 0.1895890169712471, "nauc_recall_at_3_std": -0.0771361002484376, "nauc_recall_at_5_diff1": 0.13559382090130945, "nauc_recall_at_5_max": 0.1663957063451556, "nauc_recall_at_5_std": -0.09905717747836443, "ndcg_at_1": 0.08958, "ndcg_at_10": 0.13208, "ndcg_at_100": 0.19471, "ndcg_at_1000": 0.26676, "ndcg_at_20": 0.15152, "ndcg_at_3": 0.09829, "ndcg_at_5": 0.11236, "precision_at_1": 0.08958, "precision_at_10": 0.03257, "precision_at_100": 0.00827, "precision_at_1000": 0.00167, "precision_at_20": 0.02191, "precision_at_3": 0.05917, "precision_at_5": 0.04723, "recall_at_1": 0.05869, "recall_at_10": 0.19125, "recall_at_100": 0.46301, "recall_at_1000": 1.0, "recall_at_20": 0.25857, "recall_at_3": 0.10437, "recall_at_5": 0.1374 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/model_meta.json ================================================ {"name": "jhgan/ko-sroberta-multitask", "revision": "no_revision_available", "release_date": null, "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "loader": null} ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 41.10066056251526, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.76104, "map_at_1": 0.59649, "map_at_10": 0.70926, "map_at_100": 0.71425, "map_at_1000": 0.71425, "map_at_20": 0.71406, "map_at_3": 0.69006, "map_at_5": 0.70058, "mrr_at_1": 0.5964912280701754, "mrr_at_10": 0.7092592592592594, "mrr_at_100": 0.7142522444779211, "mrr_at_1000": 0.7142522444779211, "mrr_at_20": 0.7140560706350181, "mrr_at_3": 0.6900584795321638, "mrr_at_5": 0.7005847953216374, "nauc_map_at_1000_diff1": 0.5239958070884113, "nauc_map_at_1000_max": 0.3096268445996021, "nauc_map_at_1000_std": -0.09860198242005956, "nauc_map_at_100_diff1": 0.5239958070884113, "nauc_map_at_100_max": 0.3096268445996021, "nauc_map_at_100_std": -0.09860198242005956, "nauc_map_at_10_diff1": 0.5257485700639195, "nauc_map_at_10_max": 0.3004730187357267, "nauc_map_at_10_std": -0.10570448325456437, "nauc_map_at_1_diff1": 0.5270656332483367, "nauc_map_at_1_max": 0.2997382273345491, "nauc_map_at_1_std": -0.08460522678016429, "nauc_map_at_20_diff1": 0.52420233068558, "nauc_map_at_20_max": 0.3102036982325753, "nauc_map_at_20_std": -0.09806415639379118, "nauc_map_at_3_diff1": 0.5253492012809279, "nauc_map_at_3_max": 0.3144296142174382, "nauc_map_at_3_std": -0.1121978163326179, "nauc_map_at_5_diff1": 0.5269886990139881, "nauc_map_at_5_max": 0.31118773097573915, "nauc_map_at_5_std": -0.09722073597757512, "nauc_mrr_at_1000_diff1": 0.5239958070884113, "nauc_mrr_at_1000_max": 0.3096268445996021, "nauc_mrr_at_1000_std": -0.09860198242005956, "nauc_mrr_at_100_diff1": 0.5239958070884113, "nauc_mrr_at_100_max": 0.3096268445996021, "nauc_mrr_at_100_std": -0.09860198242005956, "nauc_mrr_at_10_diff1": 0.5257485700639195, "nauc_mrr_at_10_max": 0.3004730187357267, "nauc_mrr_at_10_std": -0.10570448325456437, "nauc_mrr_at_1_diff1": 0.5270656332483367, "nauc_mrr_at_1_max": 0.2997382273345491, "nauc_mrr_at_1_std": -0.08460522678016429, "nauc_mrr_at_20_diff1": 0.52420233068558, "nauc_mrr_at_20_max": 0.3102036982325753, "nauc_mrr_at_20_std": -0.09806415639379118, "nauc_mrr_at_3_diff1": 0.5253492012809279, "nauc_mrr_at_3_max": 0.3144296142174382, "nauc_mrr_at_3_std": -0.1121978163326179, "nauc_mrr_at_5_diff1": 0.5269886990139881, "nauc_mrr_at_5_max": 0.31118773097573915, "nauc_mrr_at_5_std": -0.09722073597757512, "nauc_ndcg_at_1000_diff1": 0.5257901238239717, "nauc_ndcg_at_1000_max": 0.3126792178348435, "nauc_ndcg_at_1000_std": -0.0967700334246146, "nauc_ndcg_at_100_diff1": 0.5257901238239717, "nauc_ndcg_at_100_max": 0.3126792178348435, "nauc_ndcg_at_100_std": -0.0967700334246146, "nauc_ndcg_at_10_diff1": 0.533232658128605, "nauc_ndcg_at_10_max": 0.28188484529812124, "nauc_ndcg_at_10_std": -0.1187167303529919, "nauc_ndcg_at_1_diff1": 0.5270656332483367, "nauc_ndcg_at_1_max": 0.2997382273345491, "nauc_ndcg_at_1_std": -0.08460522678016429, "nauc_ndcg_at_20_diff1": 0.5294503341807054, "nauc_ndcg_at_20_max": 0.3227331547901982, "nauc_ndcg_at_20_std": -0.0874396499178423, "nauc_ndcg_at_3_diff1": 0.5304384782368492, "nauc_ndcg_at_3_max": 0.32058457218018066, "nauc_ndcg_at_3_std": -0.12308251548793804, "nauc_ndcg_at_5_diff1": 0.5345686226982408, "nauc_ndcg_at_5_max": 0.31397010350723664, "nauc_ndcg_at_5_std": -0.09551990374421933, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.6096638560941005, "nauc_precision_at_10_max": 0.07507825251175508, "nauc_precision_at_10_std": -0.2479500686091432, "nauc_precision_at_1_diff1": 0.5270656332483367, "nauc_precision_at_1_max": 0.2997382273345491, "nauc_precision_at_1_std": -0.08460522678016429, "nauc_precision_at_20_diff1": 0.7773798018086919, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 0.539967216074137, "nauc_precision_at_3_diff1": 0.554747649217572, "nauc_precision_at_3_max": 0.3469283752456029, "nauc_precision_at_3_std": -0.16937634560817524, "nauc_precision_at_5_diff1": 0.5794871062690461, "nauc_precision_at_5_max": 0.3264628971418981, "nauc_precision_at_5_std": -0.0814431864235027, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.6096638560941018, "nauc_recall_at_10_max": 0.07507825251175473, "nauc_recall_at_10_std": -0.24795006860914381, "nauc_recall_at_1_diff1": 0.5270656332483367, "nauc_recall_at_1_max": 0.2997382273345491, "nauc_recall_at_1_std": -0.08460522678016429, "nauc_recall_at_20_diff1": 0.7773798018086994, "nauc_recall_at_20_max": 1.0, "nauc_recall_at_20_std": 0.5399672160741448, "nauc_recall_at_3_diff1": 0.5547476492175731, "nauc_recall_at_3_max": 0.34692837524560327, "nauc_recall_at_3_std": -0.1693763456081746, "nauc_recall_at_5_diff1": 0.5794871062690465, "nauc_recall_at_5_max": 0.3264628971418974, "nauc_recall_at_5_std": -0.08144318642350515, "ndcg_at_1": 0.59649, "ndcg_at_10": 0.76104, "ndcg_at_100": 0.77992, "ndcg_at_1000": 0.77992, "ndcg_at_20": 0.77722, "ndcg_at_3": 0.72222, "ndcg_at_5": 0.74072, "precision_at_1": 0.59649, "precision_at_10": 0.09211, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04912, "precision_at_3": 0.27193, "precision_at_5": 0.17193, "recall_at_1": 0.59649, "recall_at_10": 0.92105, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.98246, "recall_at_3": 0.81579, "recall_at_5": 0.85965 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 57.27869248390198, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.91203, "map_at_1": 0.85222, "map_at_10": 0.89425, "map_at_100": 0.89538, "map_at_1000": 0.89543, "map_at_20": 0.89498, "map_at_3": 0.88704, "map_at_5": 0.89204, "mrr_at_1": 0.8522222222222222, "mrr_at_10": 0.8942500000000002, "mrr_at_100": 0.8953839155802804, "mrr_at_1000": 0.8954270182792048, "mrr_at_20": 0.8949815662732332, "mrr_at_3": 0.8870370370370371, "mrr_at_5": 0.8920370370370372, "nauc_map_at_1000_diff1": 0.9020917639244876, "nauc_map_at_1000_max": 0.8337947139919248, "nauc_map_at_1000_std": -0.13942249660085268, "nauc_map_at_100_diff1": 0.9021144491033539, "nauc_map_at_100_max": 0.8338652988739003, "nauc_map_at_100_std": -0.1391935604293546, "nauc_map_at_10_diff1": 0.9021967915206724, "nauc_map_at_10_max": 0.8348616633465842, "nauc_map_at_10_std": -0.1370753656652194, "nauc_map_at_1_diff1": 0.9145342061089405, "nauc_map_at_1_max": 0.8262609677253842, "nauc_map_at_1_std": -0.1763595581390236, "nauc_map_at_20_diff1": 0.902147551209322, "nauc_map_at_20_max": 0.8341413002501225, "nauc_map_at_20_std": -0.1390126071598066, "nauc_map_at_3_diff1": 0.8945147579693029, "nauc_map_at_3_max": 0.8294443132624945, "nauc_map_at_3_std": -0.139357733175914, "nauc_map_at_5_diff1": 0.9012312541128398, "nauc_map_at_5_max": 0.8347399861132424, "nauc_map_at_5_std": -0.13882816867958603, "nauc_mrr_at_1000_diff1": 0.9020917639244876, "nauc_mrr_at_1000_max": 0.8337947139919248, "nauc_mrr_at_1000_std": -0.13942249660085268, "nauc_mrr_at_100_diff1": 0.9021144491033539, "nauc_mrr_at_100_max": 0.8338652988739003, "nauc_mrr_at_100_std": -0.1391935604293546, "nauc_mrr_at_10_diff1": 0.9021967915206724, "nauc_mrr_at_10_max": 0.8348616633465842, "nauc_mrr_at_10_std": -0.1370753656652194, "nauc_mrr_at_1_diff1": 0.9145342061089405, "nauc_mrr_at_1_max": 0.8262609677253842, "nauc_mrr_at_1_std": -0.1763595581390236, "nauc_mrr_at_20_diff1": 0.902147551209322, "nauc_mrr_at_20_max": 0.8341413002501225, "nauc_mrr_at_20_std": -0.1390126071598066, "nauc_mrr_at_3_diff1": 0.8945147579693029, "nauc_mrr_at_3_max": 0.8294443132624945, "nauc_mrr_at_3_std": -0.139357733175914, "nauc_mrr_at_5_diff1": 0.9012312541128398, "nauc_mrr_at_5_max": 0.8347399861132424, "nauc_mrr_at_5_std": -0.13882816867958603, "nauc_ndcg_at_1000_diff1": 0.9015621894371831, "nauc_ndcg_at_1000_max": 0.8368285497658826, "nauc_ndcg_at_1000_std": -0.13046781497976448, "nauc_ndcg_at_100_diff1": 0.9021755169669389, "nauc_ndcg_at_100_max": 0.8386747960567419, "nauc_ndcg_at_100_std": -0.12445994533332687, "nauc_ndcg_at_10_diff1": 0.9023787269981611, "nauc_ndcg_at_10_max": 0.8439996596181932, "nauc_ndcg_at_10_std": -0.11394157814254431, "nauc_ndcg_at_1_diff1": 0.9145342061089405, "nauc_ndcg_at_1_max": 0.8262609677253842, "nauc_ndcg_at_1_std": -0.1763595581390236, "nauc_ndcg_at_20_diff1": 0.9022049378154124, "nauc_ndcg_at_20_max": 0.8410091236224608, "nauc_ndcg_at_20_std": -0.12123038116426346, "nauc_ndcg_at_3_diff1": 0.8865681442555119, "nauc_ndcg_at_3_max": 0.8313441701334897, "nauc_ndcg_at_3_std": -0.12506625223359308, "nauc_ndcg_at_5_diff1": 0.8995328660323624, "nauc_ndcg_at_5_max": 0.8424065352194856, "nauc_ndcg_at_5_std": -0.12271550695612689, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9564270152505334, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.4064737005913334, "nauc_precision_at_10_diff1": 0.9137877373171471, "nauc_precision_at_10_max": 0.9339402427637699, "nauc_precision_at_10_std": 0.1071739807033904, "nauc_precision_at_1_diff1": 0.9145342061089405, "nauc_precision_at_1_max": 0.8262609677253842, "nauc_precision_at_1_std": -0.1763595581390236, "nauc_precision_at_20_diff1": 0.9172502334267043, "nauc_precision_at_20_max": 0.9344070961718046, "nauc_precision_at_20_std": 0.11559290382819244, "nauc_precision_at_3_diff1": 0.8521582992171257, "nauc_precision_at_3_max": 0.8398836457660015, "nauc_precision_at_3_std": -0.06337714573008646, "nauc_precision_at_5_diff1": 0.8934018051665107, "nauc_precision_at_5_max": 0.8921049901442025, "nauc_precision_at_5_std": -0.026247536051460216, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9564270152505339, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.4064737005913486, "nauc_recall_at_10_diff1": 0.9137877373171508, "nauc_recall_at_10_max": 0.9339402427637726, "nauc_recall_at_10_std": 0.10717398070338875, "nauc_recall_at_1_diff1": 0.9145342061089405, "nauc_recall_at_1_max": 0.8262609677253842, "nauc_recall_at_1_std": -0.1763595581390236, "nauc_recall_at_20_diff1": 0.9172502334267004, "nauc_recall_at_20_max": 0.9344070961717991, "nauc_recall_at_20_std": 0.11559290382819323, "nauc_recall_at_3_diff1": 0.8521582992171224, "nauc_recall_at_3_max": 0.839883645765999, "nauc_recall_at_3_std": -0.06337714573008725, "nauc_recall_at_5_diff1": 0.8934018051665108, "nauc_recall_at_5_max": 0.8921049901442053, "nauc_recall_at_5_std": -0.026247536051455612, "ndcg_at_1": 0.85222, "ndcg_at_10": 0.91203, "ndcg_at_100": 0.9177, "ndcg_at_1000": 0.9186, "ndcg_at_20": 0.91479, "ndcg_at_3": 0.89756, "ndcg_at_5": 0.90665, "precision_at_1": 0.85222, "precision_at_10": 0.09667, "precision_at_100": 0.00993, "precision_at_1000": 0.001, "precision_at_20": 0.04889, "precision_at_3": 0.30926, "precision_at_5": 0.19, "recall_at_1": 0.85222, "recall_at_10": 0.96667, "recall_at_100": 0.99333, "recall_at_1000": 1.0, "recall_at_20": 0.97778, "recall_at_3": 0.92778, "recall_at_5": 0.95 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.9109, "map_at_1": 0.84222, "map_at_10": 0.88998, "map_at_100": 0.89082, "map_at_1000": 0.89085, "map_at_20": 0.89051, "map_at_3": 0.88093, "map_at_5": 0.88681, "mrr_at_1": 0.8422222222222222, "mrr_at_10": 0.8899757495590832, "mrr_at_100": 0.890822716078263, "mrr_at_1000": 0.890851807543744, "mrr_at_20": 0.8905073602629074, "mrr_at_3": 0.8809259259259262, "mrr_at_5": 0.8868148148148152, "nauc_map_at_1000_diff1": 0.8793289073281108, "nauc_map_at_1000_max": 0.7645882618060904, "nauc_map_at_1000_std": -0.20608621707079874, "nauc_map_at_100_diff1": 0.8793283088179963, "nauc_map_at_100_max": 0.7646368045228797, "nauc_map_at_100_std": -0.20601745471532495, "nauc_map_at_10_diff1": 0.8784631740060715, "nauc_map_at_10_max": 0.7645963195365795, "nauc_map_at_10_std": -0.20413278583470437, "nauc_map_at_1_diff1": 0.8904469551525532, "nauc_map_at_1_max": 0.7414745652684358, "nauc_map_at_1_std": -0.2645336390381344, "nauc_map_at_20_diff1": 0.8791660884950596, "nauc_map_at_20_max": 0.7646298819142401, "nauc_map_at_20_std": -0.20490132009022927, "nauc_map_at_3_diff1": 0.8767390557772566, "nauc_map_at_3_max": 0.7669129126156815, "nauc_map_at_3_std": -0.20528469001054214, "nauc_map_at_5_diff1": 0.8791163628524895, "nauc_map_at_5_max": 0.767040239258405, "nauc_map_at_5_std": -0.20443632861492111, "nauc_mrr_at_1000_diff1": 0.8793289073281108, "nauc_mrr_at_1000_max": 0.7645882618060904, "nauc_mrr_at_1000_std": -0.20608621707079874, "nauc_mrr_at_100_diff1": 0.8793283088179963, "nauc_mrr_at_100_max": 0.7646368045228797, "nauc_mrr_at_100_std": -0.20601745471532495, "nauc_mrr_at_10_diff1": 0.8784631740060715, "nauc_mrr_at_10_max": 0.7645963195365795, "nauc_mrr_at_10_std": -0.20413278583470437, "nauc_mrr_at_1_diff1": 0.8904469551525532, "nauc_mrr_at_1_max": 0.7414745652684358, "nauc_mrr_at_1_std": -0.2645336390381344, "nauc_mrr_at_20_diff1": 0.8791660884950596, "nauc_mrr_at_20_max": 0.7646298819142401, "nauc_mrr_at_20_std": -0.20490132009022927, "nauc_mrr_at_3_diff1": 0.8767390557772566, "nauc_mrr_at_3_max": 0.7669129126156815, "nauc_mrr_at_3_std": -0.20528469001054214, "nauc_mrr_at_5_diff1": 0.8791163628524895, "nauc_mrr_at_5_max": 0.767040239258405, "nauc_mrr_at_5_std": -0.20443632861492111, "nauc_ndcg_at_1000_diff1": 0.8781291299916675, "nauc_ndcg_at_1000_max": 0.7685134207651526, "nauc_ndcg_at_1000_std": -0.19370618376382695, "nauc_ndcg_at_100_diff1": 0.8780237661206428, "nauc_ndcg_at_100_max": 0.7700858390514594, "nauc_ndcg_at_100_std": -0.1906833920754185, "nauc_ndcg_at_10_diff1": 0.8742177171336183, "nauc_ndcg_at_10_max": 0.7708266681095408, "nauc_ndcg_at_10_std": -0.17522437091850582, "nauc_ndcg_at_1_diff1": 0.8904469551525532, "nauc_ndcg_at_1_max": 0.7414745652684358, "nauc_ndcg_at_1_std": -0.2645336390381344, "nauc_ndcg_at_20_diff1": 0.8767375659903873, "nauc_ndcg_at_20_max": 0.7702559043816767, "nauc_ndcg_at_20_std": -0.1802098019957146, "nauc_ndcg_at_3_diff1": 0.8723314505390427, "nauc_ndcg_at_3_max": 0.7773943742824141, "nauc_ndcg_at_3_std": -0.1809413234611165, "nauc_ndcg_at_5_diff1": 0.8767596497220361, "nauc_ndcg_at_5_max": 0.7782724004984772, "nauc_ndcg_at_5_std": -0.17687341992420602, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8627450980391947, "nauc_precision_at_100_max": 0.9477124183006403, "nauc_precision_at_100_std": 0.17665732959849742, "nauc_precision_at_10_diff1": 0.8208131737543488, "nauc_precision_at_10_max": 0.8309141838553583, "nauc_precision_at_10_std": 0.18203038791273896, "nauc_precision_at_1_diff1": 0.8904469551525532, "nauc_precision_at_1_max": 0.7414745652684358, "nauc_precision_at_1_std": -0.2645336390381344, "nauc_precision_at_20_diff1": 0.8403361344537821, "nauc_precision_at_20_max": 0.838811079987551, "nauc_precision_at_20_std": 0.2273887332710908, "nauc_precision_at_3_diff1": 0.8531675296381216, "nauc_precision_at_3_max": 0.8244277508983406, "nauc_precision_at_3_std": -0.07173980703392634, "nauc_precision_at_5_diff1": 0.8637548042472811, "nauc_precision_at_5_max": 0.8511823333984757, "nauc_precision_at_5_std": 0.003984539552250845, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8627450980392323, "nauc_recall_at_100_max": 0.9477124183006633, "nauc_recall_at_100_std": 0.17665732959849603, "nauc_recall_at_10_diff1": 0.8208131737543504, "nauc_recall_at_10_max": 0.8309141838553625, "nauc_recall_at_10_std": 0.1820303879127387, "nauc_recall_at_1_diff1": 0.8904469551525532, "nauc_recall_at_1_max": 0.7414745652684358, "nauc_recall_at_1_std": -0.2645336390381344, "nauc_recall_at_20_diff1": 0.8403361344537867, "nauc_recall_at_20_max": 0.8388110799875537, "nauc_recall_at_20_std": 0.22738873327109257, "nauc_recall_at_3_diff1": 0.8531675296381188, "nauc_recall_at_3_max": 0.8244277508983399, "nauc_recall_at_3_std": -0.07173980703392417, "nauc_recall_at_5_diff1": 0.8637548042472826, "nauc_recall_at_5_max": 0.8511823333984765, "nauc_recall_at_5_std": 0.003984539552254944, "ndcg_at_1": 0.84222, "ndcg_at_10": 0.9109, "ndcg_at_100": 0.91498, "ndcg_at_1000": 0.91571, "ndcg_at_20": 0.91286, "ndcg_at_3": 0.89274, "ndcg_at_5": 0.9033, "precision_at_1": 0.84222, "precision_at_10": 0.09756, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "precision_at_20": 0.04917, "precision_at_3": 0.30889, "precision_at_5": 0.19044, "recall_at_1": 0.84222, "recall_at_10": 0.97556, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "recall_at_20": 0.98333, "recall_at_3": 0.92667, "recall_at_5": 0.95222 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.87617, "map_at_1": 0.80333, "map_at_10": 0.85398, "map_at_100": 0.8561, "map_at_1000": 0.85615, "map_at_20": 0.8554, "map_at_3": 0.84444, "map_at_5": 0.85078, "mrr_at_1": 0.8033333333333333, "mrr_at_10": 0.8539775132275135, "mrr_at_100": 0.8560964976509596, "mrr_at_1000": 0.8561516994848337, "mrr_at_20": 0.8554018242965613, "mrr_at_3": 0.8444444444444446, "mrr_at_5": 0.850777777777778, "nauc_map_at_1000_diff1": 0.856390879963964, "nauc_map_at_1000_max": 0.7431504658889044, "nauc_map_at_1000_std": -0.03859630853653877, "nauc_map_at_100_diff1": 0.8563949613083448, "nauc_map_at_100_max": 0.7432512772690879, "nauc_map_at_100_std": -0.0383305630011282, "nauc_map_at_10_diff1": 0.8555895378186136, "nauc_map_at_10_max": 0.7446655324173755, "nauc_map_at_10_std": -0.03790518210814961, "nauc_map_at_1_diff1": 0.870348447180458, "nauc_map_at_1_max": 0.7173073562324179, "nauc_map_at_1_std": -0.09652623923962796, "nauc_map_at_20_diff1": 0.8566700428604839, "nauc_map_at_20_max": 0.7443445689375534, "nauc_map_at_20_std": -0.03640168482969062, "nauc_map_at_3_diff1": 0.8514236742689241, "nauc_map_at_3_max": 0.7390426052889324, "nauc_map_at_3_std": -0.039633762417797626, "nauc_map_at_5_diff1": 0.8528231089195483, "nauc_map_at_5_max": 0.7411566867600459, "nauc_map_at_5_std": -0.04003385146956129, "nauc_mrr_at_1000_diff1": 0.856390879963964, "nauc_mrr_at_1000_max": 0.7431504658889044, "nauc_mrr_at_1000_std": -0.03859630853653877, "nauc_mrr_at_100_diff1": 0.8563949613083448, "nauc_mrr_at_100_max": 0.7432512772690879, "nauc_mrr_at_100_std": -0.0383305630011282, "nauc_mrr_at_10_diff1": 0.8555895378186136, "nauc_mrr_at_10_max": 0.7446655324173755, "nauc_mrr_at_10_std": -0.03790518210814961, "nauc_mrr_at_1_diff1": 0.870348447180458, "nauc_mrr_at_1_max": 0.7173073562324179, "nauc_mrr_at_1_std": -0.09652623923962796, "nauc_mrr_at_20_diff1": 0.8566700428604839, "nauc_mrr_at_20_max": 0.7443445689375534, "nauc_mrr_at_20_std": -0.03640168482969062, "nauc_mrr_at_3_diff1": 0.8514236742689241, "nauc_mrr_at_3_max": 0.7390426052889324, "nauc_mrr_at_3_std": -0.039633762417797626, "nauc_mrr_at_5_diff1": 0.8528231089195483, "nauc_mrr_at_5_max": 0.7411566867600459, "nauc_mrr_at_5_std": -0.04003385146956129, "nauc_ndcg_at_1000_diff1": 0.8551568401764285, "nauc_ndcg_at_1000_max": 0.7496569905288137, "nauc_ndcg_at_1000_std": -0.024187597548567658, "nauc_ndcg_at_100_diff1": 0.8549084962983187, "nauc_ndcg_at_100_max": 0.7522749195374062, "nauc_ndcg_at_100_std": -0.016738618446300355, "nauc_ndcg_at_10_diff1": 0.8533642901306143, "nauc_ndcg_at_10_max": 0.7624729405925561, "nauc_ndcg_at_10_std": -0.007143010527758984, "nauc_ndcg_at_1_diff1": 0.870348447180458, "nauc_ndcg_at_1_max": 0.7173073562324179, "nauc_ndcg_at_1_std": -0.09652623923962796, "nauc_ndcg_at_20_diff1": 0.8573483667416811, "nauc_ndcg_at_20_max": 0.7609706238162958, "nauc_ndcg_at_20_std": -0.0017325455148094052, "nauc_ndcg_at_3_diff1": 0.8433223914045559, "nauc_ndcg_at_3_max": 0.7467564185298718, "nauc_ndcg_at_3_std": -0.01610650093526701, "nauc_ndcg_at_5_diff1": 0.8454774209016609, "nauc_ndcg_at_5_max": 0.7515843383252132, "nauc_ndcg_at_5_std": -0.015396728354773289, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8231792717086875, "nauc_precision_at_100_max": 0.9836601307189621, "nauc_precision_at_100_std": 0.6563375350140266, "nauc_precision_at_10_diff1": 0.8464052287581659, "nauc_precision_at_10_max": 0.9017833800186726, "nauc_precision_at_10_std": 0.22231559290382727, "nauc_precision_at_1_diff1": 0.870348447180458, "nauc_precision_at_1_max": 0.7173073562324179, "nauc_precision_at_1_std": -0.09652623923962796, "nauc_precision_at_20_diff1": 0.8883928571428547, "nauc_precision_at_20_max": 0.9554300887021441, "nauc_precision_at_20_std": 0.40242763772175133, "nauc_precision_at_3_diff1": 0.8102429610569564, "nauc_precision_at_3_max": 0.7776040378049395, "nauc_precision_at_3_std": 0.07962294027413584, "nauc_precision_at_5_diff1": 0.807296251834067, "nauc_precision_at_5_max": 0.8067627050820332, "nauc_precision_at_5_std": 0.11367213552087513, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8231792717086877, "nauc_recall_at_100_max": 0.9836601307189647, "nauc_recall_at_100_std": 0.656337535014012, "nauc_recall_at_10_diff1": 0.8464052287581701, "nauc_recall_at_10_max": 0.9017833800186756, "nauc_recall_at_10_std": 0.22231559290383007, "nauc_recall_at_1_diff1": 0.870348447180458, "nauc_recall_at_1_max": 0.7173073562324179, "nauc_recall_at_1_std": -0.09652623923962796, "nauc_recall_at_20_diff1": 0.888392857142854, "nauc_recall_at_20_max": 0.9554300887021479, "nauc_recall_at_20_std": 0.4024276377217557, "nauc_recall_at_3_diff1": 0.8102429610569536, "nauc_recall_at_3_max": 0.7776040378049385, "nauc_recall_at_3_std": 0.07962294027413575, "nauc_recall_at_5_diff1": 0.8072962518340645, "nauc_recall_at_5_max": 0.8067627050820312, "nauc_recall_at_5_std": 0.11367213552087382, "ndcg_at_1": 0.80333, "ndcg_at_10": 0.87617, "ndcg_at_100": 0.88626, "ndcg_at_1000": 0.88746, "ndcg_at_20": 0.88128, "ndcg_at_3": 0.85733, "ndcg_at_5": 0.86875, "precision_at_1": 0.80333, "precision_at_10": 0.09444, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "precision_at_20": 0.04822, "precision_at_3": 0.29815, "precision_at_5": 0.18444, "recall_at_1": 0.80333, "recall_at_10": 0.94444, "recall_at_100": 0.99111, "recall_at_1000": 1.0, "recall_at_20": 0.96444, "recall_at_3": 0.89444, "recall_at_5": 0.92222 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 223.86375641822815, "kg_co2_emissions": null, "mteb_version": "1.19.4", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.79807, "map_at_1": 0.49486, "map_at_10": 0.7529, "map_at_100": 0.75814, "map_at_1000": 0.75847, "map_at_20": 0.7557, "map_at_3": 0.71293, "map_at_5": 0.74013, "mrr_at_1": 0.7567567567567568, "mrr_at_10": 0.8125087140712136, "mrr_at_100": 0.8138507914768789, "mrr_at_1000": 0.8140220957363901, "mrr_at_20": 0.8129562972490598, "mrr_at_3": 0.8051801801801798, "mrr_at_5": 0.8092342342342338, "nauc_map_at_1000_diff1": 0.5254239984606249, "nauc_map_at_1000_max": 0.6076357258092393, "nauc_map_at_1000_std": 0.16831170782698113, "nauc_map_at_100_diff1": 0.5256160280279527, "nauc_map_at_100_max": 0.6081500700165519, "nauc_map_at_100_std": 0.16879223667707566, "nauc_map_at_10_diff1": 0.5255581538726148, "nauc_map_at_10_max": 0.6060662610276891, "nauc_map_at_10_std": 0.15876076676111253, "nauc_map_at_1_diff1": 0.5677261561403673, "nauc_map_at_1_max": 0.28508354972378114, "nauc_map_at_1_std": 0.007280734732612492, "nauc_map_at_20_diff1": 0.5238966366214702, "nauc_map_at_20_max": 0.6066736461021203, "nauc_map_at_20_std": 0.16629182338592297, "nauc_map_at_3_diff1": 0.5256290575169561, "nauc_map_at_3_max": 0.5526053340090126, "nauc_map_at_3_std": 0.10745590181814504, "nauc_map_at_5_diff1": 0.5239196813503921, "nauc_map_at_5_max": 0.5879505989347982, "nauc_map_at_5_std": 0.12928009325744128, "nauc_mrr_at_1000_diff1": 0.6045317079492099, "nauc_mrr_at_1000_max": 0.6818506916734917, "nauc_mrr_at_1000_std": 0.24684168932238623, "nauc_mrr_at_100_diff1": 0.6044974400624296, "nauc_mrr_at_100_max": 0.6821498379087556, "nauc_mrr_at_100_std": 0.2473342205845724, "nauc_mrr_at_10_diff1": 0.6052759747606176, "nauc_mrr_at_10_max": 0.6836968268951451, "nauc_mrr_at_10_std": 0.24698918584519347, "nauc_mrr_at_1_diff1": 0.6255977425590653, "nauc_mrr_at_1_max": 0.6085394560621038, "nauc_mrr_at_1_std": 0.1886186491267808, "nauc_mrr_at_20_diff1": 0.6048472825511458, "nauc_mrr_at_20_max": 0.6833297533840166, "nauc_mrr_at_20_std": 0.24723589434904442, "nauc_mrr_at_3_diff1": 0.6053260554638301, "nauc_mrr_at_3_max": 0.6787525540078904, "nauc_mrr_at_3_std": 0.22958233180443266, "nauc_mrr_at_5_diff1": 0.6068903852966356, "nauc_mrr_at_5_max": 0.6817668951172293, "nauc_mrr_at_5_std": 0.23765401001481948, "nauc_ndcg_at_1000_diff1": 0.5341293298685438, "nauc_ndcg_at_1000_max": 0.6606052848336096, "nauc_ndcg_at_1000_std": 0.24354374496497427, "nauc_ndcg_at_100_diff1": 0.538109131538343, "nauc_ndcg_at_100_max": 0.6738354047774087, "nauc_ndcg_at_100_std": 0.2607595035941354, "nauc_ndcg_at_10_diff1": 0.5371973679564122, "nauc_ndcg_at_10_max": 0.6675617107893623, "nauc_ndcg_at_10_std": 0.2275146878229136, "nauc_ndcg_at_1_diff1": 0.6255977425590653, "nauc_ndcg_at_1_max": 0.6085394560621038, "nauc_ndcg_at_1_std": 0.1886186491267808, "nauc_ndcg_at_20_diff1": 0.5329460641472812, "nauc_ndcg_at_20_max": 0.6703160563440603, "nauc_ndcg_at_20_std": 0.24665172910639852, "nauc_ndcg_at_3_diff1": 0.5339934869900126, "nauc_ndcg_at_3_max": 0.6016187617062881, "nauc_ndcg_at_3_std": 0.14319284440610835, "nauc_ndcg_at_5_diff1": 0.5349249568810696, "nauc_ndcg_at_5_max": 0.6360683663995738, "nauc_ndcg_at_5_std": 0.17195232578453204, "nauc_precision_at_1000_diff1": -0.3601704644023477, "nauc_precision_at_1000_max": 0.006183725884216995, "nauc_precision_at_1000_std": 0.1607732772589181, "nauc_precision_at_100_diff1": -0.24363085502433576, "nauc_precision_at_100_max": 0.15825274923442087, "nauc_precision_at_100_std": 0.2571706793459895, "nauc_precision_at_10_diff1": -0.15330981722695072, "nauc_precision_at_10_max": 0.2705490424818717, "nauc_precision_at_10_std": 0.2420369043349826, "nauc_precision_at_1_diff1": 0.6255977425590653, "nauc_precision_at_1_max": 0.6085394560621038, "nauc_precision_at_1_std": 0.1886186491267808, "nauc_precision_at_20_diff1": -0.1818736784887125, "nauc_precision_at_20_max": 0.23765136032825976, "nauc_precision_at_20_std": 0.26033290546867716, "nauc_precision_at_3_diff1": -0.018528841312410074, "nauc_precision_at_3_max": 0.37026854207409304, "nauc_precision_at_3_std": 0.18878760818610282, "nauc_precision_at_5_diff1": -0.09487222809490398, "nauc_precision_at_5_max": 0.30644126690762064, "nauc_precision_at_5_std": 0.186628279906234, "nauc_recall_at_1000_diff1": 0.269619819244052, "nauc_recall_at_1000_max": 0.9169502263402253, "nauc_recall_at_1000_std": 0.9105457073478973, "nauc_recall_at_100_diff1": 0.4796192580516074, "nauc_recall_at_100_max": 0.9314620341130307, "nauc_recall_at_100_std": 0.736430564541567, "nauc_recall_at_10_diff1": 0.49088061271022754, "nauc_recall_at_10_max": 0.7916733282343257, "nauc_recall_at_10_std": 0.3813267309899073, "nauc_recall_at_1_diff1": 0.5677261561403673, "nauc_recall_at_1_max": 0.28508354972378114, "nauc_recall_at_1_std": 0.007280734732612492, "nauc_recall_at_20_diff1": 0.4703645522060505, "nauc_recall_at_20_max": 0.8261306027221935, "nauc_recall_at_20_std": 0.4863410668980038, "nauc_recall_at_3_diff1": 0.4974352305750985, "nauc_recall_at_3_max": 0.6019940531052395, "nauc_recall_at_3_std": 0.1376677180157724, "nauc_recall_at_5_diff1": 0.49113118706864095, "nauc_recall_at_5_max": 0.6838318771771169, "nauc_recall_at_5_std": 0.2000724466684479, "ndcg_at_1": 0.75676, "ndcg_at_10": 0.79807, "ndcg_at_100": 0.81503, "ndcg_at_1000": 0.82257, "ndcg_at_20": 0.80396, "ndcg_at_3": 0.75288, "ndcg_at_5": 0.77926, "precision_at_1": 0.75676, "precision_at_10": 0.15912, "precision_at_100": 0.01738, "precision_at_1000": 0.00185, "precision_at_20": 0.08176, "precision_at_3": 0.4482, "precision_at_5": 0.29662, "recall_at_1": 0.49486, "recall_at_10": 0.85983, "recall_at_100": 0.92379, "recall_at_1000": 0.97179, "recall_at_20": 0.87769, "recall_at_3": 0.76166, "recall_at_5": 0.81523 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 25491.57020497322, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.63716, "map_at_1": 0.35427, "map_at_10": 0.53931, "map_at_100": 0.56379, "map_at_1000": 0.56456, "map_at_20": 0.55514, "map_at_3": 0.47065, "map_at_5": 0.50777, "mrr_at_1": 0.5868544600938967, "mrr_at_10": 0.7000670690811536, "mrr_at_100": 0.7034974166221164, "mrr_at_1000": 0.7035448268034526, "mrr_at_20": 0.7023941629575432, "mrr_at_3": 0.6776212832550861, "mrr_at_5": 0.6907668231611894, "nauc_map_at_1000_diff1": 0.3538606032876468, "nauc_map_at_1000_max": 0.30569293033048445, "nauc_map_at_1000_std": 0.05336084598734498, "nauc_map_at_100_diff1": 0.3539638864548362, "nauc_map_at_100_max": 0.3052910650189014, "nauc_map_at_100_std": 0.053094415110458495, "nauc_map_at_10_diff1": 0.36670236809047213, "nauc_map_at_10_max": 0.29209555278562255, "nauc_map_at_10_std": 0.027150365173378658, "nauc_map_at_1_diff1": 0.5002958094670257, "nauc_map_at_1_max": 0.23186362968659052, "nauc_map_at_1_std": -0.07863597756490925, "nauc_map_at_20_diff1": 0.35482695302369316, "nauc_map_at_20_max": 0.29979355976346383, "nauc_map_at_20_std": 0.045918334252931396, "nauc_map_at_3_diff1": 0.3936843797578524, "nauc_map_at_3_max": 0.25104131301314014, "nauc_map_at_3_std": -0.042697716050571405, "nauc_map_at_5_diff1": 0.3886516261861197, "nauc_map_at_5_max": 0.2791491859907928, "nauc_map_at_5_std": -0.006220989761089889, "nauc_mrr_at_1000_diff1": 0.26291894061688476, "nauc_mrr_at_1000_max": 0.4391066247779524, "nauc_mrr_at_1000_std": 0.16581006216156477, "nauc_mrr_at_100_diff1": 0.26288711136494536, "nauc_mrr_at_100_max": 0.4391620469458485, "nauc_mrr_at_100_std": 0.1658715626685403, "nauc_mrr_at_10_diff1": 0.26327333945185516, "nauc_mrr_at_10_max": 0.44341102883000105, "nauc_mrr_at_10_std": 0.16561645302869527, "nauc_mrr_at_1_diff1": 0.32704171564870577, "nauc_mrr_at_1_max": 0.3808335590314431, "nauc_mrr_at_1_std": 0.09589191598803506, "nauc_mrr_at_20_diff1": 0.26233626230983836, "nauc_mrr_at_20_max": 0.43936074098385325, "nauc_mrr_at_20_std": 0.16439843164683948, "nauc_mrr_at_3_diff1": 0.24495062088317518, "nauc_mrr_at_3_max": 0.43702373111006576, "nauc_mrr_at_3_std": 0.17206049009398258, "nauc_mrr_at_5_diff1": 0.25894678361787016, "nauc_mrr_at_5_max": 0.4448638903301484, "nauc_mrr_at_5_std": 0.1670169640451593, "nauc_ndcg_at_1000_diff1": 0.31157890959803164, "nauc_ndcg_at_1000_max": 0.3787818528153347, "nauc_ndcg_at_1000_std": 0.1378188447932894, "nauc_ndcg_at_100_diff1": 0.31408540427491544, "nauc_ndcg_at_100_max": 0.3798291424552418, "nauc_ndcg_at_100_std": 0.14054899933149664, "nauc_ndcg_at_10_diff1": 0.3340133128816092, "nauc_ndcg_at_10_max": 0.3530631227422352, "nauc_ndcg_at_10_std": 0.08413970348393185, "nauc_ndcg_at_1_diff1": 0.32704171564870577, "nauc_ndcg_at_1_max": 0.3808335590314431, "nauc_ndcg_at_1_std": 0.09589191598803506, "nauc_ndcg_at_20_diff1": 0.31345342246852975, "nauc_ndcg_at_20_max": 0.3609002826111666, "nauc_ndcg_at_20_std": 0.11523614914186356, "nauc_ndcg_at_3_diff1": 0.3049869785748296, "nauc_ndcg_at_3_max": 0.31520128443479706, "nauc_ndcg_at_3_std": 0.05417152725470993, "nauc_ndcg_at_5_diff1": 0.34478365245075243, "nauc_ndcg_at_5_max": 0.338438239943765, "nauc_ndcg_at_5_std": 0.05851861662259562, "nauc_precision_at_1000_diff1": -0.26629851978468416, "nauc_precision_at_1000_max": 0.11562956098174823, "nauc_precision_at_1000_std": 0.27421924351585414, "nauc_precision_at_100_diff1": -0.2527735638671963, "nauc_precision_at_100_max": 0.1365209282494595, "nauc_precision_at_100_std": 0.2957988594298787, "nauc_precision_at_10_diff1": -0.22643208215831126, "nauc_precision_at_10_max": 0.20607697750630086, "nauc_precision_at_10_std": 0.32229173147305973, "nauc_precision_at_1_diff1": 0.32704171564870577, "nauc_precision_at_1_max": 0.3808335590314431, "nauc_precision_at_1_std": 0.09589191598803506, "nauc_precision_at_20_diff1": -0.2662570792892436, "nauc_precision_at_20_max": 0.16827088690376713, "nauc_precision_at_20_std": 0.32503195274576097, "nauc_precision_at_3_diff1": -0.13026666909692144, "nauc_precision_at_3_max": 0.2600232366878702, "nauc_precision_at_3_std": 0.22074812382745637, "nauc_precision_at_5_diff1": -0.16903010461344328, "nauc_precision_at_5_max": 0.258303257695209, "nauc_precision_at_5_std": 0.2947510670253368, "nauc_recall_at_1000_diff1": 0.2964611316607367, "nauc_recall_at_1000_max": 0.6546561478690773, "nauc_recall_at_1000_std": 0.649403837651015, "nauc_recall_at_100_diff1": 0.28004258527111986, "nauc_recall_at_100_max": 0.45743295110839927, "nauc_recall_at_100_std": 0.35274895339401297, "nauc_recall_at_10_diff1": 0.2994792481317373, "nauc_recall_at_10_max": 0.3025812872802068, "nauc_recall_at_10_std": 0.04293106237441407, "nauc_recall_at_1_diff1": 0.5002958094670257, "nauc_recall_at_1_max": 0.23186362968659052, "nauc_recall_at_1_std": -0.07863597756490925, "nauc_recall_at_20_diff1": 0.2364476758056192, "nauc_recall_at_20_max": 0.30283747367825137, "nauc_recall_at_20_std": 0.13645724497263667, "nauc_recall_at_3_diff1": 0.33947285515212827, "nauc_recall_at_3_max": 0.22402624108514535, "nauc_recall_at_3_std": -0.058609048229718526, "nauc_recall_at_5_diff1": 0.3510054602379324, "nauc_recall_at_5_max": 0.27941319302455747, "nauc_recall_at_5_std": -0.005699858068950503, "ndcg_at_1": 0.58685, "ndcg_at_10": 0.63716, "ndcg_at_100": 0.6936, "ndcg_at_1000": 0.70286, "ndcg_at_20": 0.66546, "ndcg_at_3": 0.5878, "ndcg_at_5": 0.60371, "precision_at_1": 0.58685, "precision_at_10": 0.16761, "precision_at_100": 0.02352, "precision_at_1000": 0.00252, "precision_at_20": 0.09765, "precision_at_3": 0.35681, "precision_at_5": 0.26479, "recall_at_1": 0.35427, "recall_at_10": 0.7454, "recall_at_100": 0.9241, "recall_at_1000": 0.97746, "recall_at_20": 0.82278, "recall_at_3": 0.54536, "recall_at_5": 0.64111 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 25669.12554335594, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.55759, "map_at_1": 0.39509, "map_at_10": 0.50117, "map_at_100": 0.50963, "map_at_1000": 0.51001, "map_at_20": 0.50643, "map_at_3": 0.47037, "map_at_5": 0.48954, "mrr_at_1": 0.4251781472684085, "mrr_at_10": 0.5239405421709459, "mrr_at_100": 0.5305428352926489, "mrr_at_1000": 0.5308327496755653, "mrr_at_20": 0.5283895013787722, "mrr_at_3": 0.49683293745051477, "mrr_at_5": 0.5141726049089469, "nauc_map_at_1000_diff1": 0.35768639217367965, "nauc_map_at_1000_max": 0.3127376629580209, "nauc_map_at_1000_std": 0.043092776426519014, "nauc_map_at_100_diff1": 0.35758257680060296, "nauc_map_at_100_max": 0.31288275659144477, "nauc_map_at_100_std": 0.04331621897398727, "nauc_map_at_10_diff1": 0.358366707566828, "nauc_map_at_10_max": 0.3115301607781894, "nauc_map_at_10_std": 0.04119729223137315, "nauc_map_at_1_diff1": 0.3949573880581667, "nauc_map_at_1_max": 0.2941296103717735, "nauc_map_at_1_std": 0.01596221803523446, "nauc_map_at_20_diff1": 0.3565999350909673, "nauc_map_at_20_max": 0.31171024526967717, "nauc_map_at_20_std": 0.04207850136539487, "nauc_map_at_3_diff1": 0.37185946319709634, "nauc_map_at_3_max": 0.31504489843931266, "nauc_map_at_3_std": 0.026970161911460212, "nauc_map_at_5_diff1": 0.3578279526186646, "nauc_map_at_5_max": 0.3132459007812547, "nauc_map_at_5_std": 0.03693394458808769, "nauc_mrr_at_1000_diff1": 0.37550702514920065, "nauc_mrr_at_1000_max": 0.32978063926834633, "nauc_mrr_at_1000_std": 0.05199455641912942, "nauc_mrr_at_100_diff1": 0.37550262283237407, "nauc_mrr_at_100_max": 0.329873044791498, "nauc_mrr_at_100_std": 0.0521235638902929, "nauc_mrr_at_10_diff1": 0.37424020131037755, "nauc_mrr_at_10_max": 0.3286138818338544, "nauc_mrr_at_10_std": 0.051066433093199644, "nauc_mrr_at_1_diff1": 0.4192345581167409, "nauc_mrr_at_1_max": 0.3231101367860559, "nauc_mrr_at_1_std": 0.021685405644284943, "nauc_mrr_at_20_diff1": 0.3743936968494584, "nauc_mrr_at_20_max": 0.32916353278399135, "nauc_mrr_at_20_std": 0.05203860723902159, "nauc_mrr_at_3_diff1": 0.3847302816551476, "nauc_mrr_at_3_max": 0.3347963090797576, "nauc_mrr_at_3_std": 0.04498844178608618, "nauc_mrr_at_5_diff1": 0.3718607744339898, "nauc_mrr_at_5_max": 0.32809428903031135, "nauc_mrr_at_5_std": 0.05023093673209267, "nauc_ndcg_at_1000_diff1": 0.35232958746898174, "nauc_ndcg_at_1000_max": 0.3267938788741968, "nauc_ndcg_at_1000_std": 0.07049668808261762, "nauc_ndcg_at_100_diff1": 0.34657357607202904, "nauc_ndcg_at_100_max": 0.3262411105412631, "nauc_ndcg_at_100_std": 0.07721461181768907, "nauc_ndcg_at_10_diff1": 0.3420391143687474, "nauc_ndcg_at_10_max": 0.3163522985059599, "nauc_ndcg_at_10_std": 0.06318839209229431, "nauc_ndcg_at_1_diff1": 0.4192345581167409, "nauc_ndcg_at_1_max": 0.3231101367860559, "nauc_ndcg_at_1_std": 0.021685405644284943, "nauc_ndcg_at_20_diff1": 0.3385432035839586, "nauc_ndcg_at_20_max": 0.3184889703424418, "nauc_ndcg_at_20_std": 0.06815622662433982, "nauc_ndcg_at_3_diff1": 0.36622986860695933, "nauc_ndcg_at_3_max": 0.32554490444430517, "nauc_ndcg_at_3_std": 0.040110507562312735, "nauc_ndcg_at_5_diff1": 0.3398974842672672, "nauc_ndcg_at_5_max": 0.31829697901112036, "nauc_ndcg_at_5_std": 0.05499807904087406, "nauc_precision_at_1000_diff1": 0.026143237232001355, "nauc_precision_at_1000_max": 0.11022017814202459, "nauc_precision_at_1000_std": 0.12788466185750233, "nauc_precision_at_100_diff1": 0.05807266851074599, "nauc_precision_at_100_max": 0.17341475160832712, "nauc_precision_at_100_std": 0.19119373356730543, "nauc_precision_at_10_diff1": 0.18924741747564552, "nauc_precision_at_10_max": 0.25807631150353, "nauc_precision_at_10_std": 0.13675581065614384, "nauc_precision_at_1_diff1": 0.4192345581167409, "nauc_precision_at_1_max": 0.3231101367860559, "nauc_precision_at_1_std": 0.021685405644284943, "nauc_precision_at_20_diff1": 0.11656462987214906, "nauc_precision_at_20_max": 0.22302218143130395, "nauc_precision_at_20_std": 0.14814699825400385, "nauc_precision_at_3_diff1": 0.32207915108527485, "nauc_precision_at_3_max": 0.34216083411170406, "nauc_precision_at_3_std": 0.06928379184934734, "nauc_precision_at_5_diff1": 0.22531763594715176, "nauc_precision_at_5_max": 0.3100051074487403, "nauc_precision_at_5_std": 0.10760600326457498, "nauc_recall_at_1000_diff1": 0.3934047833765424, "nauc_recall_at_1000_max": 0.6125500319319205, "nauc_recall_at_1000_std": 0.4713280539660798, "nauc_recall_at_100_diff1": 0.25321522083323295, "nauc_recall_at_100_max": 0.4060827868852693, "nauc_recall_at_100_std": 0.34986498161422214, "nauc_recall_at_10_diff1": 0.25080737050315377, "nauc_recall_at_10_max": 0.3019992812742259, "nauc_recall_at_10_std": 0.13096854230053764, "nauc_recall_at_1_diff1": 0.3949573880581667, "nauc_recall_at_1_max": 0.2941296103717735, "nauc_recall_at_1_std": 0.01596221803523446, "nauc_recall_at_20_diff1": 0.22151949169256163, "nauc_recall_at_20_max": 0.31584000828616215, "nauc_recall_at_20_std": 0.17824486465526673, "nauc_recall_at_3_diff1": 0.3333329644687242, "nauc_recall_at_3_max": 0.33303649355475684, "nauc_recall_at_3_std": 0.060408299659899106, "nauc_recall_at_5_diff1": 0.2568414426238719, "nauc_recall_at_5_max": 0.3069180984271601, "nauc_recall_at_5_std": 0.09745113228464206, "ndcg_at_1": 0.42518, "ndcg_at_10": 0.55759, "ndcg_at_100": 0.59498, "ndcg_at_1000": 0.60508, "ndcg_at_20": 0.57564, "ndcg_at_3": 0.49833, "ndcg_at_5": 0.53163, "precision_at_1": 0.42518, "precision_at_10": 0.08005, "precision_at_100": 0.01017, "precision_at_1000": 0.00111, "precision_at_20": 0.04454, "precision_at_3": 0.20348, "precision_at_5": 0.14109, "recall_at_1": 0.39509, "recall_at_10": 0.70823, "recall_at_100": 0.87173, "recall_at_1000": 0.95012, "recall_at_20": 0.77553, "recall_at_3": 0.55384, "recall_at_5": 0.63222 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 18277.223182439804, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.30512, "map_at_1": 0.195, "map_at_10": 0.27131, "map_at_100": 0.28206, "map_at_1000": 0.28283, "map_at_20": 0.27705, "map_at_3": 0.25833, "map_at_5": 0.26608, "mrr_at_1": 0.195, "mrr_at_10": 0.2713095238095238, "mrr_at_100": 0.2820597065294532, "mrr_at_1000": 0.282825812851299, "mrr_at_20": 0.27705249274289523, "mrr_at_3": 0.2583333333333333, "mrr_at_5": 0.2660833333333333, "nauc_map_at_1000_diff1": 0.3987585414361193, "nauc_map_at_1000_max": 0.3473662893880687, "nauc_map_at_1000_std": -0.09380515261794065, "nauc_map_at_100_diff1": 0.3982652612420423, "nauc_map_at_100_max": 0.34728774894924935, "nauc_map_at_100_std": -0.09367496734064513, "nauc_map_at_10_diff1": 0.4012067329787962, "nauc_map_at_10_max": 0.35114595841736845, "nauc_map_at_10_std": -0.09965138320034828, "nauc_map_at_1_diff1": 0.4858458319949265, "nauc_map_at_1_max": 0.3316695905710263, "nauc_map_at_1_std": -0.15254544406282694, "nauc_map_at_20_diff1": 0.3985792389914664, "nauc_map_at_20_max": 0.34785847989247876, "nauc_map_at_20_std": -0.09555232301049926, "nauc_map_at_3_diff1": 0.3993831796412346, "nauc_map_at_3_max": 0.33816652918508316, "nauc_map_at_3_std": -0.12238405377787065, "nauc_map_at_5_diff1": 0.4093754976901047, "nauc_map_at_5_max": 0.3508738074287637, "nauc_map_at_5_std": -0.11071043205048879, "nauc_mrr_at_1000_diff1": 0.3987585414361193, "nauc_mrr_at_1000_max": 0.3473662893880687, "nauc_mrr_at_1000_std": -0.09380515261794065, "nauc_mrr_at_100_diff1": 0.3982652612420423, "nauc_mrr_at_100_max": 0.34728774894924935, "nauc_mrr_at_100_std": -0.09367496734064513, "nauc_mrr_at_10_diff1": 0.4012067329787962, "nauc_mrr_at_10_max": 0.35114595841736845, "nauc_mrr_at_10_std": -0.09965138320034828, "nauc_mrr_at_1_diff1": 0.4858458319949265, "nauc_mrr_at_1_max": 0.3316695905710263, "nauc_mrr_at_1_std": -0.15254544406282694, "nauc_mrr_at_20_diff1": 0.3985792389914664, "nauc_mrr_at_20_max": 0.34785847989247876, "nauc_mrr_at_20_std": -0.09555232301049926, "nauc_mrr_at_3_diff1": 0.3993831796412346, "nauc_mrr_at_3_max": 0.33816652918508316, "nauc_mrr_at_3_std": -0.12238405377787065, "nauc_mrr_at_5_diff1": 0.4093754976901047, "nauc_mrr_at_5_max": 0.3508738074287637, "nauc_mrr_at_5_std": -0.11071043205048879, "nauc_ndcg_at_1000_diff1": 0.36310659576315085, "nauc_ndcg_at_1000_max": 0.35153095780592464, "nauc_ndcg_at_1000_std": -0.037921327018792604, "nauc_ndcg_at_100_diff1": 0.35398837871625544, "nauc_ndcg_at_100_max": 0.3444722430366631, "nauc_ndcg_at_100_std": -0.03749615779295948, "nauc_ndcg_at_10_diff1": 0.3775534480968359, "nauc_ndcg_at_10_max": 0.36469609198907116, "nauc_ndcg_at_10_std": -0.06592871290792952, "nauc_ndcg_at_1_diff1": 0.4858458319949265, "nauc_ndcg_at_1_max": 0.3316695905710263, "nauc_ndcg_at_1_std": -0.15254544406282694, "nauc_ndcg_at_20_diff1": 0.3644847342354063, "nauc_ndcg_at_20_max": 0.3530827407169127, "nauc_ndcg_at_20_std": -0.05043897349253442, "nauc_ndcg_at_3_diff1": 0.3797573603059863, "nauc_ndcg_at_3_max": 0.3405088627197903, "nauc_ndcg_at_3_std": -0.11319623713123547, "nauc_ndcg_at_5_diff1": 0.39656109885163643, "nauc_ndcg_at_5_max": 0.3629088460780966, "nauc_ndcg_at_5_std": -0.09186138574375118, "nauc_precision_at_1000_diff1": 0.03953229398663588, "nauc_precision_at_1000_max": 0.4081689468660505, "nauc_precision_at_1000_std": 0.5251352211263127, "nauc_precision_at_100_diff1": 0.14331346383780955, "nauc_precision_at_100_max": 0.3019261483142932, "nauc_precision_at_100_std": 0.21624003979717482, "nauc_precision_at_10_diff1": 0.3135551623028535, "nauc_precision_at_10_max": 0.4040585401139285, "nauc_precision_at_10_std": 0.032686619218001235, "nauc_precision_at_1_diff1": 0.4858458319949265, "nauc_precision_at_1_max": 0.3316695905710263, "nauc_precision_at_1_std": -0.15254544406282694, "nauc_precision_at_20_diff1": 0.25659903034656145, "nauc_precision_at_20_max": 0.3622965756203322, "nauc_precision_at_20_std": 0.09387947832911026, "nauc_precision_at_3_diff1": 0.330077230076566, "nauc_precision_at_3_max": 0.34674511418496706, "nauc_precision_at_3_std": -0.08911009436280216, "nauc_precision_at_5_diff1": 0.3666605727418748, "nauc_precision_at_5_max": 0.3969887305919381, "nauc_precision_at_5_std": -0.039873855756800025, "nauc_recall_at_1000_diff1": 0.03953229398663671, "nauc_recall_at_1000_max": 0.4081689468660511, "nauc_recall_at_1000_std": 0.5251352211263131, "nauc_recall_at_100_diff1": 0.1433134638378088, "nauc_recall_at_100_max": 0.3019261483142927, "nauc_recall_at_100_std": 0.2162400397971738, "nauc_recall_at_10_diff1": 0.31355516230285324, "nauc_recall_at_10_max": 0.40405854011392833, "nauc_recall_at_10_std": 0.03268661921800123, "nauc_recall_at_1_diff1": 0.4858458319949265, "nauc_recall_at_1_max": 0.3316695905710263, "nauc_recall_at_1_std": -0.15254544406282694, "nauc_recall_at_20_diff1": 0.2565990303465613, "nauc_recall_at_20_max": 0.36229657562033213, "nauc_recall_at_20_std": 0.09387947832911028, "nauc_recall_at_3_diff1": 0.330077230076566, "nauc_recall_at_3_max": 0.34674511418496695, "nauc_recall_at_3_std": -0.0891100943628021, "nauc_recall_at_5_diff1": 0.36666057274187436, "nauc_recall_at_5_max": 0.39698873059193784, "nauc_recall_at_5_std": -0.03987385575680014, "ndcg_at_1": 0.195, "ndcg_at_10": 0.30512, "ndcg_at_100": 0.36182, "ndcg_at_1000": 0.38562, "ndcg_at_20": 0.3264, "ndcg_at_3": 0.27809, "ndcg_at_5": 0.29229, "precision_at_1": 0.195, "precision_at_10": 0.041, "precision_at_100": 0.00685, "precision_at_1000": 0.00088, "precision_at_20": 0.02475, "precision_at_3": 0.11167, "precision_at_5": 0.074, "recall_at_1": 0.195, "recall_at_10": 0.41, "recall_at_100": 0.685, "recall_at_1000": 0.88, "recall_at_20": 0.495, "recall_at_3": 0.335, "recall_at_5": 0.37 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.32286, "map_at_1": 0.235, "map_at_10": 0.28976, "map_at_100": 0.29636, "map_at_1000": 0.29717, "map_at_20": 0.29224, "map_at_3": 0.27083, "map_at_5": 0.28308, "mrr_at_1": 0.235, "mrr_at_10": 0.289763888888889, "mrr_at_100": 0.29636360098283915, "mrr_at_1000": 0.29716967927805255, "mrr_at_20": 0.2922370448179273, "mrr_at_3": 0.27083333333333337, "mrr_at_5": 0.28308333333333335, "nauc_map_at_1000_diff1": 0.49987230321102, "nauc_map_at_1000_max": 0.37241054403503265, "nauc_map_at_1000_std": -0.014615828090165943, "nauc_map_at_100_diff1": 0.4999165899809139, "nauc_map_at_100_max": 0.37225331598738776, "nauc_map_at_100_std": -0.015118425056351593, "nauc_map_at_10_diff1": 0.49672665674569366, "nauc_map_at_10_max": 0.3697606912534292, "nauc_map_at_10_std": -0.020212373283840207, "nauc_map_at_1_diff1": 0.571295785504311, "nauc_map_at_1_max": 0.40229277029222615, "nauc_map_at_1_std": -0.035268157395582166, "nauc_map_at_20_diff1": 0.5009344455788867, "nauc_map_at_20_max": 0.37195164827322513, "nauc_map_at_20_std": -0.018461911265518847, "nauc_map_at_3_diff1": 0.5100392656898904, "nauc_map_at_3_max": 0.36405669989453676, "nauc_map_at_3_std": -0.04279185609333193, "nauc_map_at_5_diff1": 0.5063385834276932, "nauc_map_at_5_max": 0.37114301949476286, "nauc_map_at_5_std": -0.022538869520912948, "nauc_mrr_at_1000_diff1": 0.49987230321102, "nauc_mrr_at_1000_max": 0.37241054403503265, "nauc_mrr_at_1000_std": -0.014615828090165943, "nauc_mrr_at_100_diff1": 0.4999165899809139, "nauc_mrr_at_100_max": 0.37225331598738776, "nauc_mrr_at_100_std": -0.015118425056351593, "nauc_mrr_at_10_diff1": 0.49672665674569366, "nauc_mrr_at_10_max": 0.3697606912534292, "nauc_mrr_at_10_std": -0.020212373283840207, "nauc_mrr_at_1_diff1": 0.571295785504311, "nauc_mrr_at_1_max": 0.40229277029222615, "nauc_mrr_at_1_std": -0.035268157395582166, "nauc_mrr_at_20_diff1": 0.5009344455788867, "nauc_mrr_at_20_max": 0.37195164827322513, "nauc_mrr_at_20_std": -0.018461911265518847, "nauc_mrr_at_3_diff1": 0.5100392656898904, "nauc_mrr_at_3_max": 0.36405669989453676, "nauc_mrr_at_3_std": -0.04279185609333193, "nauc_mrr_at_5_diff1": 0.5063385834276932, "nauc_mrr_at_5_max": 0.37114301949476286, "nauc_mrr_at_5_std": -0.022538869520912948, "nauc_ndcg_at_1000_diff1": 0.4773076374940932, "nauc_ndcg_at_1000_max": 0.38199768395620914, "nauc_ndcg_at_1000_std": 0.03989285210339253, "nauc_ndcg_at_100_diff1": 0.46702331084570764, "nauc_ndcg_at_100_max": 0.372772171990568, "nauc_ndcg_at_100_std": 0.03503234220335347, "nauc_ndcg_at_10_diff1": 0.46391339381565455, "nauc_ndcg_at_10_max": 0.36411117503372015, "nauc_ndcg_at_10_std": -3.265755317002024e-07, "nauc_ndcg_at_1_diff1": 0.571295785504311, "nauc_ndcg_at_1_max": 0.40229277029222615, "nauc_ndcg_at_1_std": -0.035268157395582166, "nauc_ndcg_at_20_diff1": 0.4787179235043199, "nauc_ndcg_at_20_max": 0.3726808078472696, "nauc_ndcg_at_20_std": 0.007987911883677817, "nauc_ndcg_at_3_diff1": 0.4945273149712954, "nauc_ndcg_at_3_max": 0.35345218197867073, "nauc_ndcg_at_3_std": -0.04348168510957773, "nauc_ndcg_at_5_diff1": 0.48712478424651734, "nauc_ndcg_at_5_max": 0.3657748679819708, "nauc_ndcg_at_5_std": -0.007269476847689231, "nauc_precision_at_1000_diff1": 0.42128930501023487, "nauc_precision_at_1000_max": 0.5748179469109707, "nauc_precision_at_1000_std": 0.575690459411389, "nauc_precision_at_100_diff1": 0.3335762077124369, "nauc_precision_at_100_max": 0.3844074648615614, "nauc_precision_at_100_std": 0.2778639590090725, "nauc_precision_at_10_diff1": 0.36711708782562363, "nauc_precision_at_10_max": 0.3518422182049919, "nauc_precision_at_10_std": 0.06350548178590516, "nauc_precision_at_1_diff1": 0.571295785504311, "nauc_precision_at_1_max": 0.40229277029222615, "nauc_precision_at_1_std": -0.035268157395582166, "nauc_precision_at_20_diff1": 0.4210910814065983, "nauc_precision_at_20_max": 0.3851363358145596, "nauc_precision_at_20_std": 0.09729434451530505, "nauc_precision_at_3_diff1": 0.4534210103821043, "nauc_precision_at_3_max": 0.32502943380070665, "nauc_precision_at_3_std": -0.04488654607727709, "nauc_precision_at_5_diff1": 0.43536883243771396, "nauc_precision_at_5_max": 0.3532258589806222, "nauc_precision_at_5_std": 0.03872333496173276, "nauc_recall_at_1000_diff1": 0.4212893050102354, "nauc_recall_at_1000_max": 0.5748179469109707, "nauc_recall_at_1000_std": 0.5756904594113896, "nauc_recall_at_100_diff1": 0.3335762077124369, "nauc_recall_at_100_max": 0.38440746486156113, "nauc_recall_at_100_std": 0.2778639590090725, "nauc_recall_at_10_diff1": 0.3671170878256232, "nauc_recall_at_10_max": 0.35184221820499156, "nauc_recall_at_10_std": 0.06350548178590472, "nauc_recall_at_1_diff1": 0.571295785504311, "nauc_recall_at_1_max": 0.40229277029222615, "nauc_recall_at_1_std": -0.035268157395582166, "nauc_recall_at_20_diff1": 0.4210910814065985, "nauc_recall_at_20_max": 0.38513633581455947, "nauc_recall_at_20_std": 0.09729434451530516, "nauc_recall_at_3_diff1": 0.45342101038210425, "nauc_recall_at_3_max": 0.3250294338007064, "nauc_recall_at_3_std": -0.04488654607727737, "nauc_recall_at_5_diff1": 0.4353688324377136, "nauc_recall_at_5_max": 0.3532258589806219, "nauc_recall_at_5_std": 0.03872333496173255, "ndcg_at_1": 0.235, "ndcg_at_10": 0.32286, "ndcg_at_100": 0.36221, "ndcg_at_1000": 0.38925, "ndcg_at_20": 0.33177, "ndcg_at_3": 0.28339, "ndcg_at_5": 0.30576, "precision_at_1": 0.235, "precision_at_10": 0.043, "precision_at_100": 0.0063, "precision_at_1000": 0.00086, "precision_at_20": 0.02325, "precision_at_3": 0.10667, "precision_at_5": 0.075, "recall_at_1": 0.235, "recall_at_10": 0.43, "recall_at_100": 0.63, "recall_at_1000": 0.855, "recall_at_20": 0.465, "recall_at_3": 0.32, "recall_at_5": 0.375 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 2.920142412185669, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.83059, "map_at_1": 0.72727, "map_at_10": 0.79388, "map_at_100": 0.79692, "map_at_1000": 0.79692, "map_at_20": 0.79649, "map_at_3": 0.77273, "map_at_5": 0.78571, "mrr_at_1": 0.7272727272727273, "mrr_at_10": 0.7938775510204082, "mrr_at_100": 0.7969205864163847, "mrr_at_1000": 0.7969205864163847, "mrr_at_20": 0.7964876859834844, "mrr_at_3": 0.7727272727272728, "mrr_at_5": 0.7857142857142858, "nauc_map_at_1000_diff1": 0.7711878770997442, "nauc_map_at_1000_max": 0.4194451984915928, "nauc_map_at_1000_std": -0.12084263394274973, "nauc_map_at_100_diff1": 0.7711878770997442, "nauc_map_at_100_max": 0.4194451984915928, "nauc_map_at_100_std": -0.12084263394274973, "nauc_map_at_10_diff1": 0.7731327921140855, "nauc_map_at_10_max": 0.4149630080279758, "nauc_map_at_10_std": -0.11395438175046349, "nauc_map_at_1_diff1": 0.7684401693767324, "nauc_map_at_1_max": 0.41777799933942383, "nauc_map_at_1_std": -0.11502937969975355, "nauc_map_at_20_diff1": 0.7706768155924971, "nauc_map_at_20_max": 0.42081663944031555, "nauc_map_at_20_std": -0.11924645739568039, "nauc_map_at_3_diff1": 0.7705914898088022, "nauc_map_at_3_max": 0.4117800788620658, "nauc_map_at_3_std": -0.14835870908303095, "nauc_map_at_5_diff1": 0.7784917987924511, "nauc_map_at_5_max": 0.4193041458114639, "nauc_map_at_5_std": -0.14401255779763072, "nauc_mrr_at_1000_diff1": 0.7711878770997442, "nauc_mrr_at_1000_max": 0.4194451984915928, "nauc_mrr_at_1000_std": -0.12084263394274973, "nauc_mrr_at_100_diff1": 0.7711878770997442, "nauc_mrr_at_100_max": 0.4194451984915928, "nauc_mrr_at_100_std": -0.12084263394274973, "nauc_mrr_at_10_diff1": 0.7731327921140855, "nauc_mrr_at_10_max": 0.4149630080279758, "nauc_mrr_at_10_std": -0.11395438175046349, "nauc_mrr_at_1_diff1": 0.7684401693767324, "nauc_mrr_at_1_max": 0.41777799933942383, "nauc_mrr_at_1_std": -0.11502937969975355, "nauc_mrr_at_20_diff1": 0.7706768155924971, "nauc_mrr_at_20_max": 0.42081663944031555, "nauc_mrr_at_20_std": -0.11924645739568039, "nauc_mrr_at_3_diff1": 0.7705914898088022, "nauc_mrr_at_3_max": 0.4117800788620658, "nauc_mrr_at_3_std": -0.14835870908303095, "nauc_mrr_at_5_diff1": 0.7784917987924511, "nauc_mrr_at_5_max": 0.4193041458114639, "nauc_mrr_at_5_std": -0.14401255779763072, "nauc_ndcg_at_1000_diff1": 0.77360058189681, "nauc_ndcg_at_1000_max": 0.4194356326749995, "nauc_ndcg_at_1000_std": -0.11839678586793939, "nauc_ndcg_at_100_diff1": 0.77360058189681, "nauc_ndcg_at_100_max": 0.4194356326749995, "nauc_ndcg_at_100_std": -0.11839678586793939, "nauc_ndcg_at_10_diff1": 0.7811901296021619, "nauc_ndcg_at_10_max": 0.40235956792512756, "nauc_ndcg_at_10_std": -0.08244172952345144, "nauc_ndcg_at_1_diff1": 0.7684401693767324, "nauc_ndcg_at_1_max": 0.41777799933942383, "nauc_ndcg_at_1_std": -0.11502937969975355, "nauc_ndcg_at_20_diff1": 0.7696320195793978, "nauc_ndcg_at_20_max": 0.4299680418339879, "nauc_ndcg_at_20_std": -0.10618302323604849, "nauc_ndcg_at_3_diff1": 0.7798864907322665, "nauc_ndcg_at_3_max": 0.40461393835085435, "nauc_ndcg_at_3_std": -0.17216329739386393, "nauc_ndcg_at_5_diff1": 0.7960670192269287, "nauc_ndcg_at_5_max": 0.4182363168980118, "nauc_ndcg_at_5_std": -0.16648908400791346, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.8558898391832962, "nauc_precision_at_10_max": 0.24031819702144644, "nauc_precision_at_10_std": 0.3456539015073844, "nauc_precision_at_1_diff1": 0.7684401693767324, "nauc_precision_at_1_max": 0.41777799933942383, "nauc_precision_at_1_std": -0.11502937969975355, "nauc_precision_at_20_diff1": 0.5548466542608996, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 0.5548466542608996, "nauc_precision_at_3_diff1": 0.8174093269102958, "nauc_precision_at_3_max": 0.3763299073053593, "nauc_precision_at_3_std": -0.2641254997849845, "nauc_precision_at_5_diff1": 0.8889212754660993, "nauc_precision_at_5_max": 0.4130763535290174, "nauc_precision_at_5_std": -0.2802349509686518, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.855889839183298, "nauc_recall_at_10_max": 0.24031819702144214, "nauc_recall_at_10_std": 0.3456539015073782, "nauc_recall_at_1_diff1": 0.7684401693767324, "nauc_recall_at_1_max": 0.41777799933942383, "nauc_recall_at_1_std": -0.11502937969975355, "nauc_recall_at_20_diff1": 0.5548466542608921, "nauc_recall_at_20_max": 1.0, "nauc_recall_at_20_std": 0.5548466542608921, "nauc_recall_at_3_diff1": 0.8174093269102954, "nauc_recall_at_3_max": 0.3763299073053579, "nauc_recall_at_3_std": -0.264125499784987, "nauc_recall_at_5_diff1": 0.8889212754660999, "nauc_recall_at_5_max": 0.4130763535290157, "nauc_recall_at_5_std": -0.28023495096865575, "ndcg_at_1": 0.72727, "ndcg_at_10": 0.83059, "ndcg_at_100": 0.84295, "ndcg_at_1000": 0.84295, "ndcg_at_20": 0.84033, "ndcg_at_3": 0.78772, "ndcg_at_5": 0.8101, "precision_at_1": 0.72727, "precision_at_10": 0.09481, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04935, "precision_at_3": 0.27706, "precision_at_5": 0.17662, "recall_at_1": 0.72727, "recall_at_10": 0.94805, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.98701, "recall_at_3": 0.83117, "recall_at_5": 0.88312 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 56.67747497558594, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.41272, "map_at_1": 0.2278, "map_at_10": 0.36059, "map_at_100": 0.37682, "map_at_1000": 0.3779, "map_at_20": 0.37011, "map_at_3": 0.31887, "map_at_5": 0.34439, "mrr_at_1": 0.3256880733944954, "mrr_at_10": 0.39670889762632877, "mrr_at_100": 0.40823785596071943, "mrr_at_1000": 0.40895929543989445, "mrr_at_20": 0.40345750777926914, "mrr_at_3": 0.37079510703363927, "mrr_at_5": 0.3851681957186544, "nauc_map_at_1000_diff1": 0.3986349246433768, "nauc_map_at_1000_max": 0.270056315928616, "nauc_map_at_1000_std": -0.2995773555241248, "nauc_map_at_100_diff1": 0.3981539647860276, "nauc_map_at_100_max": 0.26985257487081127, "nauc_map_at_100_std": -0.29917691881901637, "nauc_map_at_10_diff1": 0.4005826917242611, "nauc_map_at_10_max": 0.26944691669751575, "nauc_map_at_10_std": -0.3080734589306485, "nauc_map_at_1_diff1": 0.5537918873542623, "nauc_map_at_1_max": 0.23598262232798495, "nauc_map_at_1_std": -0.280734302808289, "nauc_map_at_20_diff1": 0.39753222673837235, "nauc_map_at_20_max": 0.26668810318015923, "nauc_map_at_20_std": -0.30326743096985315, "nauc_map_at_3_diff1": 0.43653927487324706, "nauc_map_at_3_max": 0.28162209617369505, "nauc_map_at_3_std": -0.2983177222727111, "nauc_map_at_5_diff1": 0.40367560513004136, "nauc_map_at_5_max": 0.2699722571975168, "nauc_map_at_5_std": -0.3091694992647524, "nauc_mrr_at_1000_diff1": 0.41742567194309316, "nauc_mrr_at_1000_max": 0.2820274297138866, "nauc_mrr_at_1000_std": -0.3018893641253346, "nauc_mrr_at_100_diff1": 0.4172364490671897, "nauc_mrr_at_100_max": 0.2822267655441534, "nauc_mrr_at_100_std": -0.30138980130897775, "nauc_mrr_at_10_diff1": 0.4157358712718055, "nauc_mrr_at_10_max": 0.2816783228400729, "nauc_mrr_at_10_std": -0.306050948512259, "nauc_mrr_at_1_diff1": 0.46818090820086133, "nauc_mrr_at_1_max": 0.29005758430178175, "nauc_mrr_at_1_std": -0.3121692789770937, "nauc_mrr_at_20_diff1": 0.415385717783091, "nauc_mrr_at_20_max": 0.28080951122709097, "nauc_mrr_at_20_std": -0.3021692911975301, "nauc_mrr_at_3_diff1": 0.4311479156241253, "nauc_mrr_at_3_max": 0.2794712072601227, "nauc_mrr_at_3_std": -0.3142678714409581, "nauc_mrr_at_5_diff1": 0.4191764166827879, "nauc_mrr_at_5_max": 0.27731591547418033, "nauc_mrr_at_5_std": -0.3115256041998101, "nauc_ndcg_at_1000_diff1": 0.3871236683473045, "nauc_ndcg_at_1000_max": 0.27785009471536276, "nauc_ndcg_at_1000_std": -0.28313792161123275, "nauc_ndcg_at_100_diff1": 0.3796164282303305, "nauc_ndcg_at_100_max": 0.27848400403373913, "nauc_ndcg_at_100_std": -0.2703759465076034, "nauc_ndcg_at_10_diff1": 0.37786294135057596, "nauc_ndcg_at_10_max": 0.2666940160427433, "nauc_ndcg_at_10_std": -0.3054250955363569, "nauc_ndcg_at_1_diff1": 0.46818090820086133, "nauc_ndcg_at_1_max": 0.29005758430178175, "nauc_ndcg_at_1_std": -0.3121692789770937, "nauc_ndcg_at_20_diff1": 0.3681411672128963, "nauc_ndcg_at_20_max": 0.2595929246670804, "nauc_ndcg_at_20_std": -0.2894346139742596, "nauc_ndcg_at_3_diff1": 0.4159717306246669, "nauc_ndcg_at_3_max": 0.2761781698823905, "nauc_ndcg_at_3_std": -0.3077653844871884, "nauc_ndcg_at_5_diff1": 0.38586966345751245, "nauc_ndcg_at_5_max": 0.26496204815418567, "nauc_ndcg_at_5_std": -0.31286943575059006, "nauc_precision_at_1000_diff1": -0.14270436916844806, "nauc_precision_at_1000_max": 0.16425763194215595, "nauc_precision_at_1000_std": 0.06244073872574862, "nauc_precision_at_100_diff1": -0.07711500789191686, "nauc_precision_at_100_max": 0.18697597542703154, "nauc_precision_at_100_std": 0.028796920726206685, "nauc_precision_at_10_diff1": 0.05996282095605551, "nauc_precision_at_10_max": 0.19845035281828807, "nauc_precision_at_10_std": -0.21365705395109136, "nauc_precision_at_1_diff1": 0.46818090820086133, "nauc_precision_at_1_max": 0.29005758430178175, "nauc_precision_at_1_std": -0.3121692789770937, "nauc_precision_at_20_diff1": -0.00267670668235737, "nauc_precision_at_20_max": 0.1562177469875502, "nauc_precision_at_20_std": -0.15212370508658465, "nauc_precision_at_3_diff1": 0.1820797182649027, "nauc_precision_at_3_max": 0.2695441385762009, "nauc_precision_at_3_std": -0.2555002518667556, "nauc_precision_at_5_diff1": 0.09317805884242907, "nauc_precision_at_5_max": 0.21814113346085354, "nauc_precision_at_5_std": -0.24796979874344902, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.2682562964253447, "nauc_recall_at_100_max": 0.3042738281713388, "nauc_recall_at_100_std": -0.05638100774172797, "nauc_recall_at_10_diff1": 0.2840213882598167, "nauc_recall_at_10_max": 0.23509166753266306, "nauc_recall_at_10_std": -0.287616611982279, "nauc_recall_at_1_diff1": 0.5537918873542623, "nauc_recall_at_1_max": 0.23598262232798495, "nauc_recall_at_1_std": -0.280734302808289, "nauc_recall_at_20_diff1": 0.2360234237411498, "nauc_recall_at_20_max": 0.20401243027726493, "nauc_recall_at_20_std": -0.22432250204647355, "nauc_recall_at_3_diff1": 0.3880967451963076, "nauc_recall_at_3_max": 0.26597328508638085, "nauc_recall_at_3_std": -0.2925767551949979, "nauc_recall_at_5_diff1": 0.31396729989609323, "nauc_recall_at_5_max": 0.23783285556474018, "nauc_recall_at_5_std": -0.3109615047535852, "ndcg_at_1": 0.32569, "ndcg_at_10": 0.41272, "ndcg_at_100": 0.48506, "ndcg_at_1000": 0.50662, "ndcg_at_20": 0.44224, "ndcg_at_3": 0.3491, "ndcg_at_5": 0.37955, "precision_at_1": 0.32569, "precision_at_10": 0.08685, "precision_at_100": 0.01369, "precision_at_1000": 0.00156, "precision_at_20": 0.05161, "precision_at_3": 0.19725, "precision_at_5": 0.14495, "recall_at_1": 0.2278, "recall_at_10": 0.5329, "recall_at_100": 0.84544, "recall_at_1000": 1.0, "recall_at_20": 0.63693, "recall_at_3": 0.36621, "recall_at_5": 0.4408 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.37842, "map_at_1": 0.1815, "map_at_10": 0.32196, "map_at_100": 0.33971, "map_at_1000": 0.3412, "map_at_20": 0.33094, "map_at_3": 0.27112, "map_at_5": 0.30222, "mrr_at_1": 0.308868501529052, "mrr_at_10": 0.3830026455026454, "mrr_at_100": 0.3926271293660191, "mrr_at_1000": 0.39333624218330904, "mrr_at_20": 0.3874286203634769, "mrr_at_3": 0.3588175331294598, "mrr_at_5": 0.3708970438328237, "nauc_map_at_1000_diff1": 0.34960257110425647, "nauc_map_at_1000_max": 0.17485055091809065, "nauc_map_at_1000_std": -0.3643929345214113, "nauc_map_at_100_diff1": 0.3498111374525095, "nauc_map_at_100_max": 0.17383876312682625, "nauc_map_at_100_std": -0.36514910056851757, "nauc_map_at_10_diff1": 0.3517695153014903, "nauc_map_at_10_max": 0.1712516330827489, "nauc_map_at_10_std": -0.3709969243384287, "nauc_map_at_1_diff1": 0.4389487356981317, "nauc_map_at_1_max": 0.13070144601986813, "nauc_map_at_1_std": -0.3257177968261257, "nauc_map_at_20_diff1": 0.35150108432175453, "nauc_map_at_20_max": 0.1732034757031926, "nauc_map_at_20_std": -0.3694300615140005, "nauc_map_at_3_diff1": 0.37487194965631493, "nauc_map_at_3_max": 0.1554453027838439, "nauc_map_at_3_std": -0.3660209323134048, "nauc_map_at_5_diff1": 0.3582798160112775, "nauc_map_at_5_max": 0.17274661077722417, "nauc_map_at_5_std": -0.3662379164960217, "nauc_mrr_at_1000_diff1": 0.33320372878371957, "nauc_mrr_at_1000_max": 0.17229056799897927, "nauc_mrr_at_1000_std": -0.3572354959222832, "nauc_mrr_at_100_diff1": 0.33304522516161006, "nauc_mrr_at_100_max": 0.17227506166169268, "nauc_mrr_at_100_std": -0.35686316755936126, "nauc_mrr_at_10_diff1": 0.32924862992084486, "nauc_mrr_at_10_max": 0.17008699502436947, "nauc_mrr_at_10_std": -0.36143807221850954, "nauc_mrr_at_1_diff1": 0.40000594128429456, "nauc_mrr_at_1_max": 0.18713315063533922, "nauc_mrr_at_1_std": -0.36482574431254056, "nauc_mrr_at_20_diff1": 0.3321526172498427, "nauc_mrr_at_20_max": 0.1716268070758386, "nauc_mrr_at_20_std": -0.3582418906714954, "nauc_mrr_at_3_diff1": 0.34332399650158973, "nauc_mrr_at_3_max": 0.166974823366417, "nauc_mrr_at_3_std": -0.3667690569859602, "nauc_mrr_at_5_diff1": 0.3332286480105909, "nauc_mrr_at_5_max": 0.1709185280029645, "nauc_mrr_at_5_std": -0.3629173129534889, "nauc_ndcg_at_1000_diff1": 0.32373166349302934, "nauc_ndcg_at_1000_max": 0.1858059225739381, "nauc_ndcg_at_1000_std": -0.34015450334779784, "nauc_ndcg_at_100_diff1": 0.3242121817496865, "nauc_ndcg_at_100_max": 0.18022093662951252, "nauc_ndcg_at_100_std": -0.3375038271705809, "nauc_ndcg_at_10_diff1": 0.32455074708282194, "nauc_ndcg_at_10_max": 0.16615377791035843, "nauc_ndcg_at_10_std": -0.3731626507503713, "nauc_ndcg_at_1_diff1": 0.40000594128429456, "nauc_ndcg_at_1_max": 0.18713315063533922, "nauc_ndcg_at_1_std": -0.36482574431254056, "nauc_ndcg_at_20_diff1": 0.32698955871944246, "nauc_ndcg_at_20_max": 0.16997034992597687, "nauc_ndcg_at_20_std": -0.36750360083621514, "nauc_ndcg_at_3_diff1": 0.3436379653839748, "nauc_ndcg_at_3_max": 0.16313575812628048, "nauc_ndcg_at_3_std": -0.36953626791892746, "nauc_ndcg_at_5_diff1": 0.3365356003484116, "nauc_ndcg_at_5_max": 0.17035446999498027, "nauc_ndcg_at_5_std": -0.37041579827735527, "nauc_precision_at_1000_diff1": -0.11479120592320564, "nauc_precision_at_1000_max": 0.18532885988969905, "nauc_precision_at_1000_std": 0.09181657680195955, "nauc_precision_at_100_diff1": -0.027723270420175344, "nauc_precision_at_100_max": 0.1804800858662913, "nauc_precision_at_100_std": -0.002671742958024478, "nauc_precision_at_10_diff1": 0.10625793640540403, "nauc_precision_at_10_max": 0.17103040268214398, "nauc_precision_at_10_std": -0.24522215545298606, "nauc_precision_at_1_diff1": 0.40000594128429456, "nauc_precision_at_1_max": 0.18713315063533922, "nauc_precision_at_1_std": -0.36482574431254056, "nauc_precision_at_20_diff1": 0.08115934112015194, "nauc_precision_at_20_max": 0.17243668491400857, "nauc_precision_at_20_std": -0.19145773832420498, "nauc_precision_at_3_diff1": 0.20449380728257122, "nauc_precision_at_3_max": 0.18721037486519035, "nauc_precision_at_3_std": -0.3022611337135435, "nauc_precision_at_5_diff1": 0.1525188864909601, "nauc_precision_at_5_max": 0.19713305548617693, "nauc_precision_at_5_std": -0.26661909838876846, "nauc_recall_at_1000_diff1": 0.31723599245329515, "nauc_recall_at_1000_max": 0.8078769144027147, "nauc_recall_at_1000_std": 0.5661298338911104, "nauc_recall_at_100_diff1": 0.26528024492414976, "nauc_recall_at_100_max": 0.17958589910696324, "nauc_recall_at_100_std": -0.16203827885804548, "nauc_recall_at_10_diff1": 0.26503688174517775, "nauc_recall_at_10_max": 0.13462330096229136, "nauc_recall_at_10_std": -0.3740352999875769, "nauc_recall_at_1_diff1": 0.4389487356981317, "nauc_recall_at_1_max": 0.13070144601986813, "nauc_recall_at_1_std": -0.3257177968261257, "nauc_recall_at_20_diff1": 0.27969933464128294, "nauc_recall_at_20_max": 0.1362455670688532, "nauc_recall_at_20_std": -0.36929345951037745, "nauc_recall_at_3_diff1": 0.3270194544815536, "nauc_recall_at_3_max": 0.1300087809804867, "nauc_recall_at_3_std": -0.37113956325500835, "nauc_recall_at_5_diff1": 0.2922894050062497, "nauc_recall_at_5_max": 0.1490330127476058, "nauc_recall_at_5_std": -0.37110288849601253, "ndcg_at_1": 0.30887, "ndcg_at_10": 0.37842, "ndcg_at_100": 0.45658, "ndcg_at_1000": 0.48338, "ndcg_at_20": 0.40231, "ndcg_at_3": 0.32192, "ndcg_at_5": 0.34213, "precision_at_1": 0.30887, "precision_at_10": 0.10046, "precision_at_100": 0.01654, "precision_at_1000": 0.00195, "precision_at_20": 0.05894, "precision_at_3": 0.21254, "precision_at_5": 0.16116, "recall_at_1": 0.1815, "recall_at_10": 0.47497, "recall_at_100": 0.81081, "recall_at_1000": 0.99541, "recall_at_20": 0.55089, "recall_at_3": 0.31254, "recall_at_5": 0.38252 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.40223, "map_at_1": 0.21299, "map_at_10": 0.34846, "map_at_100": 0.36474, "map_at_1000": 0.36602, "map_at_20": 0.3575, "map_at_3": 0.3062, "map_at_5": 0.33126, "mrr_at_1": 0.30944625407166126, "mrr_at_10": 0.3896153249573444, "mrr_at_100": 0.4003699212287417, "mrr_at_1000": 0.4011819839650043, "mrr_at_20": 0.3959947132991476, "mrr_at_3": 0.3637350705754616, "mrr_at_5": 0.37741585233441916, "nauc_map_at_1000_diff1": 0.3938221230645532, "nauc_map_at_1000_max": 0.19625194395439952, "nauc_map_at_1000_std": -0.294610595078822, "nauc_map_at_100_diff1": 0.39295853338115067, "nauc_map_at_100_max": 0.19638087558933384, "nauc_map_at_100_std": -0.29389410176273423, "nauc_map_at_10_diff1": 0.3996875180636347, "nauc_map_at_10_max": 0.19161522554768412, "nauc_map_at_10_std": -0.30393649399067396, "nauc_map_at_1_diff1": 0.5313204171828388, "nauc_map_at_1_max": 0.128356899752893, "nauc_map_at_1_std": -0.30478917758174107, "nauc_map_at_20_diff1": 0.3928799169599587, "nauc_map_at_20_max": 0.19069973379303265, "nauc_map_at_20_std": -0.2999390962991156, "nauc_map_at_3_diff1": 0.42998007649481695, "nauc_map_at_3_max": 0.1981280295393313, "nauc_map_at_3_std": -0.30182869125853073, "nauc_map_at_5_diff1": 0.4095094249935181, "nauc_map_at_5_max": 0.19213544756315376, "nauc_map_at_5_std": -0.3121213859877235, "nauc_mrr_at_1000_diff1": 0.41329312260501055, "nauc_mrr_at_1000_max": 0.22256546352384424, "nauc_mrr_at_1000_std": -0.2615215661882381, "nauc_mrr_at_100_diff1": 0.4128214900473252, "nauc_mrr_at_100_max": 0.22302683998816905, "nauc_mrr_at_100_std": -0.26068224887493174, "nauc_mrr_at_10_diff1": 0.4150293639606534, "nauc_mrr_at_10_max": 0.22143723479359165, "nauc_mrr_at_10_std": -0.26424389757331473, "nauc_mrr_at_1_diff1": 0.45950523789844566, "nauc_mrr_at_1_max": 0.212384855922023, "nauc_mrr_at_1_std": -0.2903672779891095, "nauc_mrr_at_20_diff1": 0.41203878019046625, "nauc_mrr_at_20_max": 0.2214042859918071, "nauc_mrr_at_20_std": -0.26177487417580636, "nauc_mrr_at_3_diff1": 0.4296857380310018, "nauc_mrr_at_3_max": 0.22033601859056587, "nauc_mrr_at_3_std": -0.27386519411362786, "nauc_mrr_at_5_diff1": 0.41735491637004696, "nauc_mrr_at_5_max": 0.2186850687203513, "nauc_mrr_at_5_std": -0.2726504552695364, "nauc_ndcg_at_1000_diff1": 0.38055336452170857, "nauc_ndcg_at_1000_max": 0.21849504565789374, "nauc_ndcg_at_1000_std": -0.2606109672159647, "nauc_ndcg_at_100_diff1": 0.36409748024563005, "nauc_ndcg_at_100_max": 0.22659367268973427, "nauc_ndcg_at_100_std": -0.24038803575013593, "nauc_ndcg_at_10_diff1": 0.3784656842619079, "nauc_ndcg_at_10_max": 0.19918626187829117, "nauc_ndcg_at_10_std": -0.2823101234456151, "nauc_ndcg_at_1_diff1": 0.45950523789844566, "nauc_ndcg_at_1_max": 0.212384855922023, "nauc_ndcg_at_1_std": -0.2903672779891095, "nauc_ndcg_at_20_diff1": 0.36056300913528344, "nauc_ndcg_at_20_max": 0.19460362878213833, "nauc_ndcg_at_20_std": -0.271733284598135, "nauc_ndcg_at_3_diff1": 0.4132197548547503, "nauc_ndcg_at_3_max": 0.20930222238796498, "nauc_ndcg_at_3_std": -0.2901717567296085, "nauc_ndcg_at_5_diff1": 0.39295566654495695, "nauc_ndcg_at_5_max": 0.19730365676943964, "nauc_ndcg_at_5_std": -0.30373276370527186, "nauc_precision_at_1000_diff1": -0.10224146102287582, "nauc_precision_at_1000_max": 0.260939691014381, "nauc_precision_at_1000_std": 0.17467020896444718, "nauc_precision_at_100_diff1": -0.07151893728003765, "nauc_precision_at_100_max": 0.2909177879066712, "nauc_precision_at_100_std": 0.14859534211857378, "nauc_precision_at_10_diff1": 0.07640552089770371, "nauc_precision_at_10_max": 0.20909361194189371, "nauc_precision_at_10_std": -0.12620522925239366, "nauc_precision_at_1_diff1": 0.45950523789844566, "nauc_precision_at_1_max": 0.212384855922023, "nauc_precision_at_1_std": -0.2903672779891095, "nauc_precision_at_20_diff1": 0.005183844322358487, "nauc_precision_at_20_max": 0.1895615818885953, "nauc_precision_at_20_std": -0.07614703524831418, "nauc_precision_at_3_diff1": 0.20528382610923085, "nauc_precision_at_3_max": 0.2568535683149407, "nauc_precision_at_3_std": -0.21011193797063527, "nauc_precision_at_5_diff1": 0.1350809038263745, "nauc_precision_at_5_max": 0.21494250884185573, "nauc_precision_at_5_std": -0.20641992216879387, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.16128929663116692, "nauc_recall_at_100_max": 0.3322887006507102, "nauc_recall_at_100_std": 0.03544452371521469, "nauc_recall_at_10_diff1": 0.2895975117034754, "nauc_recall_at_10_max": 0.17920042543312975, "nauc_recall_at_10_std": -0.24172817505144845, "nauc_recall_at_1_diff1": 0.5313204171828388, "nauc_recall_at_1_max": 0.128356899752893, "nauc_recall_at_1_std": -0.30478917758174107, "nauc_recall_at_20_diff1": 0.21900103521061778, "nauc_recall_at_20_max": 0.1523084242145846, "nauc_recall_at_20_std": -0.20233485015860317, "nauc_recall_at_3_diff1": 0.39265123892543263, "nauc_recall_at_3_max": 0.20090609438025173, "nauc_recall_at_3_std": -0.2828331887306345, "nauc_recall_at_5_diff1": 0.33216577481355436, "nauc_recall_at_5_max": 0.17840365515545123, "nauc_recall_at_5_std": -0.30304571681383957, "ndcg_at_1": 0.30945, "ndcg_at_10": 0.40223, "ndcg_at_100": 0.47296, "ndcg_at_1000": 0.49856, "ndcg_at_20": 0.42947, "ndcg_at_3": 0.34064, "ndcg_at_5": 0.3688, "precision_at_1": 0.30945, "precision_at_10": 0.08925, "precision_at_100": 0.01419, "precision_at_1000": 0.00167, "precision_at_20": 0.05252, "precision_at_3": 0.19815, "precision_at_5": 0.14593, "recall_at_1": 0.21299, "recall_at_10": 0.51805, "recall_at_100": 0.81796, "recall_at_1000": 1.0, "recall_at_20": 0.61218, "recall_at_3": 0.3574, "recall_at_5": 0.4282 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/model_meta.json ================================================ {"name": "jinaai/jina-embeddings-v3", "revision": "215a6e121fa0183376388ac6b1ae230326bfeaed", "release_date": "2024-09-18", "languages": ["afr_Latn", "amh_Latn", "ara_Latn", "asm_Latn", "aze_Latn", "bel_Latn", "bul_Latn", "ben_Latn", "ben_Beng", "bre_Latn", "bos_Latn", "cat_Latn", "ces_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "ell_Latn", "eng_Latn", "epo_Latn", "spa_Latn", "est_Latn", "eus_Latn", "fas_Latn", "fin_Latn", "fra_Latn", "fry_Latn", "gle_Latn", "gla_Latn", "glg_Latn", "guj_Latn", "hau_Latn", "heb_Latn", "hin_Latn", "hin_Deva", "hrv_Latn", "hun_Latn", "hye_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jpn_Latn", "jav_Latn", "kat_Latn", "kaz_Latn", "khm_Latn", "kan_Latn", "kor_Latn", "kur_Latn", "kir_Latn", "lat_Latn", "lao_Latn", "lit_Latn", "lav_Latn", "mlg_Latn", "mkd_Latn", "mal_Latn", "mon_Latn", "mar_Latn", "msa_Latn", "mya_Latn", "nep_Latn", "nld_Latn", "nob_Latn", "orm_Latn", "ori_Latn", "pan_Latn", "pol_Latn", "pus_Latn", "por_Latn", "ron_Latn", "rus_Latn", "san_Latn", "snd_Latn", "sin_Latn", "slk_Latn", "slv_Latn", "som_Latn", "sqi_Latn", "srp_Latn", "sun_Latn", "swe_Latn", "swa_Latn", "tam_Latn", "tam_Taml", "tel_Latn", "tel_Telu", "tha_Latn", "tgl_Latn", "tur_Latn", "uig_Latn", "ukr_Latn", "urd_Latn", "urd_Arab", "uzb_Latn", "vie_Latn", "xho_Latn", "yid_Latn", "zho_Hant", "zho_Hans"], "n_parameters": 572000000, "memory_usage": null, "max_tokens": 8194, "embed_dim": 4096, "license": "cc-by-nc-4.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/jinaai/jina-embeddings-v3", "similarity_fn_name": "cosine", "use_instructions": false, "zero_shot_benchmarks": null, "loader": "JinaWrapper"} ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "task_name": "AutoRAGRetrieval", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.73684, "ndcg_at_3": 0.84847, "ndcg_at_5": 0.86243, "ndcg_at_10": 0.87076, "ndcg_at_20": 0.8751, "ndcg_at_100": 0.8751, "ndcg_at_1000": 0.8751, "map_at_1": 0.73684, "map_at_3": 0.8231, "map_at_5": 0.83056, "map_at_10": 0.83388, "map_at_20": 0.83501, "map_at_100": 0.83501, "map_at_1000": 0.83501, "recall_at_1": 0.73684, "recall_at_3": 0.92105, "recall_at_5": 0.95614, "recall_at_10": 0.98246, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.73684, "precision_at_3": 0.30702, "precision_at_5": 0.19123, "precision_at_10": 0.09825, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.736842, "mrr_at_3": 0.823099, "mrr_at_5": 0.830556, "mrr_at_10": 0.83388, "mrr_at_20": 0.835013, "mrr_at_100": 0.835013, "mrr_at_1000": 0.835013, "nauc_ndcg_at_1_max": 0.071494, "nauc_ndcg_at_1_std": -0.574049, "nauc_ndcg_at_1_diff1": 0.85592, "nauc_ndcg_at_3_max": 0.225962, "nauc_ndcg_at_3_std": -0.62872, "nauc_ndcg_at_3_diff1": 0.813741, "nauc_ndcg_at_5_max": 0.210624, "nauc_ndcg_at_5_std": -0.666077, "nauc_ndcg_at_5_diff1": 0.836702, "nauc_ndcg_at_10_max": 0.164149, "nauc_ndcg_at_10_std": -0.641323, "nauc_ndcg_at_10_diff1": 0.835098, "nauc_ndcg_at_20_max": 0.149825, "nauc_ndcg_at_20_std": -0.628672, "nauc_ndcg_at_20_diff1": 0.839869, "nauc_ndcg_at_100_max": 0.149825, "nauc_ndcg_at_100_std": -0.628672, "nauc_ndcg_at_100_diff1": 0.839869, "nauc_ndcg_at_1000_max": 0.149825, "nauc_ndcg_at_1000_std": -0.628672, "nauc_ndcg_at_1000_diff1": 0.839869, "nauc_map_at_1_max": 0.071494, "nauc_map_at_1_std": -0.574049, "nauc_map_at_1_diff1": 0.85592, "nauc_map_at_3_max": 0.168506, "nauc_map_at_3_std": -0.616759, "nauc_map_at_3_diff1": 0.829642, "nauc_map_at_5_max": 0.158708, "nauc_map_at_5_std": -0.633542, "nauc_map_at_5_diff1": 0.840851, "nauc_map_at_10_max": 0.142977, "nauc_map_at_10_std": -0.625369, "nauc_map_at_10_diff1": 0.840596, "nauc_map_at_20_max": 0.140008, "nauc_map_at_20_std": -0.622612, "nauc_map_at_20_diff1": 0.841608, "nauc_map_at_100_max": 0.140008, "nauc_map_at_100_std": -0.622612, "nauc_map_at_100_diff1": 0.841608, "nauc_map_at_1000_max": 0.140008, "nauc_map_at_1000_std": -0.622612, "nauc_map_at_1000_diff1": 0.841608, "nauc_recall_at_1_max": 0.071494, "nauc_recall_at_1_std": -0.574049, "nauc_recall_at_1_diff1": 0.85592, "nauc_recall_at_3_max": 0.567546, "nauc_recall_at_3_std": -0.693225, "nauc_recall_at_3_diff1": 0.719042, "nauc_recall_at_5_max": 0.734426, "nauc_recall_at_5_std": -0.999727, "nauc_recall_at_5_diff1": 0.803349, "nauc_recall_at_10_max": 0.539967, "nauc_recall_at_10_std": -0.962931, "nauc_recall_at_10_diff1": 0.712274, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.071494, "nauc_precision_at_1_std": -0.574049, "nauc_precision_at_1_diff1": 0.85592, "nauc_precision_at_3_max": 0.567546, "nauc_precision_at_3_std": -0.693225, "nauc_precision_at_3_diff1": 0.719042, "nauc_precision_at_5_max": 0.734426, "nauc_precision_at_5_std": -0.999727, "nauc_precision_at_5_diff1": 0.803349, "nauc_precision_at_10_max": 0.539967, "nauc_precision_at_10_std": -0.962931, "nauc_precision_at_10_diff1": 0.712274, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_1000_diff1": 1.0, "nauc_mrr_at_1_max": 0.071494, "nauc_mrr_at_1_std": -0.574049, "nauc_mrr_at_1_diff1": 0.85592, "nauc_mrr_at_3_max": 0.168506, "nauc_mrr_at_3_std": -0.616759, "nauc_mrr_at_3_diff1": 0.829642, "nauc_mrr_at_5_max": 0.158708, "nauc_mrr_at_5_std": -0.633542, "nauc_mrr_at_5_diff1": 0.840851, "nauc_mrr_at_10_max": 0.142977, "nauc_mrr_at_10_std": -0.625369, "nauc_mrr_at_10_diff1": 0.840596, "nauc_mrr_at_20_max": 0.140008, "nauc_mrr_at_20_std": -0.622612, "nauc_mrr_at_20_diff1": 0.841608, "nauc_mrr_at_100_max": 0.140008, "nauc_mrr_at_100_std": -0.622612, "nauc_mrr_at_100_diff1": 0.841608, "nauc_mrr_at_1000_max": 0.140008, "nauc_mrr_at_1000_std": -0.622612, "nauc_mrr_at_1000_diff1": 0.841608, "main_score": 0.87076, "hf_subset": "default", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 23.355713367462158, "kg_co2_emissions": null } ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "task_name": "BelebeleRetrieval", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.90778, "ndcg_at_3": 0.94389, "ndcg_at_5": 0.948, "ndcg_at_10": 0.95019, "ndcg_at_20": 0.95047, "ndcg_at_100": 0.95212, "ndcg_at_1000": 0.95285, "map_at_1": 0.90778, "map_at_3": 0.93556, "map_at_5": 0.93783, "map_at_10": 0.93876, "map_at_20": 0.93883, "map_at_100": 0.93907, "map_at_1000": 0.9391, "recall_at_1": 0.90778, "recall_at_3": 0.96778, "recall_at_5": 0.97778, "recall_at_10": 0.98444, "recall_at_20": 0.98556, "recall_at_100": 0.99444, "recall_at_1000": 1.0, "precision_at_1": 0.90778, "precision_at_3": 0.32259, "precision_at_5": 0.19556, "precision_at_10": 0.09844, "precision_at_20": 0.04928, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "mrr_at_1": 0.907778, "mrr_at_3": 0.935556, "mrr_at_5": 0.937833, "mrr_at_10": 0.938756, "mrr_at_20": 0.93883, "mrr_at_100": 0.939068, "mrr_at_1000": 0.939097, "nauc_ndcg_at_1_max": 0.803545, "nauc_ndcg_at_1_std": -0.058582, "nauc_ndcg_at_1_diff1": 0.924882, "nauc_ndcg_at_3_max": 0.84107, "nauc_ndcg_at_3_std": -0.042141, "nauc_ndcg_at_3_diff1": 0.930107, "nauc_ndcg_at_5_max": 0.838696, "nauc_ndcg_at_5_std": -0.006393, "nauc_ndcg_at_5_diff1": 0.928079, "nauc_ndcg_at_10_max": 0.835397, "nauc_ndcg_at_10_std": -0.034705, "nauc_ndcg_at_10_diff1": 0.928834, "nauc_ndcg_at_20_max": 0.834474, "nauc_ndcg_at_20_std": -0.035588, "nauc_ndcg_at_20_diff1": 0.929168, "nauc_ndcg_at_100_max": 0.832028, "nauc_ndcg_at_100_std": -0.023947, "nauc_ndcg_at_100_diff1": 0.928054, "nauc_ndcg_at_1000_max": 0.829818, "nauc_ndcg_at_1000_std": -0.034697, "nauc_ndcg_at_1000_diff1": 0.927751, "nauc_map_at_1_max": 0.803545, "nauc_map_at_1_std": -0.058582, "nauc_map_at_1_diff1": 0.924882, "nauc_map_at_3_max": 0.828584, "nauc_map_at_3_std": -0.050976, "nauc_map_at_3_diff1": 0.927852, "nauc_map_at_5_max": 0.827058, "nauc_map_at_5_std": -0.034059, "nauc_map_at_5_diff1": 0.926786, "nauc_map_at_10_max": 0.825674, "nauc_map_at_10_std": -0.044254, "nauc_map_at_10_diff1": 0.926887, "nauc_map_at_20_max": 0.825463, "nauc_map_at_20_std": -0.044456, "nauc_map_at_20_diff1": 0.926957, "nauc_map_at_100_max": 0.825211, "nauc_map_at_100_std": -0.042846, "nauc_map_at_100_diff1": 0.926884, "nauc_map_at_1000_max": 0.825138, "nauc_map_at_1000_std": -0.043204, "nauc_map_at_1000_diff1": 0.926874, "nauc_recall_at_1_max": 0.803545, "nauc_recall_at_1_std": -0.058582, "nauc_recall_at_1_diff1": 0.924882, "nauc_recall_at_3_max": 0.912747, "nauc_recall_at_3_std": 0.011462, "nauc_recall_at_3_diff1": 0.943495, "nauc_recall_at_5_max": 0.931139, "nauc_recall_at_5_std": 0.231162, "nauc_recall_at_5_diff1": 0.938492, "nauc_recall_at_10_max": 0.940143, "nauc_recall_at_10_std": 0.058023, "nauc_recall_at_10_diff1": 0.953315, "nauc_recall_at_20_max": 0.935538, "nauc_recall_at_20_std": 0.053042, "nauc_recall_at_20_diff1": 0.959779, "nauc_recall_at_100_max": 0.973856, "nauc_recall_at_100_std": 0.661158, "nauc_recall_at_100_diff1": 0.947712, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.803545, "nauc_precision_at_1_std": -0.058582, "nauc_precision_at_1_diff1": 0.924882, "nauc_precision_at_3_max": 0.912747, "nauc_precision_at_3_std": 0.011462, "nauc_precision_at_3_diff1": 0.943495, "nauc_precision_at_5_max": 0.931139, "nauc_precision_at_5_std": 0.231162, "nauc_precision_at_5_diff1": 0.938492, "nauc_precision_at_10_max": 0.940143, "nauc_precision_at_10_std": 0.058023, "nauc_precision_at_10_diff1": 0.953315, "nauc_precision_at_20_max": 0.935538, "nauc_precision_at_20_std": 0.053042, "nauc_precision_at_20_diff1": 0.959779, "nauc_precision_at_100_max": 0.973856, "nauc_precision_at_100_std": 0.661158, "nauc_precision_at_100_diff1": 0.947712, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.803545, "nauc_mrr_at_1_std": -0.058582, "nauc_mrr_at_1_diff1": 0.924882, "nauc_mrr_at_3_max": 0.828584, "nauc_mrr_at_3_std": -0.050976, "nauc_mrr_at_3_diff1": 0.927852, "nauc_mrr_at_5_max": 0.827058, "nauc_mrr_at_5_std": -0.034059, "nauc_mrr_at_5_diff1": 0.926786, "nauc_mrr_at_10_max": 0.825674, "nauc_mrr_at_10_std": -0.044254, "nauc_mrr_at_10_diff1": 0.926887, "nauc_mrr_at_20_max": 0.825463, "nauc_mrr_at_20_std": -0.044456, "nauc_mrr_at_20_diff1": 0.926957, "nauc_mrr_at_100_max": 0.825211, "nauc_mrr_at_100_std": -0.042846, "nauc_mrr_at_100_diff1": 0.926884, "nauc_mrr_at_1000_max": 0.825138, "nauc_mrr_at_1000_std": -0.043204, "nauc_mrr_at_1000_diff1": 0.926874, "main_score": 0.95019, "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ] }, { "ndcg_at_1": 0.85556, "ndcg_at_3": 0.91315, "ndcg_at_5": 0.91587, "ndcg_at_10": 0.92367, "ndcg_at_20": 0.92533, "ndcg_at_100": 0.92614, "ndcg_at_1000": 0.92671, "map_at_1": 0.85556, "map_at_3": 0.89889, "map_at_5": 0.90039, "map_at_10": 0.90376, "map_at_20": 0.9042, "map_at_100": 0.90431, "map_at_1000": 0.90433, "recall_at_1": 0.85556, "recall_at_3": 0.95444, "recall_at_5": 0.96111, "recall_at_10": 0.98444, "recall_at_20": 0.99111, "recall_at_100": 0.99556, "recall_at_1000": 1.0, "precision_at_1": 0.85556, "precision_at_3": 0.31815, "precision_at_5": 0.19222, "precision_at_10": 0.09844, "precision_at_20": 0.04956, "precision_at_100": 0.00996, "precision_at_1000": 0.001, "mrr_at_1": 0.855556, "mrr_at_3": 0.898889, "mrr_at_5": 0.900389, "mrr_at_10": 0.903761, "mrr_at_20": 0.904204, "mrr_at_100": 0.904305, "mrr_at_1000": 0.904328, "nauc_ndcg_at_1_max": 0.740968, "nauc_ndcg_at_1_std": -0.085018, "nauc_ndcg_at_1_diff1": 0.906555, "nauc_ndcg_at_3_max": 0.7718, "nauc_ndcg_at_3_std": -0.020834, "nauc_ndcg_at_3_diff1": 0.887238, "nauc_ndcg_at_5_max": 0.766817, "nauc_ndcg_at_5_std": -0.024873, "nauc_ndcg_at_5_diff1": 0.88571, "nauc_ndcg_at_10_max": 0.764196, "nauc_ndcg_at_10_std": -0.011096, "nauc_ndcg_at_10_diff1": 0.886106, "nauc_ndcg_at_20_max": 0.760449, "nauc_ndcg_at_20_std": -0.027665, "nauc_ndcg_at_20_diff1": 0.887146, "nauc_ndcg_at_100_max": 0.761483, "nauc_ndcg_at_100_std": -0.02794, "nauc_ndcg_at_100_diff1": 0.89009, "nauc_ndcg_at_1000_max": 0.759555, "nauc_ndcg_at_1000_std": -0.034295, "nauc_ndcg_at_1000_diff1": 0.890284, "nauc_map_at_1_max": 0.740968, "nauc_map_at_1_std": -0.085018, "nauc_map_at_1_diff1": 0.906555, "nauc_map_at_3_max": 0.76093, "nauc_map_at_3_std": -0.043628, "nauc_map_at_3_diff1": 0.892675, "nauc_map_at_5_max": 0.758503, "nauc_map_at_5_std": -0.045672, "nauc_map_at_5_diff1": 0.892024, "nauc_map_at_10_max": 0.7573, "nauc_map_at_10_std": -0.041865, "nauc_map_at_10_diff1": 0.892401, "nauc_map_at_20_max": 0.756524, "nauc_map_at_20_std": -0.045332, "nauc_map_at_20_diff1": 0.892601, "nauc_map_at_100_max": 0.756602, "nauc_map_at_100_std": -0.045248, "nauc_map_at_100_diff1": 0.892845, "nauc_map_at_1000_max": 0.756542, "nauc_map_at_1000_std": -0.04542, "nauc_map_at_1000_diff1": 0.89285, "nauc_recall_at_1_max": 0.740968, "nauc_recall_at_1_std": -0.085018, "nauc_recall_at_1_diff1": 0.906555, "nauc_recall_at_3_max": 0.839755, "nauc_recall_at_3_std": 0.121815, "nauc_recall_at_3_diff1": 0.854433, "nauc_recall_at_5_max": 0.825023, "nauc_recall_at_5_std": 0.123436, "nauc_recall_at_5_diff1": 0.84115, "nauc_recall_at_10_max": 0.87445, "nauc_recall_at_10_std": 0.546552, "nauc_recall_at_10_diff1": 0.779445, "nauc_recall_at_20_max": 0.831349, "nauc_recall_at_20_std": 0.411881, "nauc_recall_at_20_diff1": 0.737162, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.780696, "nauc_recall_at_100_diff1": 0.865196, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.740968, "nauc_precision_at_1_std": -0.085018, "nauc_precision_at_1_diff1": 0.906555, "nauc_precision_at_3_max": 0.839755, "nauc_precision_at_3_std": 0.121815, "nauc_precision_at_3_diff1": 0.854433, "nauc_precision_at_5_max": 0.825023, "nauc_precision_at_5_std": 0.123436, "nauc_precision_at_5_diff1": 0.84115, "nauc_precision_at_10_max": 0.87445, "nauc_precision_at_10_std": 0.546552, "nauc_precision_at_10_diff1": 0.779445, "nauc_precision_at_20_max": 0.831349, "nauc_precision_at_20_std": 0.411881, "nauc_precision_at_20_diff1": 0.737162, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.780696, "nauc_precision_at_100_diff1": 0.865196, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.740968, "nauc_mrr_at_1_std": -0.085018, "nauc_mrr_at_1_diff1": 0.906555, "nauc_mrr_at_3_max": 0.76093, "nauc_mrr_at_3_std": -0.043628, "nauc_mrr_at_3_diff1": 0.892675, "nauc_mrr_at_5_max": 0.758503, "nauc_mrr_at_5_std": -0.045672, "nauc_mrr_at_5_diff1": 0.892024, "nauc_mrr_at_10_max": 0.7573, "nauc_mrr_at_10_std": -0.041865, "nauc_mrr_at_10_diff1": 0.892401, "nauc_mrr_at_20_max": 0.756524, "nauc_mrr_at_20_std": -0.045332, "nauc_mrr_at_20_diff1": 0.892601, "nauc_mrr_at_100_max": 0.756602, "nauc_mrr_at_100_std": -0.045248, "nauc_mrr_at_100_diff1": 0.892845, "nauc_mrr_at_1000_max": 0.756542, "nauc_mrr_at_1000_std": -0.04542, "nauc_mrr_at_1000_diff1": 0.89285, "main_score": 0.92367, "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ] }, { "ndcg_at_1": 0.84, "ndcg_at_3": 0.8923, "ndcg_at_5": 0.90148, "ndcg_at_10": 0.90577, "ndcg_at_20": 0.91004, "ndcg_at_100": 0.91226, "ndcg_at_1000": 0.91319, "map_at_1": 0.84, "map_at_3": 0.87963, "map_at_5": 0.88474, "map_at_10": 0.88649, "map_at_20": 0.88769, "map_at_100": 0.88798, "map_at_1000": 0.88803, "recall_at_1": 0.84, "recall_at_3": 0.92889, "recall_at_5": 0.95111, "recall_at_10": 0.96444, "recall_at_20": 0.98111, "recall_at_100": 0.99333, "recall_at_1000": 1.0, "precision_at_1": 0.84, "precision_at_3": 0.30963, "precision_at_5": 0.19022, "precision_at_10": 0.09644, "precision_at_20": 0.04906, "precision_at_100": 0.00993, "precision_at_1000": 0.001, "mrr_at_1": 0.84, "mrr_at_3": 0.87963, "mrr_at_5": 0.884741, "mrr_at_10": 0.886493, "mrr_at_20": 0.887693, "mrr_at_100": 0.887978, "mrr_at_1000": 0.888027, "nauc_ndcg_at_1_max": 0.669871, "nauc_ndcg_at_1_std": -0.013885, "nauc_ndcg_at_1_diff1": 0.884402, "nauc_ndcg_at_3_max": 0.727295, "nauc_ndcg_at_3_std": 0.071507, "nauc_ndcg_at_3_diff1": 0.873162, "nauc_ndcg_at_5_max": 0.71879, "nauc_ndcg_at_5_std": 0.044811, "nauc_ndcg_at_5_diff1": 0.876589, "nauc_ndcg_at_10_max": 0.718449, "nauc_ndcg_at_10_std": 0.060303, "nauc_ndcg_at_10_diff1": 0.877957, "nauc_ndcg_at_20_max": 0.719446, "nauc_ndcg_at_20_std": 0.069762, "nauc_ndcg_at_20_diff1": 0.880694, "nauc_ndcg_at_100_max": 0.714788, "nauc_ndcg_at_100_std": 0.056841, "nauc_ndcg_at_100_diff1": 0.879395, "nauc_ndcg_at_1000_max": 0.712396, "nauc_ndcg_at_1000_std": 0.050198, "nauc_ndcg_at_1000_diff1": 0.878525, "nauc_map_at_1_max": 0.669871, "nauc_map_at_1_std": -0.013885, "nauc_map_at_1_diff1": 0.884402, "nauc_map_at_3_max": 0.711326, "nauc_map_at_3_std": 0.04492, "nauc_map_at_3_diff1": 0.875477, "nauc_map_at_5_max": 0.706614, "nauc_map_at_5_std": 0.030796, "nauc_map_at_5_diff1": 0.877186, "nauc_map_at_10_max": 0.70648, "nauc_map_at_10_std": 0.036856, "nauc_map_at_10_diff1": 0.877641, "nauc_map_at_20_max": 0.70693, "nauc_map_at_20_std": 0.03963, "nauc_map_at_20_diff1": 0.878354, "nauc_map_at_100_max": 0.706407, "nauc_map_at_100_std": 0.03814, "nauc_map_at_100_diff1": 0.878192, "nauc_map_at_1000_max": 0.706298, "nauc_map_at_1000_std": 0.037843, "nauc_map_at_1000_diff1": 0.878154, "nauc_recall_at_1_max": 0.669871, "nauc_recall_at_1_std": -0.013885, "nauc_recall_at_1_diff1": 0.884402, "nauc_recall_at_3_max": 0.801317, "nauc_recall_at_3_std": 0.196932, "nauc_recall_at_3_diff1": 0.863051, "nauc_recall_at_5_max": 0.793874, "nauc_recall_at_5_std": 0.128873, "nauc_recall_at_5_diff1": 0.875021, "nauc_recall_at_10_max": 0.816279, "nauc_recall_at_10_std": 0.267419, "nauc_recall_at_10_diff1": 0.885636, "nauc_recall_at_20_max": 0.905943, "nauc_recall_at_20_std": 0.583512, "nauc_recall_at_20_diff1": 0.937524, "nauc_recall_at_100_max": 0.925692, "nauc_recall_at_100_std": 0.643324, "nauc_recall_at_100_diff1": 0.956427, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.669871, "nauc_precision_at_1_std": -0.013885, "nauc_precision_at_1_diff1": 0.884402, "nauc_precision_at_3_max": 0.801317, "nauc_precision_at_3_std": 0.196932, "nauc_precision_at_3_diff1": 0.863051, "nauc_precision_at_5_max": 0.793874, "nauc_precision_at_5_std": 0.128873, "nauc_precision_at_5_diff1": 0.875021, "nauc_precision_at_10_max": 0.816279, "nauc_precision_at_10_std": 0.267419, "nauc_precision_at_10_diff1": 0.885636, "nauc_precision_at_20_max": 0.905943, "nauc_precision_at_20_std": 0.583512, "nauc_precision_at_20_diff1": 0.937524, "nauc_precision_at_100_max": 0.925692, "nauc_precision_at_100_std": 0.643324, "nauc_precision_at_100_diff1": 0.956427, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.669871, "nauc_mrr_at_1_std": -0.013885, "nauc_mrr_at_1_diff1": 0.884402, "nauc_mrr_at_3_max": 0.711326, "nauc_mrr_at_3_std": 0.04492, "nauc_mrr_at_3_diff1": 0.875477, "nauc_mrr_at_5_max": 0.706614, "nauc_mrr_at_5_std": 0.030796, "nauc_mrr_at_5_diff1": 0.877186, "nauc_mrr_at_10_max": 0.70648, "nauc_mrr_at_10_std": 0.036856, "nauc_mrr_at_10_diff1": 0.877641, "nauc_mrr_at_20_max": 0.70693, "nauc_mrr_at_20_std": 0.03963, "nauc_mrr_at_20_diff1": 0.878354, "nauc_mrr_at_100_max": 0.706407, "nauc_mrr_at_100_std": 0.03814, "nauc_mrr_at_100_diff1": 0.878192, "nauc_mrr_at_1000_max": 0.706298, "nauc_mrr_at_1000_std": 0.037843, "nauc_mrr_at_1000_diff1": 0.878154, "main_score": 0.90577, "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ] } ] }, "evaluation_time": 19.279202222824097, "kg_co2_emissions": null } ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "task_name": "Ko-StrategyQA", "mteb_version": "1.21.6", "scores": { "dev": [ { "ndcg_at_1": 0.76182, "ndcg_at_3": 0.7545, "ndcg_at_5": 0.78417, "ndcg_at_10": 0.7999, "ndcg_at_20": 0.80687, "ndcg_at_100": 0.8145, "ndcg_at_1000": 0.82292, "map_at_1": 0.49595, "map_at_3": 0.71511, "map_at_5": 0.74568, "map_at_10": 0.7559, "map_at_20": 0.75907, "map_at_100": 0.76054, "map_at_1000": 0.76093, "recall_at_1": 0.49595, "recall_at_3": 0.76143, "recall_at_5": 0.82177, "recall_at_10": 0.85963, "recall_at_20": 0.88024, "recall_at_100": 0.91352, "recall_at_1000": 0.96824, "precision_at_1": 0.76182, "precision_at_3": 0.4482, "precision_at_5": 0.3, "precision_at_10": 0.15912, "precision_at_20": 0.08226, "precision_at_100": 0.01718, "precision_at_1000": 0.00184, "mrr_at_1": 0.761824, "mrr_at_3": 0.808277, "mrr_at_5": 0.812669, "mrr_at_10": 0.815498, "mrr_at_20": 0.816074, "mrr_at_100": 0.816796, "mrr_at_1000": 0.816951, "nauc_ndcg_at_1_max": 0.614025, "nauc_ndcg_at_1_std": 0.261642, "nauc_ndcg_at_1_diff1": 0.701575, "nauc_ndcg_at_3_max": 0.62098, "nauc_ndcg_at_3_std": 0.230317, "nauc_ndcg_at_3_diff1": 0.625084, "nauc_ndcg_at_5_max": 0.668734, "nauc_ndcg_at_5_std": 0.300087, "nauc_ndcg_at_5_diff1": 0.630877, "nauc_ndcg_at_10_max": 0.685111, "nauc_ndcg_at_10_std": 0.344968, "nauc_ndcg_at_10_diff1": 0.640037, "nauc_ndcg_at_20_max": 0.689795, "nauc_ndcg_at_20_std": 0.357351, "nauc_ndcg_at_20_diff1": 0.641725, "nauc_ndcg_at_100_max": 0.688772, "nauc_ndcg_at_100_std": 0.361662, "nauc_ndcg_at_100_diff1": 0.643659, "nauc_ndcg_at_1000_max": 0.677585, "nauc_ndcg_at_1000_std": 0.345124, "nauc_ndcg_at_1000_diff1": 0.642909, "nauc_map_at_1_max": 0.272905, "nauc_map_at_1_std": 0.002876, "nauc_map_at_1_diff1": 0.646467, "nauc_map_at_3_max": 0.567092, "nauc_map_at_3_std": 0.175638, "nauc_map_at_3_diff1": 0.615539, "nauc_map_at_5_max": 0.609239, "nauc_map_at_5_std": 0.223555, "nauc_map_at_5_diff1": 0.616314, "nauc_map_at_10_max": 0.619726, "nauc_map_at_10_std": 0.250831, "nauc_map_at_10_diff1": 0.620354, "nauc_map_at_20_max": 0.621416, "nauc_map_at_20_std": 0.255703, "nauc_map_at_20_diff1": 0.620546, "nauc_map_at_100_max": 0.621935, "nauc_map_at_100_std": 0.257508, "nauc_map_at_100_diff1": 0.621364, "nauc_map_at_1000_max": 0.621471, "nauc_map_at_1000_std": 0.256787, "nauc_map_at_1000_diff1": 0.6213, "nauc_recall_at_1_max": 0.272905, "nauc_recall_at_1_std": 0.002876, "nauc_recall_at_1_diff1": 0.646467, "nauc_recall_at_3_max": 0.638878, "nauc_recall_at_3_std": 0.234589, "nauc_recall_at_3_diff1": 0.59418, "nauc_recall_at_5_max": 0.747306, "nauc_recall_at_5_std": 0.391205, "nauc_recall_at_5_diff1": 0.597477, "nauc_recall_at_10_max": 0.811865, "nauc_recall_at_10_std": 0.547802, "nauc_recall_at_10_diff1": 0.622797, "nauc_recall_at_20_max": 0.859455, "nauc_recall_at_20_std": 0.63551, "nauc_recall_at_20_diff1": 0.636328, "nauc_recall_at_100_max": 0.901394, "nauc_recall_at_100_std": 0.750433, "nauc_recall_at_100_diff1": 0.646077, "nauc_recall_at_1000_max": 0.916075, "nauc_recall_at_1000_std": 0.879498, "nauc_recall_at_1000_diff1": 0.662774, "nauc_precision_at_1_max": 0.614025, "nauc_precision_at_1_std": 0.261642, "nauc_precision_at_1_diff1": 0.701575, "nauc_precision_at_3_max": 0.404117, "nauc_precision_at_3_std": 0.279052, "nauc_precision_at_3_diff1": 0.014184, "nauc_precision_at_5_max": 0.330371, "nauc_precision_at_5_std": 0.305218, "nauc_precision_at_5_diff1": -0.083218, "nauc_precision_at_10_max": 0.278977, "nauc_precision_at_10_std": 0.335057, "nauc_precision_at_10_diff1": -0.133429, "nauc_precision_at_20_max": 0.239069, "nauc_precision_at_20_std": 0.330963, "nauc_precision_at_20_diff1": -0.16744, "nauc_precision_at_100_max": 0.176842, "nauc_precision_at_100_std": 0.311736, "nauc_precision_at_100_diff1": -0.225768, "nauc_precision_at_1000_max": 0.025097, "nauc_precision_at_1000_std": 0.201658, "nauc_precision_at_1000_diff1": -0.352137, "nauc_mrr_at_1_max": 0.614025, "nauc_mrr_at_1_std": 0.261642, "nauc_mrr_at_1_diff1": 0.701575, "nauc_mrr_at_3_max": 0.701277, "nauc_mrr_at_3_std": 0.3518, "nauc_mrr_at_3_diff1": 0.695926, "nauc_mrr_at_5_max": 0.702635, "nauc_mrr_at_5_std": 0.367677, "nauc_mrr_at_5_diff1": 0.695743, "nauc_mrr_at_10_max": 0.702514, "nauc_mrr_at_10_std": 0.37052, "nauc_mrr_at_10_diff1": 0.699126, "nauc_mrr_at_20_max": 0.70173, "nauc_mrr_at_20_std": 0.369437, "nauc_mrr_at_20_diff1": 0.698954, "nauc_mrr_at_100_max": 0.700801, "nauc_mrr_at_100_std": 0.368442, "nauc_mrr_at_100_diff1": 0.698849, "nauc_mrr_at_1000_max": 0.700591, "nauc_mrr_at_1000_std": 0.368133, "nauc_mrr_at_1000_diff1": 0.698854, "main_score": 0.7999, "hf_subset": "default", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 120.75947451591492, "kg_co2_emissions": null } ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "task_name": "MIRACLRetrieval", "mteb_version": "1.21.6", "scores": { "dev": [ { "ndcg_at_1": 0.64319, "ndcg_at_3": 0.63092, "ndcg_at_5": 0.64564, "ndcg_at_10": 0.68157, "ndcg_at_20": 0.71623, "ndcg_at_100": 0.73858, "ndcg_at_1000": 0.7451, "map_at_1": 0.40126, "map_at_3": 0.51814, "map_at_5": 0.55611, "map_at_10": 0.59121, "map_at_20": 0.61048, "map_at_100": 0.61787, "map_at_1000": 0.61845, "recall_at_1": 0.40126, "recall_at_3": 0.58407, "recall_at_5": 0.66815, "recall_at_10": 0.77895, "recall_at_20": 0.87459, "recall_at_100": 0.95436, "recall_at_1000": 0.99296, "precision_at_1": 0.64319, "precision_at_3": 0.37715, "precision_at_5": 0.28263, "precision_at_10": 0.177, "precision_at_20": 0.1061, "precision_at_100": 0.02451, "precision_at_1000": 0.00256, "mrr_at_1": 0.643192, "mrr_at_3": 0.722222, "mrr_at_5": 0.731612, "mrr_at_10": 0.740785, "mrr_at_20": 0.743542, "mrr_at_100": 0.743799, "mrr_at_1000": 0.74388, "nauc_ndcg_at_1_max": 0.418025, "nauc_ndcg_at_1_std": 0.141527, "nauc_ndcg_at_1_diff1": 0.350911, "nauc_ndcg_at_3_max": 0.389608, "nauc_ndcg_at_3_std": 0.066272, "nauc_ndcg_at_3_diff1": 0.297137, "nauc_ndcg_at_5_max": 0.3766, "nauc_ndcg_at_5_std": 0.031175, "nauc_ndcg_at_5_diff1": 0.313027, "nauc_ndcg_at_10_max": 0.392274, "nauc_ndcg_at_10_std": 0.06215, "nauc_ndcg_at_10_diff1": 0.296271, "nauc_ndcg_at_20_max": 0.41232, "nauc_ndcg_at_20_std": 0.126069, "nauc_ndcg_at_20_diff1": 0.290829, "nauc_ndcg_at_100_max": 0.430835, "nauc_ndcg_at_100_std": 0.145252, "nauc_ndcg_at_100_diff1": 0.294684, "nauc_ndcg_at_1000_max": 0.425904, "nauc_ndcg_at_1000_std": 0.135031, "nauc_ndcg_at_1000_diff1": 0.293088, "nauc_map_at_1_max": 0.241848, "nauc_map_at_1_std": -0.118254, "nauc_map_at_1_diff1": 0.457322, "nauc_map_at_3_max": 0.303597, "nauc_map_at_3_std": -0.064791, "nauc_map_at_3_diff1": 0.366994, "nauc_map_at_5_max": 0.317647, "nauc_map_at_5_std": -0.048155, "nauc_map_at_5_diff1": 0.349392, "nauc_map_at_10_max": 0.347327, "nauc_map_at_10_std": -0.002871, "nauc_map_at_10_diff1": 0.327944, "nauc_map_at_20_max": 0.359119, "nauc_map_at_20_std": 0.032617, "nauc_map_at_20_diff1": 0.31913, "nauc_map_at_100_max": 0.366819, "nauc_map_at_100_std": 0.04224, "nauc_map_at_100_diff1": 0.318221, "nauc_map_at_1000_max": 0.366912, "nauc_map_at_1000_std": 0.041759, "nauc_map_at_1000_diff1": 0.318344, "nauc_recall_at_1_max": 0.241848, "nauc_recall_at_1_std": -0.118254, "nauc_recall_at_1_diff1": 0.457322, "nauc_recall_at_3_max": 0.299411, "nauc_recall_at_3_std": -0.086247, "nauc_recall_at_3_diff1": 0.317932, "nauc_recall_at_5_max": 0.289107, "nauc_recall_at_5_std": -0.113191, "nauc_recall_at_5_diff1": 0.287347, "nauc_recall_at_10_max": 0.329945, "nauc_recall_at_10_std": 0.019325, "nauc_recall_at_10_diff1": 0.211357, "nauc_recall_at_20_max": 0.391822, "nauc_recall_at_20_std": 0.255635, "nauc_recall_at_20_diff1": 0.218277, "nauc_recall_at_100_max": 0.647316, "nauc_recall_at_100_std": 0.566925, "nauc_recall_at_100_diff1": 0.353585, "nauc_recall_at_1000_max": 1.0, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_1000_diff1": 0.609708, "nauc_precision_at_1_max": 0.418025, "nauc_precision_at_1_std": 0.141527, "nauc_precision_at_1_diff1": 0.350911, "nauc_precision_at_3_max": 0.277168, "nauc_precision_at_3_std": 0.296765, "nauc_precision_at_3_diff1": -0.152035, "nauc_precision_at_5_max": 0.201806, "nauc_precision_at_5_std": 0.301295, "nauc_precision_at_5_diff1": -0.206681, "nauc_precision_at_10_max": 0.178954, "nauc_precision_at_10_std": 0.372459, "nauc_precision_at_10_diff1": -0.270715, "nauc_precision_at_20_max": 0.139584, "nauc_precision_at_20_std": 0.417198, "nauc_precision_at_20_diff1": -0.286054, "nauc_precision_at_100_max": 0.117258, "nauc_precision_at_100_std": 0.403073, "nauc_precision_at_100_diff1": -0.271039, "nauc_precision_at_1000_max": 0.09153, "nauc_precision_at_1000_std": 0.370089, "nauc_precision_at_1000_diff1": -0.279217, "nauc_mrr_at_1_max": 0.418025, "nauc_mrr_at_1_std": 0.141527, "nauc_mrr_at_1_diff1": 0.350911, "nauc_mrr_at_3_max": 0.476687, "nauc_mrr_at_3_std": 0.19241, "nauc_mrr_at_3_diff1": 0.294637, "nauc_mrr_at_5_max": 0.468326, "nauc_mrr_at_5_std": 0.172536, "nauc_mrr_at_5_diff1": 0.29644, "nauc_mrr_at_10_max": 0.472805, "nauc_mrr_at_10_std": 0.186911, "nauc_mrr_at_10_diff1": 0.299354, "nauc_mrr_at_20_max": 0.469419, "nauc_mrr_at_20_std": 0.186845, "nauc_mrr_at_20_diff1": 0.299805, "nauc_mrr_at_100_max": 0.469152, "nauc_mrr_at_100_std": 0.186332, "nauc_mrr_at_100_diff1": 0.300123, "nauc_mrr_at_1000_max": 0.469017, "nauc_mrr_at_1000_std": 0.186168, "nauc_mrr_at_1000_diff1": 0.300252, "main_score": 0.68157, "hf_subset": "ko", "languages": [ "kor-Kore" ] } ] }, "evaluation_time": 10866.9804418087, "kg_co2_emissions": null } ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "task_name": "MrTidyRetrieval", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.44893, "ndcg_at_3": 0.53551, "ndcg_at_5": 0.56295, "ndcg_at_10": 0.59092, "ndcg_at_20": 0.60819, "ndcg_at_100": 0.62813, "ndcg_at_1000": 0.63513, "map_at_1": 0.41449, "map_at_3": 0.50205, "map_at_5": 0.51788, "map_at_10": 0.53133, "map_at_20": 0.53695, "map_at_100": 0.54014, "map_at_1000": 0.54042, "recall_at_1": 0.41449, "recall_at_3": 0.60293, "recall_at_5": 0.66667, "recall_at_10": 0.74624, "recall_at_20": 0.80918, "recall_at_100": 0.91013, "recall_at_1000": 0.96318, "precision_at_1": 0.44893, "precision_at_3": 0.22407, "precision_at_5": 0.15012, "precision_at_10": 0.08575, "precision_at_20": 0.04691, "precision_at_100": 0.01062, "precision_at_1000": 0.00112, "mrr_at_1": 0.448931, "mrr_at_3": 0.532462, "mrr_at_5": 0.54707, "mrr_at_10": 0.55664, "mrr_at_20": 0.560524, "mrr_at_100": 0.563012, "mrr_at_1000": 0.563258, "nauc_ndcg_at_1_max": 0.46382, "nauc_ndcg_at_1_std": 0.020944, "nauc_ndcg_at_1_diff1": 0.541324, "nauc_ndcg_at_3_max": 0.455264, "nauc_ndcg_at_3_std": 0.091935, "nauc_ndcg_at_3_diff1": 0.437052, "nauc_ndcg_at_5_max": 0.445921, "nauc_ndcg_at_5_std": 0.082727, "nauc_ndcg_at_5_diff1": 0.434188, "nauc_ndcg_at_10_max": 0.4541, "nauc_ndcg_at_10_std": 0.088446, "nauc_ndcg_at_10_diff1": 0.426756, "nauc_ndcg_at_20_max": 0.450817, "nauc_ndcg_at_20_std": 0.089577, "nauc_ndcg_at_20_diff1": 0.420949, "nauc_ndcg_at_100_max": 0.463446, "nauc_ndcg_at_100_std": 0.089819, "nauc_ndcg_at_100_diff1": 0.436819, "nauc_ndcg_at_1000_max": 0.459856, "nauc_ndcg_at_1000_std": 0.083848, "nauc_ndcg_at_1000_diff1": 0.4413, "nauc_map_at_1_max": 0.421572, "nauc_map_at_1_std": 0.00283, "nauc_map_at_1_diff1": 0.505448, "nauc_map_at_3_max": 0.437627, "nauc_map_at_3_std": 0.060121, "nauc_map_at_3_diff1": 0.448115, "nauc_map_at_5_max": 0.434329, "nauc_map_at_5_std": 0.056771, "nauc_map_at_5_diff1": 0.448608, "nauc_map_at_10_max": 0.436934, "nauc_map_at_10_std": 0.05951, "nauc_map_at_10_diff1": 0.444649, "nauc_map_at_20_max": 0.435557, "nauc_map_at_20_std": 0.059837, "nauc_map_at_20_diff1": 0.442731, "nauc_map_at_100_max": 0.437744, "nauc_map_at_100_std": 0.059625, "nauc_map_at_100_diff1": 0.445463, "nauc_map_at_1000_max": 0.437639, "nauc_map_at_1000_std": 0.0595, "nauc_map_at_1000_diff1": 0.445645, "nauc_recall_at_1_max": 0.421572, "nauc_recall_at_1_std": 0.00283, "nauc_recall_at_1_diff1": 0.505448, "nauc_recall_at_3_max": 0.457277, "nauc_recall_at_3_std": 0.16318, "nauc_recall_at_3_diff1": 0.362044, "nauc_recall_at_5_max": 0.428103, "nauc_recall_at_5_std": 0.137472, "nauc_recall_at_5_diff1": 0.344122, "nauc_recall_at_10_max": 0.459757, "nauc_recall_at_10_std": 0.168446, "nauc_recall_at_10_diff1": 0.300797, "nauc_recall_at_20_max": 0.450426, "nauc_recall_at_20_std": 0.198789, "nauc_recall_at_20_diff1": 0.242019, "nauc_recall_at_100_max": 0.630929, "nauc_recall_at_100_std": 0.30797, "nauc_recall_at_100_diff1": 0.299712, "nauc_recall_at_1000_max": 0.683232, "nauc_recall_at_1000_std": 0.282184, "nauc_recall_at_1000_diff1": 0.353519, "nauc_precision_at_1_max": 0.46382, "nauc_precision_at_1_std": 0.020944, "nauc_precision_at_1_diff1": 0.541324, "nauc_precision_at_3_max": 0.4482, "nauc_precision_at_3_std": 0.176583, "nauc_precision_at_3_diff1": 0.351772, "nauc_precision_at_5_max": 0.392259, "nauc_precision_at_5_std": 0.167015, "nauc_precision_at_5_diff1": 0.303985, "nauc_precision_at_10_max": 0.308423, "nauc_precision_at_10_std": 0.172186, "nauc_precision_at_10_diff1": 0.172134, "nauc_precision_at_20_max": 0.224559, "nauc_precision_at_20_std": 0.169829, "nauc_precision_at_20_diff1": 0.074729, "nauc_precision_at_100_max": 0.184778, "nauc_precision_at_100_std": 0.156845, "nauc_precision_at_100_diff1": 0.047777, "nauc_precision_at_1000_max": 0.077079, "nauc_precision_at_1000_std": 0.109687, "nauc_precision_at_1000_diff1": -0.01916, "nauc_mrr_at_1_max": 0.46382, "nauc_mrr_at_1_std": 0.020944, "nauc_mrr_at_1_diff1": 0.541324, "nauc_mrr_at_3_max": 0.484222, "nauc_mrr_at_3_std": 0.087616, "nauc_mrr_at_3_diff1": 0.482752, "nauc_mrr_at_5_max": 0.476133, "nauc_mrr_at_5_std": 0.080856, "nauc_mrr_at_5_diff1": 0.477907, "nauc_mrr_at_10_max": 0.480353, "nauc_mrr_at_10_std": 0.080922, "nauc_mrr_at_10_diff1": 0.478511, "nauc_mrr_at_20_max": 0.479131, "nauc_mrr_at_20_std": 0.079193, "nauc_mrr_at_20_diff1": 0.478262, "nauc_mrr_at_100_max": 0.480192, "nauc_mrr_at_100_std": 0.079458, "nauc_mrr_at_100_diff1": 0.479629, "nauc_mrr_at_1000_max": 0.480079, "nauc_mrr_at_1000_std": 0.079232, "nauc_mrr_at_1000_diff1": 0.47984, "main_score": 0.59092, "hf_subset": "korean", "languages": [ "kor-Kore" ] } ] }, "evaluation_time": 11014.441509008408, "kg_co2_emissions": null } ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "task_name": "MultiLongDocRetrieval", "mteb_version": "1.21.6", "scores": { "dev": [ { "ndcg_at_1": 0.37, "ndcg_at_3": 0.42797, "ndcg_at_5": 0.45465, "ndcg_at_10": 0.47528, "ndcg_at_20": 0.49279, "ndcg_at_100": 0.52728, "ndcg_at_1000": 0.54311, "map_at_1": 0.37, "map_at_3": 0.415, "map_at_5": 0.42975, "map_at_10": 0.43803, "map_at_20": 0.44274, "map_at_100": 0.44755, "map_at_1000": 0.44814, "recall_at_1": 0.37, "recall_at_3": 0.465, "recall_at_5": 0.53, "recall_at_10": 0.595, "recall_at_20": 0.665, "recall_at_100": 0.85, "recall_at_1000": 0.975, "precision_at_1": 0.37, "precision_at_3": 0.155, "precision_at_5": 0.106, "precision_at_10": 0.0595, "precision_at_20": 0.03325, "precision_at_100": 0.0085, "precision_at_1000": 0.00098, "mrr_at_1": 0.37, "mrr_at_3": 0.415, "mrr_at_5": 0.42975, "mrr_at_10": 0.438026, "mrr_at_20": 0.442737, "mrr_at_100": 0.447555, "mrr_at_1000": 0.448138, "nauc_ndcg_at_1_max": 0.452441, "nauc_ndcg_at_1_std": -0.018203, "nauc_ndcg_at_1_diff1": 0.65166, "nauc_ndcg_at_3_max": 0.477151, "nauc_ndcg_at_3_std": 0.037488, "nauc_ndcg_at_3_diff1": 0.624535, "nauc_ndcg_at_5_max": 0.468576, "nauc_ndcg_at_5_std": 0.035081, "nauc_ndcg_at_5_diff1": 0.610375, "nauc_ndcg_at_10_max": 0.467701, "nauc_ndcg_at_10_std": 0.051146, "nauc_ndcg_at_10_diff1": 0.60673, "nauc_ndcg_at_20_max": 0.491939, "nauc_ndcg_at_20_std": 0.065654, "nauc_ndcg_at_20_diff1": 0.601935, "nauc_ndcg_at_100_max": 0.49402, "nauc_ndcg_at_100_std": 0.083501, "nauc_ndcg_at_100_diff1": 0.597839, "nauc_ndcg_at_1000_max": 0.483865, "nauc_ndcg_at_1000_std": 0.067484, "nauc_ndcg_at_1000_diff1": 0.609425, "nauc_map_at_1_max": 0.452441, "nauc_map_at_1_std": -0.018203, "nauc_map_at_1_diff1": 0.65166, "nauc_map_at_3_max": 0.470701, "nauc_map_at_3_std": 0.024512, "nauc_map_at_3_diff1": 0.630357, "nauc_map_at_5_max": 0.466168, "nauc_map_at_5_std": 0.023049, "nauc_map_at_5_diff1": 0.622837, "nauc_map_at_10_max": 0.465591, "nauc_map_at_10_std": 0.030054, "nauc_map_at_10_diff1": 0.621022, "nauc_map_at_20_max": 0.47138, "nauc_map_at_20_std": 0.033448, "nauc_map_at_20_diff1": 0.620108, "nauc_map_at_100_max": 0.471999, "nauc_map_at_100_std": 0.035941, "nauc_map_at_100_diff1": 0.620158, "nauc_map_at_1000_max": 0.471719, "nauc_map_at_1000_std": 0.035547, "nauc_map_at_1000_diff1": 0.62059, "nauc_recall_at_1_max": 0.452441, "nauc_recall_at_1_std": -0.018203, "nauc_recall_at_1_diff1": 0.65166, "nauc_recall_at_3_max": 0.496048, "nauc_recall_at_3_std": 0.074911, "nauc_recall_at_3_diff1": 0.60796, "nauc_recall_at_5_max": 0.474398, "nauc_recall_at_5_std": 0.07064, "nauc_recall_at_5_diff1": 0.570888, "nauc_recall_at_10_max": 0.473087, "nauc_recall_at_10_std": 0.123547, "nauc_recall_at_10_diff1": 0.558398, "nauc_recall_at_20_max": 0.593887, "nauc_recall_at_20_std": 0.204443, "nauc_recall_at_20_diff1": 0.526963, "nauc_recall_at_100_max": 0.694891, "nauc_recall_at_100_std": 0.523706, "nauc_recall_at_100_diff1": 0.397706, "nauc_recall_at_1000_max": 0.645191, "nauc_recall_at_1000_std": 0.973856, "nauc_recall_at_1000_diff1": 0.482446, "nauc_precision_at_1_max": 0.452441, "nauc_precision_at_1_std": -0.018203, "nauc_precision_at_1_diff1": 0.65166, "nauc_precision_at_3_max": 0.496048, "nauc_precision_at_3_std": 0.074911, "nauc_precision_at_3_diff1": 0.60796, "nauc_precision_at_5_max": 0.474398, "nauc_precision_at_5_std": 0.07064, "nauc_precision_at_5_diff1": 0.570888, "nauc_precision_at_10_max": 0.473087, "nauc_precision_at_10_std": 0.123547, "nauc_precision_at_10_diff1": 0.558398, "nauc_precision_at_20_max": 0.593887, "nauc_precision_at_20_std": 0.204443, "nauc_precision_at_20_diff1": 0.526963, "nauc_precision_at_100_max": 0.694891, "nauc_precision_at_100_std": 0.523706, "nauc_precision_at_100_diff1": 0.397706, "nauc_precision_at_1000_max": 0.645191, "nauc_precision_at_1000_std": 0.973856, "nauc_precision_at_1000_diff1": 0.482446, "nauc_mrr_at_1_max": 0.452441, "nauc_mrr_at_1_std": -0.018203, "nauc_mrr_at_1_diff1": 0.65166, "nauc_mrr_at_3_max": 0.470701, "nauc_mrr_at_3_std": 0.024512, "nauc_mrr_at_3_diff1": 0.630357, "nauc_mrr_at_5_max": 0.466168, "nauc_mrr_at_5_std": 0.023049, "nauc_mrr_at_5_diff1": 0.622837, "nauc_mrr_at_10_max": 0.465591, "nauc_mrr_at_10_std": 0.030054, "nauc_mrr_at_10_diff1": 0.621022, "nauc_mrr_at_20_max": 0.47138, "nauc_mrr_at_20_std": 0.033448, "nauc_mrr_at_20_diff1": 0.620108, "nauc_mrr_at_100_max": 0.471999, "nauc_mrr_at_100_std": 0.035941, "nauc_mrr_at_100_diff1": 0.620158, "nauc_mrr_at_1000_max": 0.471719, "nauc_mrr_at_1000_std": 0.035547, "nauc_mrr_at_1000_diff1": 0.62059, "main_score": 0.47528, "hf_subset": "ko", "languages": [ "kor-Hang" ] } ], "test": [ { "ndcg_at_1": 0.345, "ndcg_at_3": 0.41678, "ndcg_at_5": 0.43335, "ndcg_at_10": 0.45209, "ndcg_at_20": 0.47345, "ndcg_at_100": 0.49803, "ndcg_at_1000": 0.51884, "map_at_1": 0.345, "map_at_3": 0.4, "map_at_5": 0.40925, "map_at_10": 0.4166, "map_at_20": 0.4224, "map_at_100": 0.42592, "map_at_1000": 0.42668, "recall_at_1": 0.345, "recall_at_3": 0.465, "recall_at_5": 0.505, "recall_at_10": 0.565, "recall_at_20": 0.65, "recall_at_100": 0.78, "recall_at_1000": 0.945, "precision_at_1": 0.345, "precision_at_3": 0.155, "precision_at_5": 0.101, "precision_at_10": 0.0565, "precision_at_20": 0.0325, "precision_at_100": 0.0078, "precision_at_1000": 0.00095, "mrr_at_1": 0.345, "mrr_at_3": 0.4, "mrr_at_5": 0.40925, "mrr_at_10": 0.416603, "mrr_at_20": 0.422402, "mrr_at_100": 0.425924, "mrr_at_1000": 0.426684, "nauc_ndcg_at_1_max": 0.454386, "nauc_ndcg_at_1_std": -0.039274, "nauc_ndcg_at_1_diff1": 0.651395, "nauc_ndcg_at_3_max": 0.437897, "nauc_ndcg_at_3_std": -0.001092, "nauc_ndcg_at_3_diff1": 0.617187, "nauc_ndcg_at_5_max": 0.444176, "nauc_ndcg_at_5_std": 0.016693, "nauc_ndcg_at_5_diff1": 0.61489, "nauc_ndcg_at_10_max": 0.454201, "nauc_ndcg_at_10_std": 0.034226, "nauc_ndcg_at_10_diff1": 0.624684, "nauc_ndcg_at_20_max": 0.452611, "nauc_ndcg_at_20_std": 0.046561, "nauc_ndcg_at_20_diff1": 0.617859, "nauc_ndcg_at_100_max": 0.453509, "nauc_ndcg_at_100_std": 0.05897, "nauc_ndcg_at_100_diff1": 0.614857, "nauc_ndcg_at_1000_max": 0.45323, "nauc_ndcg_at_1000_std": 0.054619, "nauc_ndcg_at_1000_diff1": 0.620109, "nauc_map_at_1_max": 0.454386, "nauc_map_at_1_std": -0.039274, "nauc_map_at_1_diff1": 0.651395, "nauc_map_at_3_max": 0.439205, "nauc_map_at_3_std": -0.009306, "nauc_map_at_3_diff1": 0.624352, "nauc_map_at_5_max": 0.442441, "nauc_map_at_5_std": 0.000586, "nauc_map_at_5_diff1": 0.622811, "nauc_map_at_10_max": 0.448083, "nauc_map_at_10_std": 0.008808, "nauc_map_at_10_diff1": 0.627215, "nauc_map_at_20_max": 0.44701, "nauc_map_at_20_std": 0.011688, "nauc_map_at_20_diff1": 0.6251, "nauc_map_at_100_max": 0.447019, "nauc_map_at_100_std": 0.012824, "nauc_map_at_100_diff1": 0.625064, "nauc_map_at_1000_max": 0.446926, "nauc_map_at_1000_std": 0.012784, "nauc_map_at_1000_diff1": 0.625156, "nauc_recall_at_1_max": 0.454386, "nauc_recall_at_1_std": -0.039274, "nauc_recall_at_1_diff1": 0.651395, "nauc_recall_at_3_max": 0.435222, "nauc_recall_at_3_std": 0.022206, "nauc_recall_at_3_diff1": 0.596949, "nauc_recall_at_5_max": 0.451595, "nauc_recall_at_5_std": 0.06588, "nauc_recall_at_5_diff1": 0.592213, "nauc_recall_at_10_max": 0.475833, "nauc_recall_at_10_std": 0.119504, "nauc_recall_at_10_diff1": 0.620692, "nauc_recall_at_20_max": 0.476833, "nauc_recall_at_20_std": 0.190015, "nauc_recall_at_20_diff1": 0.591529, "nauc_recall_at_100_max": 0.495259, "nauc_recall_at_100_std": 0.358893, "nauc_recall_at_100_diff1": 0.548373, "nauc_recall_at_1000_max": 0.622782, "nauc_recall_at_1000_std": 0.898565, "nauc_recall_at_1000_diff1": 0.636576, "nauc_precision_at_1_max": 0.454386, "nauc_precision_at_1_std": -0.039274, "nauc_precision_at_1_diff1": 0.651395, "nauc_precision_at_3_max": 0.435222, "nauc_precision_at_3_std": 0.022206, "nauc_precision_at_3_diff1": 0.596949, "nauc_precision_at_5_max": 0.451595, "nauc_precision_at_5_std": 0.06588, "nauc_precision_at_5_diff1": 0.592213, "nauc_precision_at_10_max": 0.475833, "nauc_precision_at_10_std": 0.119504, "nauc_precision_at_10_diff1": 0.620692, "nauc_precision_at_20_max": 0.476833, "nauc_precision_at_20_std": 0.190015, "nauc_precision_at_20_diff1": 0.591529, "nauc_precision_at_100_max": 0.495259, "nauc_precision_at_100_std": 0.358893, "nauc_precision_at_100_diff1": 0.548373, "nauc_precision_at_1000_max": 0.622782, "nauc_precision_at_1000_std": 0.898565, "nauc_precision_at_1000_diff1": 0.636576, "nauc_mrr_at_1_max": 0.454386, "nauc_mrr_at_1_std": -0.039274, "nauc_mrr_at_1_diff1": 0.651395, "nauc_mrr_at_3_max": 0.439205, "nauc_mrr_at_3_std": -0.009306, "nauc_mrr_at_3_diff1": 0.624352, "nauc_mrr_at_5_max": 0.442441, "nauc_mrr_at_5_std": 0.000586, "nauc_mrr_at_5_diff1": 0.622811, "nauc_mrr_at_10_max": 0.448083, "nauc_mrr_at_10_std": 0.008808, "nauc_mrr_at_10_diff1": 0.627215, "nauc_mrr_at_20_max": 0.44701, "nauc_mrr_at_20_std": 0.011688, "nauc_mrr_at_20_diff1": 0.6251, "nauc_mrr_at_100_max": 0.447019, "nauc_mrr_at_100_std": 0.012824, "nauc_mrr_at_100_diff1": 0.625064, "nauc_mrr_at_1000_max": 0.446926, "nauc_mrr_at_1000_std": 0.012784, "nauc_mrr_at_1000_diff1": 0.625156, "main_score": 0.45209, "hf_subset": "ko", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 11572.161805391312, "kg_co2_emissions": null } ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "task_name": "PublicHealthQA", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.68831, "ndcg_at_3": 0.78154, "ndcg_at_5": 0.80277, "ndcg_at_10": 0.81925, "ndcg_at_20": 0.82266, "ndcg_at_100": 0.83027, "ndcg_at_1000": 0.83027, "map_at_1": 0.68831, "map_at_3": 0.75974, "map_at_5": 0.77143, "map_at_10": 0.77803, "map_at_20": 0.77902, "map_at_100": 0.7802, "map_at_1000": 0.7802, "recall_at_1": 0.68831, "recall_at_3": 0.84416, "recall_at_5": 0.8961, "recall_at_10": 0.94805, "recall_at_20": 0.96104, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.68831, "precision_at_3": 0.28139, "precision_at_5": 0.17922, "precision_at_10": 0.09481, "precision_at_20": 0.04805, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.688312, "mrr_at_3": 0.75974, "mrr_at_5": 0.771429, "mrr_at_10": 0.778025, "mrr_at_20": 0.779024, "mrr_at_100": 0.780204, "mrr_at_1000": 0.780204, "nauc_ndcg_at_1_max": 0.308626, "nauc_ndcg_at_1_std": -0.228579, "nauc_ndcg_at_1_diff1": 0.655737, "nauc_ndcg_at_3_max": 0.318182, "nauc_ndcg_at_3_std": -0.190897, "nauc_ndcg_at_3_diff1": 0.625745, "nauc_ndcg_at_5_max": 0.298169, "nauc_ndcg_at_5_std": -0.22, "nauc_ndcg_at_5_diff1": 0.611828, "nauc_ndcg_at_10_max": 0.283477, "nauc_ndcg_at_10_std": -0.216946, "nauc_ndcg_at_10_diff1": 0.595781, "nauc_ndcg_at_20_max": 0.281869, "nauc_ndcg_at_20_std": -0.19649, "nauc_ndcg_at_20_diff1": 0.593043, "nauc_ndcg_at_100_max": 0.306941, "nauc_ndcg_at_100_std": -0.206296, "nauc_ndcg_at_100_diff1": 0.624753, "nauc_ndcg_at_1000_max": 0.306941, "nauc_ndcg_at_1000_std": -0.206296, "nauc_ndcg_at_1000_diff1": 0.624753, "nauc_map_at_1_max": 0.308626, "nauc_map_at_1_std": -0.228579, "nauc_map_at_1_diff1": 0.655737, "nauc_map_at_3_max": 0.321412, "nauc_map_at_3_std": -0.19738, "nauc_map_at_3_diff1": 0.637696, "nauc_map_at_5_max": 0.310439, "nauc_map_at_5_std": -0.215003, "nauc_map_at_5_diff1": 0.631938, "nauc_map_at_10_max": 0.306275, "nauc_map_at_10_std": -0.214431, "nauc_map_at_10_diff1": 0.627561, "nauc_map_at_20_max": 0.306011, "nauc_map_at_20_std": -0.209564, "nauc_map_at_20_diff1": 0.627074, "nauc_map_at_100_max": 0.310676, "nauc_map_at_100_std": -0.209687, "nauc_map_at_100_diff1": 0.632279, "nauc_map_at_1000_max": 0.310676, "nauc_map_at_1000_std": -0.209687, "nauc_map_at_1000_diff1": 0.632279, "nauc_recall_at_1_max": 0.308626, "nauc_recall_at_1_std": -0.228579, "nauc_recall_at_1_diff1": 0.655737, "nauc_recall_at_3_max": 0.301279, "nauc_recall_at_3_std": -0.167045, "nauc_recall_at_3_diff1": 0.574785, "nauc_recall_at_5_max": 0.221933, "nauc_recall_at_5_std": -0.252462, "nauc_recall_at_5_diff1": 0.491378, "nauc_recall_at_10_max": -0.003678, "nauc_recall_at_10_std": -0.242704, "nauc_recall_at_10_diff1": 0.208922, "nauc_recall_at_20_max": -0.124209, "nauc_recall_at_20_std": 0.064517, "nauc_recall_at_20_diff1": 0.037695, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.308626, "nauc_precision_at_1_std": -0.228579, "nauc_precision_at_1_diff1": 0.655737, "nauc_precision_at_3_max": 0.301279, "nauc_precision_at_3_std": -0.167045, "nauc_precision_at_3_diff1": 0.574785, "nauc_precision_at_5_max": 0.221933, "nauc_precision_at_5_std": -0.252462, "nauc_precision_at_5_diff1": 0.491378, "nauc_precision_at_10_max": -0.003678, "nauc_precision_at_10_std": -0.242704, "nauc_precision_at_10_diff1": 0.208922, "nauc_precision_at_20_max": -0.124209, "nauc_precision_at_20_std": 0.064517, "nauc_precision_at_20_diff1": 0.037695, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.308626, "nauc_mrr_at_1_std": -0.228579, "nauc_mrr_at_1_diff1": 0.655737, "nauc_mrr_at_3_max": 0.321412, "nauc_mrr_at_3_std": -0.19738, "nauc_mrr_at_3_diff1": 0.637696, "nauc_mrr_at_5_max": 0.310439, "nauc_mrr_at_5_std": -0.215003, "nauc_mrr_at_5_diff1": 0.631938, "nauc_mrr_at_10_max": 0.306275, "nauc_mrr_at_10_std": -0.214431, "nauc_mrr_at_10_diff1": 0.627561, "nauc_mrr_at_20_max": 0.306011, "nauc_mrr_at_20_std": -0.209564, "nauc_mrr_at_20_diff1": 0.627074, "nauc_mrr_at_100_max": 0.310676, "nauc_mrr_at_100_std": -0.209687, "nauc_mrr_at_100_diff1": 0.632279, "nauc_mrr_at_1000_max": 0.310676, "nauc_mrr_at_1000_std": -0.209687, "nauc_mrr_at_1000_diff1": 0.632279, "main_score": 0.81925, "hf_subset": "korean", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 1.9287936687469482, "kg_co2_emissions": null } ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "task_name": "XPQARetrieval", "mteb_version": "1.21.6", "scores": { "test": [ { "ndcg_at_1": 0.29969, "ndcg_at_3": 0.32585, "ndcg_at_5": 0.34837, "ndcg_at_10": 0.38158, "ndcg_at_20": 0.4036, "ndcg_at_100": 0.45425, "ndcg_at_1000": 0.47954, "map_at_1": 0.20907, "map_at_3": 0.2944, "map_at_5": 0.31246, "map_at_10": 0.32934, "map_at_20": 0.33631, "map_at_100": 0.34415, "map_at_1000": 0.3453, "recall_at_1": 0.20907, "recall_at_3": 0.34778, "recall_at_5": 0.40398, "recall_at_10": 0.49478, "recall_at_20": 0.57146, "recall_at_100": 0.81863, "recall_at_1000": 1.0, "precision_at_1": 0.29969, "precision_at_3": 0.18145, "precision_at_5": 0.12813, "precision_at_10": 0.07859, "precision_at_20": 0.04557, "precision_at_100": 0.01306, "precision_at_1000": 0.00156, "mrr_at_1": 0.298165, "mrr_at_3": 0.349388, "mrr_at_5": 0.361315, "mrr_at_10": 0.372384, "mrr_at_20": 0.377904, "mrr_at_100": 0.383454, "mrr_at_1000": 0.384251, "nauc_ndcg_at_1_max": 0.185255, "nauc_ndcg_at_1_std": -0.293629, "nauc_ndcg_at_1_diff1": 0.52904, "nauc_ndcg_at_3_max": 0.182618, "nauc_ndcg_at_3_std": -0.29233, "nauc_ndcg_at_3_diff1": 0.478231, "nauc_ndcg_at_5_max": 0.173084, "nauc_ndcg_at_5_std": -0.308008, "nauc_ndcg_at_5_diff1": 0.474155, "nauc_ndcg_at_10_max": 0.180935, "nauc_ndcg_at_10_std": -0.30623, "nauc_ndcg_at_10_diff1": 0.482508, "nauc_ndcg_at_20_max": 0.189305, "nauc_ndcg_at_20_std": -0.295186, "nauc_ndcg_at_20_diff1": 0.476185, "nauc_ndcg_at_100_max": 0.212514, "nauc_ndcg_at_100_std": -0.247023, "nauc_ndcg_at_100_diff1": 0.475654, "nauc_ndcg_at_1000_max": 0.197363, "nauc_ndcg_at_1000_std": -0.276491, "nauc_ndcg_at_1000_diff1": 0.480516, "nauc_map_at_1_max": 0.142274, "nauc_map_at_1_std": -0.26814, "nauc_map_at_1_diff1": 0.542024, "nauc_map_at_3_max": 0.184188, "nauc_map_at_3_std": -0.286823, "nauc_map_at_3_diff1": 0.479828, "nauc_map_at_5_max": 0.179512, "nauc_map_at_5_std": -0.301422, "nauc_map_at_5_diff1": 0.477793, "nauc_map_at_10_max": 0.181279, "nauc_map_at_10_std": -0.303136, "nauc_map_at_10_diff1": 0.480646, "nauc_map_at_20_max": 0.183516, "nauc_map_at_20_std": -0.301271, "nauc_map_at_20_diff1": 0.479709, "nauc_map_at_100_max": 0.186373, "nauc_map_at_100_std": -0.294181, "nauc_map_at_100_diff1": 0.479856, "nauc_map_at_1000_max": 0.185876, "nauc_map_at_1000_std": -0.295295, "nauc_map_at_1000_diff1": 0.479925, "nauc_recall_at_1_max": 0.142274, "nauc_recall_at_1_std": -0.26814, "nauc_recall_at_1_diff1": 0.542024, "nauc_recall_at_3_max": 0.176409, "nauc_recall_at_3_std": -0.273774, "nauc_recall_at_3_diff1": 0.437992, "nauc_recall_at_5_max": 0.151198, "nauc_recall_at_5_std": -0.313378, "nauc_recall_at_5_diff1": 0.427717, "nauc_recall_at_10_max": 0.17051, "nauc_recall_at_10_std": -0.301517, "nauc_recall_at_10_diff1": 0.444929, "nauc_recall_at_20_max": 0.200838, "nauc_recall_at_20_std": -0.259205, "nauc_recall_at_20_diff1": 0.416033, "nauc_recall_at_100_max": 0.391323, "nauc_recall_at_100_std": 0.138514, "nauc_recall_at_100_diff1": 0.365729, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.185255, "nauc_precision_at_1_std": -0.293629, "nauc_precision_at_1_diff1": 0.52904, "nauc_precision_at_3_max": 0.19322, "nauc_precision_at_3_std": -0.265535, "nauc_precision_at_3_diff1": 0.338089, "nauc_precision_at_5_max": 0.168395, "nauc_precision_at_5_std": -0.280409, "nauc_precision_at_5_diff1": 0.312183, "nauc_precision_at_10_max": 0.16462, "nauc_precision_at_10_std": -0.256445, "nauc_precision_at_10_diff1": 0.294712, "nauc_precision_at_20_max": 0.167231, "nauc_precision_at_20_std": -0.211221, "nauc_precision_at_20_diff1": 0.252206, "nauc_precision_at_100_max": 0.212522, "nauc_precision_at_100_std": 0.06126, "nauc_precision_at_100_diff1": 0.116848, "nauc_precision_at_1000_max": 0.133829, "nauc_precision_at_1000_std": 0.041224, "nauc_precision_at_1000_diff1": 0.011564, "nauc_mrr_at_1_max": 0.185517, "nauc_mrr_at_1_std": -0.298726, "nauc_mrr_at_1_diff1": 0.534802, "nauc_mrr_at_3_max": 0.184778, "nauc_mrr_at_3_std": -0.294455, "nauc_mrr_at_3_diff1": 0.504579, "nauc_mrr_at_5_max": 0.17654, "nauc_mrr_at_5_std": -0.302472, "nauc_mrr_at_5_diff1": 0.501237, "nauc_mrr_at_10_max": 0.181937, "nauc_mrr_at_10_std": -0.299648, "nauc_mrr_at_10_diff1": 0.505392, "nauc_mrr_at_20_max": 0.185438, "nauc_mrr_at_20_std": -0.293949, "nauc_mrr_at_20_diff1": 0.503537, "nauc_mrr_at_100_max": 0.18721, "nauc_mrr_at_100_std": -0.289807, "nauc_mrr_at_100_diff1": 0.503231, "nauc_mrr_at_1000_max": 0.186581, "nauc_mrr_at_1000_std": -0.290973, "nauc_mrr_at_1000_diff1": 0.503404, "main_score": 0.38158, "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ] }, { "ndcg_at_1": 0.28135, "ndcg_at_3": 0.30664, "ndcg_at_5": 0.33055, "ndcg_at_10": 0.36466, "ndcg_at_20": 0.3875, "ndcg_at_100": 0.44398, "ndcg_at_1000": 0.47303, "map_at_1": 0.16195, "map_at_3": 0.25435, "map_at_5": 0.28328, "map_at_10": 0.30242, "map_at_20": 0.31044, "map_at_100": 0.32028, "map_at_1000": 0.32203, "recall_at_1": 0.16195, "recall_at_3": 0.31007, "recall_at_5": 0.38665, "recall_at_10": 0.47031, "recall_at_20": 0.54559, "recall_at_100": 0.80691, "recall_at_1000": 0.99949, "precision_at_1": 0.28135, "precision_at_3": 0.20133, "precision_at_5": 0.15566, "precision_at_10": 0.09694, "precision_at_20": 0.05604, "precision_at_100": 0.01609, "precision_at_1000": 0.00195, "mrr_at_1": 0.281346, "mrr_at_3": 0.345821, "mrr_at_5": 0.360805, "mrr_at_10": 0.371258, "mrr_at_20": 0.376241, "mrr_at_100": 0.382019, "mrr_at_1000": 0.382721, "nauc_ndcg_at_1_max": 0.094105, "nauc_ndcg_at_1_std": -0.316474, "nauc_ndcg_at_1_diff1": 0.516674, "nauc_ndcg_at_3_max": 0.045168, "nauc_ndcg_at_3_std": -0.343558, "nauc_ndcg_at_3_diff1": 0.42946, "nauc_ndcg_at_5_max": 0.033593, "nauc_ndcg_at_5_std": -0.348724, "nauc_ndcg_at_5_diff1": 0.405359, "nauc_ndcg_at_10_max": 0.034196, "nauc_ndcg_at_10_std": -0.347105, "nauc_ndcg_at_10_diff1": 0.402682, "nauc_ndcg_at_20_max": 0.038368, "nauc_ndcg_at_20_std": -0.344478, "nauc_ndcg_at_20_diff1": 0.401801, "nauc_ndcg_at_100_max": 0.048155, "nauc_ndcg_at_100_std": -0.320897, "nauc_ndcg_at_100_diff1": 0.399984, "nauc_ndcg_at_1000_max": 0.054938, "nauc_ndcg_at_1000_std": -0.32543, "nauc_ndcg_at_1000_diff1": 0.41171, "nauc_map_at_1_max": 0.068529, "nauc_map_at_1_std": -0.268016, "nauc_map_at_1_diff1": 0.534787, "nauc_map_at_3_max": 0.045564, "nauc_map_at_3_std": -0.32845, "nauc_map_at_3_diff1": 0.453435, "nauc_map_at_5_max": 0.041195, "nauc_map_at_5_std": -0.339515, "nauc_map_at_5_diff1": 0.427519, "nauc_map_at_10_max": 0.043165, "nauc_map_at_10_std": -0.341514, "nauc_map_at_10_diff1": 0.426386, "nauc_map_at_20_max": 0.043427, "nauc_map_at_20_std": -0.342731, "nauc_map_at_20_diff1": 0.426351, "nauc_map_at_100_max": 0.044002, "nauc_map_at_100_std": -0.340526, "nauc_map_at_100_diff1": 0.425798, "nauc_map_at_1000_max": 0.045262, "nauc_map_at_1000_std": -0.339585, "nauc_map_at_1000_diff1": 0.426386, "nauc_recall_at_1_max": 0.068529, "nauc_recall_at_1_std": -0.268016, "nauc_recall_at_1_diff1": 0.534787, "nauc_recall_at_3_max": 0.024468, "nauc_recall_at_3_std": -0.319048, "nauc_recall_at_3_diff1": 0.381687, "nauc_recall_at_5_max": 0.009382, "nauc_recall_at_5_std": -0.331003, "nauc_recall_at_5_diff1": 0.322401, "nauc_recall_at_10_max": 0.001473, "nauc_recall_at_10_std": -0.327362, "nauc_recall_at_10_diff1": 0.312948, "nauc_recall_at_20_max": 0.012057, "nauc_recall_at_20_std": -0.315145, "nauc_recall_at_20_diff1": 0.300354, "nauc_recall_at_100_max": 0.044714, "nauc_recall_at_100_std": -0.146942, "nauc_recall_at_100_diff1": 0.240661, "nauc_recall_at_1000_max": 1.0, "nauc_recall_at_1000_std": 1.0, "nauc_recall_at_1000_diff1": 1.0, "nauc_precision_at_1_max": 0.094105, "nauc_precision_at_1_std": -0.316474, "nauc_precision_at_1_diff1": 0.516674, "nauc_precision_at_3_max": 0.028083, "nauc_precision_at_3_std": -0.354435, "nauc_precision_at_3_diff1": 0.283145, "nauc_precision_at_5_max": 0.017195, "nauc_precision_at_5_std": -0.331981, "nauc_precision_at_5_diff1": 0.203304, "nauc_precision_at_10_max": 0.027622, "nauc_precision_at_10_std": -0.292894, "nauc_precision_at_10_diff1": 0.18355, "nauc_precision_at_20_max": 0.037703, "nauc_precision_at_20_std": -0.262081, "nauc_precision_at_20_diff1": 0.165698, "nauc_precision_at_100_max": 0.085114, "nauc_precision_at_100_std": -0.092042, "nauc_precision_at_100_diff1": 0.054652, "nauc_precision_at_1000_max": 0.118011, "nauc_precision_at_1000_std": -0.000622, "nauc_precision_at_1000_diff1": -0.004351, "nauc_mrr_at_1_max": 0.094105, "nauc_mrr_at_1_std": -0.316474, "nauc_mrr_at_1_diff1": 0.516674, "nauc_mrr_at_3_max": 0.056857, "nauc_mrr_at_3_std": -0.338862, "nauc_mrr_at_3_diff1": 0.448368, "nauc_mrr_at_5_max": 0.053247, "nauc_mrr_at_5_std": -0.342588, "nauc_mrr_at_5_diff1": 0.438226, "nauc_mrr_at_10_max": 0.054761, "nauc_mrr_at_10_std": -0.338175, "nauc_mrr_at_10_diff1": 0.436671, "nauc_mrr_at_20_max": 0.056653, "nauc_mrr_at_20_std": -0.335522, "nauc_mrr_at_20_diff1": 0.436089, "nauc_mrr_at_100_max": 0.057628, "nauc_mrr_at_100_std": -0.33298, "nauc_mrr_at_100_diff1": 0.437037, "nauc_mrr_at_1000_max": 0.05751, "nauc_mrr_at_1000_std": -0.333511, "nauc_mrr_at_1000_diff1": 0.437531, "main_score": 0.36466, "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ] }, { "ndcg_at_1": 0.26873, "ndcg_at_3": 0.29532, "ndcg_at_5": 0.31604, "ndcg_at_10": 0.34995, "ndcg_at_20": 0.37273, "ndcg_at_100": 0.42723, "ndcg_at_1000": 0.45554, "map_at_1": 0.18754, "map_at_3": 0.26398, "map_at_5": 0.28046, "map_at_10": 0.2974, "map_at_20": 0.30502, "map_at_100": 0.31386, "map_at_1000": 0.31512, "recall_at_1": 0.18754, "recall_at_3": 0.31788, "recall_at_5": 0.36674, "recall_at_10": 0.45766, "recall_at_20": 0.53428, "recall_at_100": 0.79612, "recall_at_1000": 1.0, "precision_at_1": 0.26873, "precision_at_3": 0.16667, "precision_at_5": 0.12085, "precision_at_10": 0.07704, "precision_at_20": 0.04577, "precision_at_100": 0.01358, "precision_at_1000": 0.00167, "mrr_at_1": 0.267101, "mrr_at_3": 0.316232, "mrr_at_5": 0.329262, "mrr_at_10": 0.341046, "mrr_at_20": 0.346495, "mrr_at_100": 0.352335, "mrr_at_1000": 0.353085, "nauc_ndcg_at_1_max": 0.185851, "nauc_ndcg_at_1_std": -0.203731, "nauc_ndcg_at_1_diff1": 0.462692, "nauc_ndcg_at_3_max": 0.147801, "nauc_ndcg_at_3_std": -0.231289, "nauc_ndcg_at_3_diff1": 0.404865, "nauc_ndcg_at_5_max": 0.1488, "nauc_ndcg_at_5_std": -0.237314, "nauc_ndcg_at_5_diff1": 0.393875, "nauc_ndcg_at_10_max": 0.152161, "nauc_ndcg_at_10_std": -0.241002, "nauc_ndcg_at_10_diff1": 0.393157, "nauc_ndcg_at_20_max": 0.15502, "nauc_ndcg_at_20_std": -0.231115, "nauc_ndcg_at_20_diff1": 0.394928, "nauc_ndcg_at_100_max": 0.173095, "nauc_ndcg_at_100_std": -0.18289, "nauc_ndcg_at_100_diff1": 0.396386, "nauc_ndcg_at_1000_max": 0.170924, "nauc_ndcg_at_1000_std": -0.203439, "nauc_ndcg_at_1000_diff1": 0.405588, "nauc_map_at_1_max": 0.14769, "nauc_map_at_1_std": -0.178742, "nauc_map_at_1_diff1": 0.488624, "nauc_map_at_3_max": 0.153568, "nauc_map_at_3_std": -0.222758, "nauc_map_at_3_diff1": 0.408914, "nauc_map_at_5_max": 0.152616, "nauc_map_at_5_std": -0.230648, "nauc_map_at_5_diff1": 0.398317, "nauc_map_at_10_max": 0.155737, "nauc_map_at_10_std": -0.233697, "nauc_map_at_10_diff1": 0.399552, "nauc_map_at_20_max": 0.155004, "nauc_map_at_20_std": -0.232309, "nauc_map_at_20_diff1": 0.399328, "nauc_map_at_100_max": 0.1576, "nauc_map_at_100_std": -0.224424, "nauc_map_at_100_diff1": 0.400345, "nauc_map_at_1000_max": 0.157888, "nauc_map_at_1000_std": -0.224825, "nauc_map_at_1000_diff1": 0.400781, "nauc_recall_at_1_max": 0.14769, "nauc_recall_at_1_std": -0.178742, "nauc_recall_at_1_diff1": 0.488624, "nauc_recall_at_3_max": 0.120369, "nauc_recall_at_3_std": -0.233571, "nauc_recall_at_3_diff1": 0.364854, "nauc_recall_at_5_max": 0.124395, "nauc_recall_at_5_std": -0.247811, "nauc_recall_at_5_diff1": 0.345413, "nauc_recall_at_10_max": 0.128996, "nauc_recall_at_10_std": -0.253887, "nauc_recall_at_10_diff1": 0.331844, "nauc_recall_at_20_max": 0.139756, "nauc_recall_at_20_std": -0.222521, "nauc_recall_at_20_diff1": 0.331895, "nauc_recall_at_100_max": 0.211622, "nauc_recall_at_100_std": 0.085916, "nauc_recall_at_100_diff1": 0.275684, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.185851, "nauc_precision_at_1_std": -0.203731, "nauc_precision_at_1_diff1": 0.462692, "nauc_precision_at_3_max": 0.148277, "nauc_precision_at_3_std": -0.228099, "nauc_precision_at_3_diff1": 0.277866, "nauc_precision_at_5_max": 0.138236, "nauc_precision_at_5_std": -0.233192, "nauc_precision_at_5_diff1": 0.237999, "nauc_precision_at_10_max": 0.131532, "nauc_precision_at_10_std": -0.211085, "nauc_precision_at_10_diff1": 0.219161, "nauc_precision_at_20_max": 0.128112, "nauc_precision_at_20_std": -0.157827, "nauc_precision_at_20_diff1": 0.200437, "nauc_precision_at_100_max": 0.202322, "nauc_precision_at_100_std": 0.110302, "nauc_precision_at_100_diff1": 0.107249, "nauc_precision_at_1000_max": 0.182725, "nauc_precision_at_1000_std": 0.115469, "nauc_precision_at_1000_diff1": 0.049456, "nauc_mrr_at_1_max": 0.18997, "nauc_mrr_at_1_std": -0.199972, "nauc_mrr_at_1_diff1": 0.468971, "nauc_mrr_at_3_max": 0.162908, "nauc_mrr_at_3_std": -0.222321, "nauc_mrr_at_3_diff1": 0.434997, "nauc_mrr_at_5_max": 0.166916, "nauc_mrr_at_5_std": -0.22507, "nauc_mrr_at_5_diff1": 0.432051, "nauc_mrr_at_10_max": 0.167261, "nauc_mrr_at_10_std": -0.223063, "nauc_mrr_at_10_diff1": 0.428935, "nauc_mrr_at_20_max": 0.169802, "nauc_mrr_at_20_std": -0.217834, "nauc_mrr_at_20_diff1": 0.43113, "nauc_mrr_at_100_max": 0.171108, "nauc_mrr_at_100_std": -0.214006, "nauc_mrr_at_100_diff1": 0.431074, "nauc_mrr_at_1000_max": 0.171026, "nauc_mrr_at_1000_std": -0.214628, "nauc_mrr_at_1000_diff1": 0.431278, "main_score": 0.34995, "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ] } ] }, "evaluation_time": 14.467206716537476, "kg_co2_emissions": null } ================================================ FILE: eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/model_meta.json ================================================ {"name": "nlpai-lab/KURE-v1", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": false, "public_training_code": null, "framework": ["PyTorch", "Sentence Transformers"], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "adapted_from": null, "superseded_by": null, "loader": null} ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 19.922383546829224, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.84339, "map_at_1": 0.7193, "map_at_10": 0.80894, "map_at_100": 0.81076, "map_at_1000": 0.81076, "map_at_20": 0.80997, "map_at_3": 0.79678, "map_at_5": 0.80687, "mrr_at_1": 0.7192982456140351, "mrr_at_10": 0.8089424951267057, "mrr_at_100": 0.8107636500019111, "mrr_at_1000": 0.8107636500019111, "mrr_at_20": 0.8099658869395712, "mrr_at_3": 0.7967836257309943, "mrr_at_5": 0.8068713450292399, "nauc_map_at_1000_diff1": 0.6320942680079616, "nauc_map_at_1000_max": -0.05701885689693603, "nauc_map_at_1000_std": -0.6853683208954745, "nauc_map_at_100_diff1": 0.6320942680079616, "nauc_map_at_100_max": -0.05701885689693603, "nauc_map_at_100_std": -0.6853683208954745, "nauc_map_at_10_diff1": 0.633309415089511, "nauc_map_at_10_max": -0.05429473185056241, "nauc_map_at_10_std": -0.6918629844580086, "nauc_map_at_1_diff1": 0.6674090056378712, "nauc_map_at_1_max": -0.1251284973664897, "nauc_map_at_1_std": -0.6485224985983171, "nauc_map_at_20_diff1": 0.6326370926444672, "nauc_map_at_20_max": -0.05507130571170382, "nauc_map_at_20_std": -0.6854871094189069, "nauc_map_at_3_diff1": 0.609689975413944, "nauc_map_at_3_max": -0.02805045628386939, "nauc_map_at_3_std": -0.6626074040597936, "nauc_map_at_5_diff1": 0.6296484757032934, "nauc_map_at_5_max": -0.05999223752680984, "nauc_map_at_5_std": -0.6860149178877104, "nauc_mrr_at_1000_diff1": 0.6320942680079616, "nauc_mrr_at_1000_max": -0.05701885689693603, "nauc_mrr_at_1000_std": -0.6853683208954745, "nauc_mrr_at_100_diff1": 0.6320942680079616, "nauc_mrr_at_100_max": -0.05701885689693603, "nauc_mrr_at_100_std": -0.6853683208954745, "nauc_mrr_at_10_diff1": 0.633309415089511, "nauc_mrr_at_10_max": -0.05429473185056241, "nauc_mrr_at_10_std": -0.6918629844580086, "nauc_mrr_at_1_diff1": 0.6674090056378712, "nauc_mrr_at_1_max": -0.1251284973664897, "nauc_mrr_at_1_std": -0.6485224985983171, "nauc_mrr_at_20_diff1": 0.6326370926444672, "nauc_mrr_at_20_max": -0.05507130571170382, "nauc_mrr_at_20_std": -0.6854871094189069, "nauc_mrr_at_3_diff1": 0.609689975413944, "nauc_mrr_at_3_max": -0.02805045628386939, "nauc_mrr_at_3_std": -0.6626074040597936, "nauc_mrr_at_5_diff1": 0.6296484757032934, "nauc_mrr_at_5_max": -0.05999223752680984, "nauc_mrr_at_5_std": -0.6860149178877104, "nauc_ndcg_at_1000_diff1": 0.6334301799598651, "nauc_ndcg_at_1000_max": -0.04468898576313594, "nauc_ndcg_at_1000_std": -0.6952142682019876, "nauc_ndcg_at_100_diff1": 0.6334301799598651, "nauc_ndcg_at_100_max": -0.04468898576313594, "nauc_ndcg_at_100_std": -0.6952142682019876, "nauc_ndcg_at_10_diff1": 0.6436986997944144, "nauc_ndcg_at_10_max": -0.02291530543046443, "nauc_ndcg_at_10_std": -0.7310258910452394, "nauc_ndcg_at_1_diff1": 0.6674090056378712, "nauc_ndcg_at_1_max": -0.1251284973664897, "nauc_ndcg_at_1_std": -0.6485224985983171, "nauc_ndcg_at_20_diff1": 0.6399616809257035, "nauc_ndcg_at_20_max": -0.02426566036832786, "nauc_ndcg_at_20_std": -0.699937879968727, "nauc_ndcg_at_3_diff1": 0.5919667530188153, "nauc_ndcg_at_3_max": 0.020530667149423275, "nauc_ndcg_at_3_std": -0.6630627158068408, "nauc_ndcg_at_5_diff1": 0.6323477601323944, "nauc_ndcg_at_5_max": -0.04307026006368585, "nauc_ndcg_at_5_std": -0.7120648723045967, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.7836190936419349, "nauc_precision_at_10_max": 0.2755948174983461, "nauc_precision_at_10_std": -1.1385948830272972, "nauc_precision_at_1_diff1": 0.6674090056378712, "nauc_precision_at_1_max": -0.1251284973664897, "nauc_precision_at_1_std": -0.6485224985983171, "nauc_precision_at_20_diff1": 0.7867387395585554, "nauc_precision_at_20_max": 0.4156498548585641, "nauc_precision_at_20_std": -0.8296243289837735, "nauc_precision_at_3_diff1": 0.5124419476805667, "nauc_precision_at_3_max": 0.2518702236748006, "nauc_precision_at_3_std": -0.6623529855930005, "nauc_precision_at_5_diff1": 0.6723384771911648, "nauc_precision_at_5_max": 0.0564740646866425, "nauc_precision_at_5_std": -0.9091407205206298, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.7836190936419378, "nauc_recall_at_10_max": 0.2755948174983493, "nauc_recall_at_10_std": -1.13859488302729, "nauc_recall_at_1_diff1": 0.6674090056378712, "nauc_recall_at_1_max": -0.1251284973664897, "nauc_recall_at_1_std": -0.6485224985983171, "nauc_recall_at_20_diff1": 0.7867387395585571, "nauc_recall_at_20_max": 0.41564985485856165, "nauc_recall_at_20_std": -0.8296243289837761, "nauc_recall_at_3_diff1": 0.5124419476805673, "nauc_recall_at_3_max": 0.2518702236748035, "nauc_recall_at_3_std": -0.6623529855929987, "nauc_recall_at_5_diff1": 0.6723384771911676, "nauc_recall_at_5_max": 0.05647406468664335, "nauc_recall_at_5_std": -0.9091407205206277, "ndcg_at_1": 0.7193, "ndcg_at_10": 0.84339, "ndcg_at_100": 0.85391, "ndcg_at_1000": 0.85391, "ndcg_at_20": 0.84758, "ndcg_at_3": 0.81986, "ndcg_at_5": 0.83798, "precision_at_1": 0.7193, "precision_at_10": 0.09474, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04825, "precision_at_3": 0.29532, "precision_at_5": 0.18596, "recall_at_1": 0.7193, "recall_at_10": 0.94737, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.96491, "recall_at_3": 0.88596, "recall_at_5": 0.92982 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 26.43672013282776, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.94251, "map_at_1": 0.89333, "map_at_10": 0.92898, "map_at_100": 0.9295, "map_at_1000": 0.92952, "map_at_20": 0.92937, "map_at_3": 0.92426, "map_at_5": 0.92793, "mrr_at_1": 0.8933333333333333, "mrr_at_10": 0.9289828042328042, "mrr_at_100": 0.9294972174704905, "mrr_at_1000": 0.9295225970670912, "mrr_at_20": 0.9293730158730159, "mrr_at_3": 0.9242592592592593, "mrr_at_5": 0.927925925925926, "nauc_map_at_1000_diff1": 0.9233864675994327, "nauc_map_at_1000_max": 0.8090206958688105, "nauc_map_at_1000_std": -0.10539329815363037, "nauc_map_at_100_diff1": 0.9233917827008663, "nauc_map_at_100_max": 0.8090898721437118, "nauc_map_at_100_std": -0.10500435662013195, "nauc_map_at_10_diff1": 0.9232644909975076, "nauc_map_at_10_max": 0.8095876086490258, "nauc_map_at_10_std": -0.10574632386413099, "nauc_map_at_1_diff1": 0.9219914489683861, "nauc_map_at_1_max": 0.7792078528281168, "nauc_map_at_1_std": -0.1528791088099168, "nauc_map_at_20_diff1": 0.9232443580286058, "nauc_map_at_20_max": 0.8091726176091726, "nauc_map_at_20_std": -0.10501053945253029, "nauc_map_at_3_diff1": 0.9245096080767995, "nauc_map_at_3_max": 0.8169103988574538, "nauc_map_at_3_std": -0.09255968166715849, "nauc_map_at_5_diff1": 0.9241431361588304, "nauc_map_at_5_max": 0.811087360962672, "nauc_map_at_5_std": -0.10305440425262838, "nauc_mrr_at_1000_diff1": 0.9233864675994327, "nauc_mrr_at_1000_max": 0.8090206958688105, "nauc_mrr_at_1000_std": -0.10539329815363037, "nauc_mrr_at_100_diff1": 0.9233917827008663, "nauc_mrr_at_100_max": 0.8090898721437118, "nauc_mrr_at_100_std": -0.10500435662013195, "nauc_mrr_at_10_diff1": 0.9232644909975076, "nauc_mrr_at_10_max": 0.8095876086490258, "nauc_mrr_at_10_std": -0.10574632386413099, "nauc_mrr_at_1_diff1": 0.9219914489683861, "nauc_mrr_at_1_max": 0.7792078528281168, "nauc_mrr_at_1_std": -0.1528791088099168, "nauc_mrr_at_20_diff1": 0.9232443580286058, "nauc_mrr_at_20_max": 0.8091726176091726, "nauc_mrr_at_20_std": -0.10501053945253029, "nauc_mrr_at_3_diff1": 0.9245096080767995, "nauc_mrr_at_3_max": 0.8169103988574538, "nauc_mrr_at_3_std": -0.09255968166715849, "nauc_mrr_at_5_diff1": 0.9241431361588304, "nauc_mrr_at_5_max": 0.811087360962672, "nauc_mrr_at_5_std": -0.10305440425262838, "nauc_ndcg_at_1000_diff1": 0.9234321310800012, "nauc_ndcg_at_1000_max": 0.8134769479311467, "nauc_ndcg_at_1000_std": -0.09530413213685224, "nauc_ndcg_at_100_diff1": 0.9235614555171696, "nauc_ndcg_at_100_max": 0.8155106567552204, "nauc_ndcg_at_100_std": -0.08371598384355847, "nauc_ndcg_at_10_diff1": 0.9225885992841035, "nauc_ndcg_at_10_max": 0.8180141073001267, "nauc_ndcg_at_10_std": -0.0908177224862573, "nauc_ndcg_at_1_diff1": 0.9219914489683861, "nauc_ndcg_at_1_max": 0.7792078528281168, "nauc_ndcg_at_1_std": -0.1528791088099168, "nauc_ndcg_at_20_diff1": 0.9225685571953032, "nauc_ndcg_at_20_max": 0.8162028689364406, "nauc_ndcg_at_20_std": -0.08806888636157015, "nauc_ndcg_at_3_diff1": 0.9256420489413464, "nauc_ndcg_at_3_max": 0.8331262056528227, "nauc_ndcg_at_3_std": -0.06283685906042585, "nauc_ndcg_at_5_diff1": 0.9251077899816641, "nauc_ndcg_at_5_max": 0.8222650665358133, "nauc_ndcg_at_5_std": -0.08101541630049999, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9346405228758011, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.9673202614378776, "nauc_precision_at_10_diff1": 0.91067538126361, "nauc_precision_at_10_max": 0.9038281979458412, "nauc_precision_at_10_std": 0.05981948334889332, "nauc_precision_at_1_diff1": 0.9219914489683861, "nauc_precision_at_1_max": 0.7792078528281168, "nauc_precision_at_1_std": -0.1528791088099168, "nauc_precision_at_20_diff1": 0.9052287581699321, "nauc_precision_at_20_max": 0.9096638655461987, "nauc_precision_at_20_std": 0.17969187675070394, "nauc_precision_at_3_diff1": 0.9315491962550828, "nauc_precision_at_3_max": 0.9172912408206573, "nauc_precision_at_3_std": 0.09368612309789238, "nauc_precision_at_5_diff1": 0.9331550802138999, "nauc_precision_at_5_max": 0.909918512859686, "nauc_precision_at_5_std": 0.0995034377387245, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9346405228758294, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.9673202614379001, "nauc_recall_at_10_diff1": 0.9106753812636191, "nauc_recall_at_10_max": 0.9038281979458463, "nauc_recall_at_10_std": 0.059819483348899666, "nauc_recall_at_1_diff1": 0.9219914489683861, "nauc_recall_at_1_max": 0.7792078528281168, "nauc_recall_at_1_std": -0.1528791088099168, "nauc_recall_at_20_diff1": 0.9052287581699371, "nauc_recall_at_20_max": 0.9096638655462097, "nauc_recall_at_20_std": 0.17969187675069964, "nauc_recall_at_3_diff1": 0.9315491962550781, "nauc_recall_at_3_max": 0.9172912408206558, "nauc_recall_at_3_std": 0.09368612309788747, "nauc_recall_at_5_diff1": 0.933155080213904, "nauc_recall_at_5_max": 0.90991851285969, "nauc_recall_at_5_std": 0.0995034377387257, "ndcg_at_1": 0.89333, "ndcg_at_10": 0.94251, "ndcg_at_100": 0.94507, "ndcg_at_1000": 0.94566, "ndcg_at_20": 0.94393, "ndcg_at_3": 0.93324, "ndcg_at_5": 0.93998, "precision_at_1": 0.89333, "precision_at_10": 0.09833, "precision_at_100": 0.00996, "precision_at_1000": 0.001, "precision_at_20": 0.04944, "precision_at_3": 0.31963, "precision_at_5": 0.19511, "recall_at_1": 0.89333, "recall_at_10": 0.98333, "recall_at_100": 0.99556, "recall_at_1000": 1.0, "recall_at_20": 0.98889, "recall_at_3": 0.95889, "recall_at_5": 0.97556 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.93651, "map_at_1": 0.88222, "map_at_10": 0.92027, "map_at_100": 0.92073, "map_at_1000": 0.92075, "map_at_20": 0.92059, "map_at_3": 0.91481, "map_at_5": 0.9177, "mrr_at_1": 0.8822222222222222, "mrr_at_10": 0.9202733686067022, "mrr_at_100": 0.920731019983486, "mrr_at_1000": 0.9207500928454461, "mrr_at_20": 0.9205878863329845, "mrr_at_3": 0.914814814814815, "mrr_at_5": 0.917703703703704, "nauc_map_at_1000_diff1": 0.910378752606316, "nauc_map_at_1000_max": 0.7677636365954222, "nauc_map_at_1000_std": -0.2069823724831138, "nauc_map_at_100_diff1": 0.9103896986298576, "nauc_map_at_100_max": 0.7678203461116878, "nauc_map_at_100_std": -0.2066944275679903, "nauc_map_at_10_diff1": 0.9103310767339169, "nauc_map_at_10_max": 0.7685995087295452, "nauc_map_at_10_std": -0.20497000510700775, "nauc_map_at_1_diff1": 0.9139154970077102, "nauc_map_at_1_max": 0.7432381431626621, "nauc_map_at_1_std": -0.23120158869938737, "nauc_map_at_20_diff1": 0.9103723300246829, "nauc_map_at_20_max": 0.7677931632230002, "nauc_map_at_20_std": -0.20619686634822945, "nauc_map_at_3_diff1": 0.910729719718484, "nauc_map_at_3_max": 0.775728484998148, "nauc_map_at_3_std": -0.20419702020825395, "nauc_map_at_5_diff1": 0.9096210108253402, "nauc_map_at_5_max": 0.7711235413411344, "nauc_map_at_5_std": -0.20651983963596712, "nauc_mrr_at_1000_diff1": 0.910378752606316, "nauc_mrr_at_1000_max": 0.7677636365954222, "nauc_mrr_at_1000_std": -0.2069823724831138, "nauc_mrr_at_100_diff1": 0.9103896986298576, "nauc_mrr_at_100_max": 0.7678203461116878, "nauc_mrr_at_100_std": -0.2066944275679903, "nauc_mrr_at_10_diff1": 0.9103310767339169, "nauc_mrr_at_10_max": 0.7685995087295452, "nauc_mrr_at_10_std": -0.20497000510700775, "nauc_mrr_at_1_diff1": 0.9139154970077102, "nauc_mrr_at_1_max": 0.7432381431626621, "nauc_mrr_at_1_std": -0.23120158869938737, "nauc_mrr_at_20_diff1": 0.9103723300246829, "nauc_mrr_at_20_max": 0.7677931632230002, "nauc_mrr_at_20_std": -0.20619686634822945, "nauc_mrr_at_3_diff1": 0.910729719718484, "nauc_mrr_at_3_max": 0.775728484998148, "nauc_mrr_at_3_std": -0.20419702020825395, "nauc_mrr_at_5_diff1": 0.9096210108253402, "nauc_mrr_at_5_max": 0.7711235413411344, "nauc_mrr_at_5_std": -0.20651983963596712, "nauc_ndcg_at_1000_diff1": 0.910309840690257, "nauc_ndcg_at_1000_max": 0.7718917686412526, "nauc_ndcg_at_1000_std": -0.19711219617555498, "nauc_ndcg_at_100_diff1": 0.9106455864754731, "nauc_ndcg_at_100_max": 0.7735693178069648, "nauc_ndcg_at_100_std": -0.18860487479968238, "nauc_ndcg_at_10_diff1": 0.9105233118069156, "nauc_ndcg_at_10_max": 0.7772978942631096, "nauc_ndcg_at_10_std": -0.18079113454823553, "nauc_ndcg_at_1_diff1": 0.9139154970077102, "nauc_ndcg_at_1_max": 0.7432381431626621, "nauc_ndcg_at_1_std": -0.23120158869938737, "nauc_ndcg_at_20_diff1": 0.9106919907682975, "nauc_ndcg_at_20_max": 0.7737891995288245, "nauc_ndcg_at_20_std": -0.18472650828540493, "nauc_ndcg_at_3_diff1": 0.9102034992574379, "nauc_ndcg_at_3_max": 0.7920840420808655, "nauc_ndcg_at_3_std": -0.18397241078146917, "nauc_ndcg_at_5_diff1": 0.9080144330930631, "nauc_ndcg_at_5_max": 0.7840036131040621, "nauc_ndcg_at_5_std": -0.18590531342613936, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.9564270152505041, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 0.9564270152505041, "nauc_precision_at_10_diff1": 0.9223856209150375, "nauc_precision_at_10_max": 0.9051120448179149, "nauc_precision_at_10_std": 0.247782446311847, "nauc_precision_at_1_diff1": 0.9139154970077102, "nauc_precision_at_1_max": 0.7432381431626621, "nauc_precision_at_1_std": -0.23120158869938737, "nauc_precision_at_20_diff1": 0.9325980392156864, "nauc_precision_at_20_max": 0.8740079365079182, "nauc_precision_at_20_std": 0.37219887955182474, "nauc_precision_at_3_diff1": 0.9079387852702043, "nauc_precision_at_3_max": 0.8792671540160817, "nauc_precision_at_3_std": -0.0728405183211521, "nauc_precision_at_5_diff1": 0.8970588235294136, "nauc_precision_at_5_max": 0.8727357609710571, "nauc_precision_at_5_std": -0.034842826019295975, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.9564270152505339, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 0.9564270152505339, "nauc_recall_at_10_diff1": 0.9223856209150244, "nauc_recall_at_10_max": 0.9051120448179192, "nauc_recall_at_10_std": 0.24778244631186175, "nauc_recall_at_1_diff1": 0.9139154970077102, "nauc_recall_at_1_max": 0.7432381431626621, "nauc_recall_at_1_std": -0.23120158869938737, "nauc_recall_at_20_diff1": 0.9325980392156854, "nauc_recall_at_20_max": 0.8740079365079421, "nauc_recall_at_20_std": 0.37219887955183756, "nauc_recall_at_3_diff1": 0.9079387852702074, "nauc_recall_at_3_max": 0.8792671540160791, "nauc_recall_at_3_std": -0.07284051832114759, "nauc_recall_at_5_diff1": 0.8970588235294114, "nauc_recall_at_5_max": 0.8727357609710548, "nauc_recall_at_5_std": -0.03484282601929706, "ndcg_at_1": 0.88222, "ndcg_at_10": 0.93651, "ndcg_at_100": 0.93868, "ndcg_at_1000": 0.93913, "ndcg_at_20": 0.93764, "ndcg_at_3": 0.92503, "ndcg_at_5": 0.93014, "precision_at_1": 0.88222, "precision_at_10": 0.09867, "precision_at_100": 0.00997, "precision_at_1000": 0.001, "precision_at_20": 0.04956, "precision_at_3": 0.31815, "precision_at_5": 0.19333, "recall_at_1": 0.88222, "recall_at_10": 0.98667, "recall_at_100": 0.99667, "recall_at_1000": 1.0, "recall_at_20": 0.99111, "recall_at_3": 0.95444, "recall_at_5": 0.96667 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.90391, "map_at_1": 0.84111, "map_at_10": 0.88401, "map_at_100": 0.88545, "map_at_1000": 0.8855, "map_at_20": 0.88521, "map_at_3": 0.87537, "map_at_5": 0.88098, "mrr_at_1": 0.8411111111111111, "mrr_at_10": 0.8840101410934745, "mrr_at_100": 0.885449650280911, "mrr_at_1000": 0.885497211675456, "mrr_at_20": 0.8852127172697349, "mrr_at_3": 0.8753703703703706, "mrr_at_5": 0.8809814814814817, "nauc_map_at_1000_diff1": 0.8800205613602536, "nauc_map_at_1000_max": 0.7926999370150674, "nauc_map_at_1000_std": 0.013580948072772584, "nauc_map_at_100_diff1": 0.8800259583134593, "nauc_map_at_100_max": 0.7927774217656682, "nauc_map_at_100_std": 0.01384112454071335, "nauc_map_at_10_diff1": 0.8788222743852696, "nauc_map_at_10_max": 0.7942137795875792, "nauc_map_at_10_std": 0.015055468916451471, "nauc_map_at_1_diff1": 0.8934627023392087, "nauc_map_at_1_max": 0.7673828165206228, "nauc_map_at_1_std": -0.04837270899521724, "nauc_map_at_20_diff1": 0.87989605902825, "nauc_map_at_20_max": 0.7930329844519671, "nauc_map_at_20_std": 0.014740922003401985, "nauc_map_at_3_diff1": 0.8784641018177448, "nauc_map_at_3_max": 0.8000216517565892, "nauc_map_at_3_std": 0.03202094054657089, "nauc_map_at_5_diff1": 0.879274314209589, "nauc_map_at_5_max": 0.7958737454163957, "nauc_map_at_5_std": 0.017080187929437385, "nauc_mrr_at_1000_diff1": 0.8800205613602536, "nauc_mrr_at_1000_max": 0.7926999370150674, "nauc_mrr_at_1000_std": 0.013580948072772584, "nauc_mrr_at_100_diff1": 0.8800259583134593, "nauc_mrr_at_100_max": 0.7927774217656682, "nauc_mrr_at_100_std": 0.01384112454071335, "nauc_mrr_at_10_diff1": 0.8788222743852696, "nauc_mrr_at_10_max": 0.7942137795875792, "nauc_mrr_at_10_std": 0.015055468916451471, "nauc_mrr_at_1_diff1": 0.8934627023392087, "nauc_mrr_at_1_max": 0.7673828165206228, "nauc_mrr_at_1_std": -0.04837270899521724, "nauc_mrr_at_20_diff1": 0.87989605902825, "nauc_mrr_at_20_max": 0.7930329844519671, "nauc_mrr_at_20_std": 0.014740922003401985, "nauc_mrr_at_3_diff1": 0.8784641018177448, "nauc_mrr_at_3_max": 0.8000216517565892, "nauc_mrr_at_3_std": 0.03202094054657089, "nauc_mrr_at_5_diff1": 0.879274314209589, "nauc_mrr_at_5_max": 0.7958737454163957, "nauc_mrr_at_5_std": 0.017080187929437385, "nauc_ndcg_at_1000_diff1": 0.8782251801092046, "nauc_ndcg_at_1000_max": 0.7977777090577682, "nauc_ndcg_at_1000_std": 0.02751690045587376, "nauc_ndcg_at_100_diff1": 0.8781033126465196, "nauc_ndcg_at_100_max": 0.799971982790663, "nauc_ndcg_at_100_std": 0.03584488608641146, "nauc_ndcg_at_10_diff1": 0.8729708547646923, "nauc_ndcg_at_10_max": 0.8066539438806426, "nauc_ndcg_at_10_std": 0.042598080649337566, "nauc_ndcg_at_1_diff1": 0.8934627023392087, "nauc_ndcg_at_1_max": 0.7673828165206228, "nauc_ndcg_at_1_std": -0.04837270899521724, "nauc_ndcg_at_20_diff1": 0.8767445156889189, "nauc_ndcg_at_20_max": 0.8021959993665218, "nauc_ndcg_at_20_std": 0.04301016312041599, "nauc_ndcg_at_3_diff1": 0.8735722119193429, "nauc_ndcg_at_3_max": 0.8169566966858675, "nauc_ndcg_at_3_std": 0.0728963995153794, "nauc_ndcg_at_5_diff1": 0.8745160029013536, "nauc_ndcg_at_5_max": 0.8102399907635793, "nauc_ndcg_at_5_std": 0.04683902089586952, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8622782446311745, "nauc_precision_at_100_max": 0.9813258636787873, "nauc_precision_at_100_std": 0.7409630518874429, "nauc_precision_at_10_diff1": 0.818183187253392, "nauc_precision_at_10_max": 0.9192042408361156, "nauc_precision_at_10_std": 0.2889671997831299, "nauc_precision_at_1_diff1": 0.8934627023392087, "nauc_precision_at_1_max": 0.7673828165206228, "nauc_precision_at_1_std": -0.04837270899521724, "nauc_precision_at_20_diff1": 0.835942220025268, "nauc_precision_at_20_max": 0.9314823968803195, "nauc_precision_at_20_std": 0.5027736584829903, "nauc_precision_at_3_diff1": 0.8532737295831584, "nauc_precision_at_3_max": 0.8908074645383278, "nauc_precision_at_3_std": 0.25084737091183645, "nauc_precision_at_5_diff1": 0.8470774976657317, "nauc_precision_at_5_max": 0.8960317460317422, "nauc_precision_at_5_std": 0.22169934640522632, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8622782446311739, "nauc_recall_at_100_max": 0.9813258636788001, "nauc_recall_at_100_std": 0.7409630518874236, "nauc_recall_at_10_diff1": 0.8181831872533971, "nauc_recall_at_10_max": 0.9192042408361157, "nauc_recall_at_10_std": 0.288967199783136, "nauc_recall_at_1_diff1": 0.8934627023392087, "nauc_recall_at_1_max": 0.7673828165206228, "nauc_recall_at_1_std": -0.04837270899521724, "nauc_recall_at_20_diff1": 0.8359422200252595, "nauc_recall_at_20_max": 0.9314823968803185, "nauc_recall_at_20_std": 0.5027736584830049, "nauc_recall_at_3_diff1": 0.8532737295831584, "nauc_recall_at_3_max": 0.8908074645383278, "nauc_recall_at_3_std": 0.25084737091183484, "nauc_recall_at_5_diff1": 0.8470774976657335, "nauc_recall_at_5_max": 0.8960317460317475, "nauc_recall_at_5_std": 0.22169934640523029, "ndcg_at_1": 0.84111, "ndcg_at_10": 0.90391, "ndcg_at_100": 0.90997, "ndcg_at_1000": 0.91102, "ndcg_at_20": 0.90799, "ndcg_at_3": 0.88655, "ndcg_at_5": 0.89687, "precision_at_1": 0.84111, "precision_at_10": 0.09656, "precision_at_100": 0.00992, "precision_at_1000": 0.001, "precision_at_20": 0.04906, "precision_at_3": 0.3063, "precision_at_5": 0.18889, "recall_at_1": 0.84111, "recall_at_10": 0.96556, "recall_at_100": 0.99222, "recall_at_1000": 1.0, "recall_at_20": 0.98111, "recall_at_3": 0.91889, "recall_at_5": 0.94444 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 106.8914806842804, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.80008, "map_at_1": 0.50224, "map_at_10": 0.75801, "map_at_100": 0.76353, "map_at_1000": 0.76387, "map_at_20": 0.76174, "map_at_3": 0.71805, "map_at_5": 0.74531, "mrr_at_1": 0.7668918918918919, "mrr_at_10": 0.8182526276276274, "mrr_at_100": 0.8198681055223843, "mrr_at_1000": 0.8200447358419406, "mrr_at_20": 0.8190945409695407, "mrr_at_3": 0.8127815315315313, "mrr_at_5": 0.8163288288288286, "nauc_map_at_1000_diff1": 0.5972954962902949, "nauc_map_at_1000_max": 0.6368142914602937, "nauc_map_at_1000_std": 0.19600667039933853, "nauc_map_at_100_diff1": 0.5971892993449279, "nauc_map_at_100_max": 0.6372732434536403, "nauc_map_at_100_std": 0.1967226339429915, "nauc_map_at_10_diff1": 0.5923749667778005, "nauc_map_at_10_max": 0.6339909454890105, "nauc_map_at_10_std": 0.18617034845772062, "nauc_map_at_1_diff1": 0.6407039079056288, "nauc_map_at_1_max": 0.29091469612516707, "nauc_map_at_1_std": -0.041544183402294585, "nauc_map_at_20_diff1": 0.5977171275491291, "nauc_map_at_20_max": 0.6388580471049708, "nauc_map_at_20_std": 0.1962525689943241, "nauc_map_at_3_diff1": 0.5900738657305518, "nauc_map_at_3_max": 0.5665593954782121, "nauc_map_at_3_std": 0.10966603066350789, "nauc_map_at_5_diff1": 0.5829464965480328, "nauc_map_at_5_max": 0.6176087413110913, "nauc_map_at_5_std": 0.16072461952483239, "nauc_mrr_at_1000_diff1": 0.709824810350201, "nauc_mrr_at_1000_max": 0.7345190416806782, "nauc_mrr_at_1000_std": 0.3035269580763068, "nauc_mrr_at_100_diff1": 0.7098141357802885, "nauc_mrr_at_100_max": 0.7347853756149684, "nauc_mrr_at_100_std": 0.30402465803211526, "nauc_mrr_at_10_diff1": 0.7091332962479558, "nauc_mrr_at_10_max": 0.7370409336686649, "nauc_mrr_at_10_std": 0.30411076208664795, "nauc_mrr_at_1_diff1": 0.7153536856458215, "nauc_mrr_at_1_max": 0.6367168927299622, "nauc_mrr_at_1_std": 0.1797255462450732, "nauc_mrr_at_20_diff1": 0.7098668857166636, "nauc_mrr_at_20_max": 0.7357816306698388, "nauc_mrr_at_20_std": 0.30487672649515535, "nauc_mrr_at_3_diff1": 0.7108485627220982, "nauc_mrr_at_3_max": 0.7366049198735798, "nauc_mrr_at_3_std": 0.2939209122656606, "nauc_mrr_at_5_diff1": 0.7094065640103121, "nauc_mrr_at_5_max": 0.736934209431478, "nauc_mrr_at_5_std": 0.3008189699564541, "nauc_ndcg_at_1000_diff1": 0.633209387968523, "nauc_ndcg_at_1000_max": 0.7017740521622267, "nauc_ndcg_at_1000_std": 0.2871940426224995, "nauc_ndcg_at_100_diff1": 0.6317275921514075, "nauc_ndcg_at_100_max": 0.7128265191344082, "nauc_ndcg_at_100_std": 0.3053516131697575, "nauc_ndcg_at_10_diff1": 0.6190533858054285, "nauc_ndcg_at_10_max": 0.7093609962144929, "nauc_ndcg_at_10_std": 0.2764905074705219, "nauc_ndcg_at_1_diff1": 0.7153536856458215, "nauc_ndcg_at_1_max": 0.6367168927299622, "nauc_ndcg_at_1_std": 0.1797255462450732, "nauc_ndcg_at_20_diff1": 0.6320458021594961, "nauc_ndcg_at_20_max": 0.7209179402000162, "nauc_ndcg_at_20_std": 0.3030488569549388, "nauc_ndcg_at_3_diff1": 0.6066549910978404, "nauc_ndcg_at_3_max": 0.6193225890761628, "nauc_ndcg_at_3_std": 0.16695273529007681, "nauc_ndcg_at_5_diff1": 0.603741227754187, "nauc_ndcg_at_5_max": 0.6818212630051793, "nauc_ndcg_at_5_std": 0.23294879593797604, "nauc_precision_at_1000_diff1": -0.32933520104718034, "nauc_precision_at_1000_max": 0.007161073542821553, "nauc_precision_at_1000_std": 0.2063019640721369, "nauc_precision_at_100_diff1": -0.23501168923658147, "nauc_precision_at_100_max": 0.14969003532471672, "nauc_precision_at_100_std": 0.3054829396918457, "nauc_precision_at_10_diff1": -0.13980158620840913, "nauc_precision_at_10_max": 0.28511337244894247, "nauc_precision_at_10_std": 0.31819918857431584, "nauc_precision_at_1_diff1": 0.7153536856458215, "nauc_precision_at_1_max": 0.6367168927299622, "nauc_precision_at_1_std": 0.1797255462450732, "nauc_precision_at_20_diff1": -0.1568592411175028, "nauc_precision_at_20_max": 0.24530994646011672, "nauc_precision_at_20_std": 0.3275801664626258, "nauc_precision_at_3_diff1": -0.0028509166513785832, "nauc_precision_at_3_max": 0.3727369576620084, "nauc_precision_at_3_std": 0.2417544691664725, "nauc_precision_at_5_diff1": -0.0951242611766236, "nauc_precision_at_5_max": 0.3373018586224311, "nauc_precision_at_5_std": 0.2866663123426568, "nauc_recall_at_1000_diff1": 0.722229751897174, "nauc_recall_at_1000_max": 0.9825486195833247, "nauc_recall_at_1000_std": 0.8508864855593524, "nauc_recall_at_100_diff1": 0.6518817183992185, "nauc_recall_at_100_max": 0.955087527405752, "nauc_recall_at_100_std": 0.7621854034955842, "nauc_recall_at_10_diff1": 0.5863070465583734, "nauc_recall_at_10_max": 0.8449272103086174, "nauc_recall_at_10_std": 0.45086935495626596, "nauc_recall_at_1_diff1": 0.6407039079056288, "nauc_recall_at_1_max": 0.29091469612516707, "nauc_recall_at_1_std": -0.041544183402294585, "nauc_recall_at_20_diff1": 0.6463966953750865, "nauc_recall_at_20_max": 0.9284309177278965, "nauc_recall_at_20_std": 0.604534239281333, "nauc_recall_at_3_diff1": 0.570011423577929, "nauc_recall_at_3_max": 0.6246426708462954, "nauc_recall_at_3_std": 0.17855913256991598, "nauc_recall_at_5_diff1": 0.5487110746355588, "nauc_recall_at_5_max": 0.7484194405605811, "nauc_recall_at_5_std": 0.3144622997742575, "ndcg_at_1": 0.76689, "ndcg_at_10": 0.80008, "ndcg_at_100": 0.81776, "ndcg_at_1000": 0.82521, "ndcg_at_20": 0.80834, "ndcg_at_3": 0.75702, "ndcg_at_5": 0.78307, "precision_at_1": 0.76689, "precision_at_10": 0.15777, "precision_at_100": 0.01733, "precision_at_1000": 0.00184, "precision_at_20": 0.08209, "precision_at_3": 0.44707, "precision_at_5": 0.29527, "recall_at_1": 0.50224, "recall_at_10": 0.85344, "recall_at_100": 0.91914, "recall_at_1000": 0.96585, "recall_at_20": 0.87809, "recall_at_3": 0.76087, "recall_at_5": 0.81504 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 10921.967575788498, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.62347, "map_at_1": 0.35651, "map_at_10": 0.53094, "map_at_100": 0.55538, "map_at_1000": 0.55625, "map_at_20": 0.5454, "map_at_3": 0.47163, "map_at_5": 0.50753, "mrr_at_1": 0.5868544600938967, "mrr_at_10": 0.7001993442134288, "mrr_at_100": 0.7042101595780245, "mrr_at_1000": 0.7042455978016589, "mrr_at_20": 0.7028372924292561, "mrr_at_3": 0.6768388106416277, "mrr_at_5": 0.6930359937402192, "nauc_map_at_1000_diff1": 0.3802267511775632, "nauc_map_at_1000_max": 0.19048610042141606, "nauc_map_at_1000_std": -0.014178318150586245, "nauc_map_at_100_diff1": 0.3798002126801598, "nauc_map_at_100_max": 0.18908979238980247, "nauc_map_at_100_std": -0.015448262064090671, "nauc_map_at_10_diff1": 0.383236046520275, "nauc_map_at_10_max": 0.17506495777101685, "nauc_map_at_10_std": -0.035756289573366726, "nauc_map_at_1_diff1": 0.4462558550231808, "nauc_map_at_1_max": 0.10671392419017076, "nauc_map_at_1_std": -0.12190099242327268, "nauc_map_at_20_diff1": 0.37968613381466515, "nauc_map_at_20_max": 0.18078138323656373, "nauc_map_at_20_std": -0.020758431921589562, "nauc_map_at_3_diff1": 0.4047470483336675, "nauc_map_at_3_max": 0.14357061794368886, "nauc_map_at_3_std": -0.09101061405383064, "nauc_map_at_5_diff1": 0.38433619003720515, "nauc_map_at_5_max": 0.15338802662837325, "nauc_map_at_5_std": -0.05731668390302651, "nauc_mrr_at_1000_diff1": 0.3709714979476026, "nauc_mrr_at_1000_max": 0.3447593541310457, "nauc_mrr_at_1000_std": 0.1836084604512021, "nauc_mrr_at_100_diff1": 0.37106380683123774, "nauc_mrr_at_100_max": 0.3448363567876449, "nauc_mrr_at_100_std": 0.1836612305412644, "nauc_mrr_at_10_diff1": 0.3715556628849159, "nauc_mrr_at_10_max": 0.3494585868210773, "nauc_mrr_at_10_std": 0.1887786307193617, "nauc_mrr_at_1_diff1": 0.3776020575107886, "nauc_mrr_at_1_max": 0.2819192654012434, "nauc_mrr_at_1_std": 0.12268082396371827, "nauc_mrr_at_20_diff1": 0.371308797431876, "nauc_mrr_at_20_max": 0.3479123610638993, "nauc_mrr_at_20_std": 0.18697030952227978, "nauc_mrr_at_3_diff1": 0.37969448696878133, "nauc_mrr_at_3_max": 0.3489639252891714, "nauc_mrr_at_3_std": 0.1625771762618304, "nauc_mrr_at_5_diff1": 0.362153262385325, "nauc_mrr_at_5_max": 0.33555805475045447, "nauc_mrr_at_5_std": 0.18240974346590325, "nauc_ndcg_at_1000_diff1": 0.3751583781231858, "nauc_ndcg_at_1000_max": 0.27117548808207703, "nauc_ndcg_at_1000_std": 0.0897526309335515, "nauc_ndcg_at_100_diff1": 0.37616095285107065, "nauc_ndcg_at_100_max": 0.2593365218742245, "nauc_ndcg_at_100_std": 0.07530443478383826, "nauc_ndcg_at_10_diff1": 0.37464050249950315, "nauc_ndcg_at_10_max": 0.2317434244594817, "nauc_ndcg_at_10_std": 0.03862097010400464, "nauc_ndcg_at_1_diff1": 0.3776020575107886, "nauc_ndcg_at_1_max": 0.2819192654012434, "nauc_ndcg_at_1_std": 0.12268082396371827, "nauc_ndcg_at_20_diff1": 0.37008298134554624, "nauc_ndcg_at_20_max": 0.23833642905140717, "nauc_ndcg_at_20_std": 0.06439132908022192, "nauc_ndcg_at_3_diff1": 0.39060167641094196, "nauc_ndcg_at_3_max": 0.2594239002284155, "nauc_ndcg_at_3_std": 0.01734384126072765, "nauc_ndcg_at_5_diff1": 0.36720392634007804, "nauc_ndcg_at_5_max": 0.2288281724358362, "nauc_ndcg_at_5_std": 0.017804644988806795, "nauc_precision_at_1000_diff1": -0.16345669343223276, "nauc_precision_at_1000_max": 0.26944451859423574, "nauc_precision_at_1000_std": 0.3243173706214259, "nauc_precision_at_100_diff1": -0.16268290130587268, "nauc_precision_at_100_max": 0.2570365653119325, "nauc_precision_at_100_std": 0.3157762055594767, "nauc_precision_at_10_diff1": -0.11165331177359057, "nauc_precision_at_10_max": 0.29596063735284117, "nauc_precision_at_10_std": 0.3403527806817954, "nauc_precision_at_1_diff1": 0.3776020575107886, "nauc_precision_at_1_max": 0.2819192654012434, "nauc_precision_at_1_std": 0.12268082396371827, "nauc_precision_at_20_diff1": -0.1531559621460741, "nauc_precision_at_20_max": 0.25594012686850015, "nauc_precision_at_20_std": 0.34699541667336714, "nauc_precision_at_3_diff1": 0.05941049842805729, "nauc_precision_at_3_max": 0.2953259707934061, "nauc_precision_at_3_std": 0.24270084666552041, "nauc_precision_at_5_diff1": -0.058609200820664654, "nauc_precision_at_5_max": 0.27794901240025927, "nauc_precision_at_5_std": 0.29911869651570383, "nauc_recall_at_1000_diff1": 0.3147765383896343, "nauc_recall_at_1000_max": 0.5100056604290448, "nauc_recall_at_1000_std": 0.7544025574496996, "nauc_recall_at_100_diff1": 0.35124680620202603, "nauc_recall_at_100_max": 0.20469848787224137, "nauc_recall_at_100_std": 0.0946357651158954, "nauc_recall_at_10_diff1": 0.3095506491637109, "nauc_recall_at_10_max": 0.18283091699784212, "nauc_recall_at_10_std": 0.03312080378888044, "nauc_recall_at_1_diff1": 0.4462558550231808, "nauc_recall_at_1_max": 0.10671392419017076, "nauc_recall_at_1_std": -0.12190099242327268, "nauc_recall_at_20_diff1": 0.2982577780687482, "nauc_recall_at_20_max": 0.19323530530562696, "nauc_recall_at_20_std": 0.1041300624127336, "nauc_recall_at_3_diff1": 0.3845104615232557, "nauc_recall_at_3_max": 0.13986804757783908, "nauc_recall_at_3_std": -0.11043292330852067, "nauc_recall_at_5_diff1": 0.3094744523243021, "nauc_recall_at_5_max": 0.11320806360380156, "nauc_recall_at_5_std": -0.047961070514019745, "ndcg_at_1": 0.58685, "ndcg_at_10": 0.62347, "ndcg_at_100": 0.68754, "ndcg_at_1000": 0.69732, "ndcg_at_20": 0.65146, "ndcg_at_3": 0.58375, "ndcg_at_5": 0.59917, "precision_at_1": 0.58685, "precision_at_10": 0.15493, "precision_at_100": 0.023, "precision_at_1000": 0.00253, "precision_at_20": 0.09085, "precision_at_3": 0.35211, "precision_at_5": 0.26197, "recall_at_1": 0.35651, "recall_at_10": 0.71945, "recall_at_100": 0.93058, "recall_at_1000": 0.98059, "recall_at_20": 0.79728, "recall_at_3": 0.54423, "recall_at_5": 0.63246 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 11006.327818870544, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.58411, "map_at_1": 0.41805, "map_at_10": 0.52884, "map_at_100": 0.53694, "map_at_1000": 0.53737, "map_at_20": 0.53353, "map_at_3": 0.49947, "map_at_5": 0.518, "mrr_at_1": 0.4513064133016627, "mrr_at_10": 0.5550852090638313, "mrr_at_100": 0.5611461188146258, "mrr_at_1000": 0.5614464312750885, "mrr_at_20": 0.5586867393535322, "mrr_at_3": 0.5304829770387967, "mrr_at_5": 0.5473475851148061, "nauc_map_at_1000_diff1": 0.41760449328203914, "nauc_map_at_1000_max": 0.3294380596195165, "nauc_map_at_1000_std": 0.07569114108854938, "nauc_map_at_100_diff1": 0.4175111507844809, "nauc_map_at_100_max": 0.3294347526478304, "nauc_map_at_100_std": 0.07562793957644538, "nauc_map_at_10_diff1": 0.41742159521872546, "nauc_map_at_10_max": 0.3283152091268034, "nauc_map_at_10_std": 0.07434936967786862, "nauc_map_at_1_diff1": 0.475616107892548, "nauc_map_at_1_max": 0.29980960217152125, "nauc_map_at_1_std": 0.023147594033258674, "nauc_map_at_20_diff1": 0.41792759661763335, "nauc_map_at_20_max": 0.32761807208712435, "nauc_map_at_20_std": 0.07385729953383378, "nauc_map_at_3_diff1": 0.4161404356977857, "nauc_map_at_3_max": 0.34133425026110237, "nauc_map_at_3_std": 0.07560165310487228, "nauc_map_at_5_diff1": 0.41369492120774864, "nauc_map_at_5_max": 0.33197752287454557, "nauc_map_at_5_std": 0.0735934727123139, "nauc_mrr_at_1000_diff1": 0.43425006536052757, "nauc_mrr_at_1000_max": 0.3310024020091237, "nauc_mrr_at_1000_std": 0.07584614898943615, "nauc_mrr_at_100_diff1": 0.43429153680356974, "nauc_mrr_at_100_max": 0.3311383359189307, "nauc_mrr_at_100_std": 0.07587475606863006, "nauc_mrr_at_10_diff1": 0.4331602379893548, "nauc_mrr_at_10_max": 0.3303249559514421, "nauc_mrr_at_10_std": 0.07517353705770899, "nauc_mrr_at_1_diff1": 0.48851459777795286, "nauc_mrr_at_1_max": 0.3061725961829622, "nauc_mrr_at_1_std": 0.027158195628080105, "nauc_mrr_at_20_diff1": 0.4345712872238659, "nauc_mrr_at_20_max": 0.33025249110418725, "nauc_mrr_at_20_std": 0.0745997468122339, "nauc_mrr_at_3_diff1": 0.43317807741797193, "nauc_mrr_at_3_max": 0.3413812271379621, "nauc_mrr_at_3_std": 0.07794841072340263, "nauc_mrr_at_5_diff1": 0.43046488691373813, "nauc_mrr_at_5_max": 0.33220204566899847, "nauc_mrr_at_5_std": 0.07367571090077339, "nauc_ndcg_at_1000_diff1": 0.4056461445113131, "nauc_ndcg_at_1000_max": 0.33847251877738377, "nauc_ndcg_at_1000_std": 0.09988485700168576, "nauc_ndcg_at_100_diff1": 0.4050811335370705, "nauc_ndcg_at_100_max": 0.3417355500075768, "nauc_ndcg_at_100_std": 0.10283628039248967, "nauc_ndcg_at_10_diff1": 0.4045446090303444, "nauc_ndcg_at_10_max": 0.330830952664249, "nauc_ndcg_at_10_std": 0.09445382646344905, "nauc_ndcg_at_1_diff1": 0.48851459777795286, "nauc_ndcg_at_1_max": 0.3061725961829622, "nauc_ndcg_at_1_std": 0.027158195628080105, "nauc_ndcg_at_20_diff1": 0.4072683827853017, "nauc_ndcg_at_20_max": 0.328225670980137, "nauc_ndcg_at_20_std": 0.09309495403978528, "nauc_ndcg_at_3_diff1": 0.4027853090100297, "nauc_ndcg_at_3_max": 0.3542400905704392, "nauc_ndcg_at_3_std": 0.09533485495668449, "nauc_ndcg_at_5_diff1": 0.39783018881462934, "nauc_ndcg_at_5_max": 0.3374839089915202, "nauc_ndcg_at_5_std": 0.09053126979087693, "nauc_precision_at_1000_diff1": -0.09871951704278333, "nauc_precision_at_1000_max": 0.05841793928506, "nauc_precision_at_1000_std": 0.12773862644439057, "nauc_precision_at_100_diff1": 0.003676016719144542, "nauc_precision_at_100_max": 0.15404278753413259, "nauc_precision_at_100_std": 0.16068194939581307, "nauc_precision_at_10_diff1": 0.17936005392549736, "nauc_precision_at_10_max": 0.21001566765138768, "nauc_precision_at_10_std": 0.14187082726161146, "nauc_precision_at_1_diff1": 0.48851459777795286, "nauc_precision_at_1_max": 0.3061725961829622, "nauc_precision_at_1_std": 0.027158195628080105, "nauc_precision_at_20_diff1": 0.12500512313614456, "nauc_precision_at_20_max": 0.1522781682963013, "nauc_precision_at_20_std": 0.12870295520287567, "nauc_precision_at_3_diff1": 0.28997975780125107, "nauc_precision_at_3_max": 0.3658453521595714, "nauc_precision_at_3_std": 0.16499998570789157, "nauc_precision_at_5_diff1": 0.2345875365715847, "nauc_precision_at_5_max": 0.2999744081349785, "nauc_precision_at_5_std": 0.15568958648649867, "nauc_recall_at_1000_diff1": 0.023859311683500334, "nauc_recall_at_1000_max": 0.518210226146323, "nauc_recall_at_1000_std": 0.5796703247407008, "nauc_recall_at_100_diff1": 0.2654252637888202, "nauc_recall_at_100_max": 0.4652171149717772, "nauc_recall_at_100_std": 0.33527057410944405, "nauc_recall_at_10_diff1": 0.33321285789728705, "nauc_recall_at_10_max": 0.3258428005903868, "nauc_recall_at_10_std": 0.1671009470268551, "nauc_recall_at_1_diff1": 0.475616107892548, "nauc_recall_at_1_max": 0.29980960217152125, "nauc_recall_at_1_std": 0.023147594033258674, "nauc_recall_at_20_diff1": 0.3382006406737999, "nauc_recall_at_20_max": 0.30995358300824283, "nauc_recall_at_20_std": 0.1733813033212232, "nauc_recall_at_3_diff1": 0.34208486634308105, "nauc_recall_at_3_max": 0.3867663480452235, "nauc_recall_at_3_std": 0.14991917986638886, "nauc_recall_at_5_diff1": 0.3192847012739187, "nauc_recall_at_5_max": 0.34254888460256167, "nauc_recall_at_5_std": 0.14054972085304943, "ndcg_at_1": 0.45131, "ndcg_at_10": 0.58411, "ndcg_at_100": 0.62068, "ndcg_at_1000": 0.63057, "ndcg_at_20": 0.59975, "ndcg_at_3": 0.53024, "ndcg_at_5": 0.56183, "precision_at_1": 0.45131, "precision_at_10": 0.08314, "precision_at_100": 0.01036, "precision_at_1000": 0.00112, "precision_at_20": 0.04525, "precision_at_3": 0.21932, "precision_at_5": 0.14917, "recall_at_1": 0.41805, "recall_at_10": 0.72447, "recall_at_100": 0.88876, "recall_at_1000": 0.96437, "recall_at_20": 0.78345, "recall_at_3": 0.58868, "recall_at_5": 0.66192 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 374.64185643196106, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.30869, "map_at_1": 0.225, "map_at_10": 0.27839, "map_at_100": 0.28562, "map_at_1000": 0.28656, "map_at_20": 0.28249, "map_at_3": 0.25917, "map_at_5": 0.27217, "mrr_at_1": 0.225, "mrr_at_10": 0.2783948412698413, "mrr_at_100": 0.2856210546112084, "mrr_at_1000": 0.2865550826753487, "mrr_at_20": 0.28249013486513497, "mrr_at_3": 0.2591666666666667, "mrr_at_5": 0.2721666666666667, "nauc_map_at_1000_diff1": 0.4955255586619396, "nauc_map_at_1000_max": 0.44504849728519563, "nauc_map_at_1000_std": -0.038992619454156975, "nauc_map_at_100_diff1": 0.49560258641813304, "nauc_map_at_100_max": 0.4450275819616694, "nauc_map_at_100_std": -0.038660364252145515, "nauc_map_at_10_diff1": 0.4962892380511699, "nauc_map_at_10_max": 0.4440807062760643, "nauc_map_at_10_std": -0.04468313245045792, "nauc_map_at_1_diff1": 0.5586960705076397, "nauc_map_at_1_max": 0.4705538236539276, "nauc_map_at_1_std": -0.058784589924181264, "nauc_map_at_20_diff1": 0.4974509233090044, "nauc_map_at_20_max": 0.4436478366100422, "nauc_map_at_20_std": -0.04357356693958735, "nauc_map_at_3_diff1": 0.5062583328599041, "nauc_map_at_3_max": 0.43715739329406483, "nauc_map_at_3_std": -0.05578045127686905, "nauc_map_at_5_diff1": 0.5039749240776535, "nauc_map_at_5_max": 0.44506965120497405, "nauc_map_at_5_std": -0.04750545663729009, "nauc_mrr_at_1000_diff1": 0.4955255586619396, "nauc_mrr_at_1000_max": 0.44504849728519563, "nauc_mrr_at_1000_std": -0.038992619454156975, "nauc_mrr_at_100_diff1": 0.49560258641813304, "nauc_mrr_at_100_max": 0.4450275819616694, "nauc_mrr_at_100_std": -0.038660364252145515, "nauc_mrr_at_10_diff1": 0.4962892380511699, "nauc_mrr_at_10_max": 0.4440807062760643, "nauc_mrr_at_10_std": -0.04468313245045792, "nauc_mrr_at_1_diff1": 0.5586960705076397, "nauc_mrr_at_1_max": 0.4705538236539276, "nauc_mrr_at_1_std": -0.058784589924181264, "nauc_mrr_at_20_diff1": 0.4974509233090044, "nauc_mrr_at_20_max": 0.4436478366100422, "nauc_mrr_at_20_std": -0.04357356693958735, "nauc_mrr_at_3_diff1": 0.5062583328599041, "nauc_mrr_at_3_max": 0.43715739329406483, "nauc_mrr_at_3_std": -0.05578045127686905, "nauc_mrr_at_5_diff1": 0.5039749240776535, "nauc_mrr_at_5_max": 0.44506965120497405, "nauc_mrr_at_5_std": -0.04750545663729009, "nauc_ndcg_at_1000_diff1": 0.4660598275346212, "nauc_ndcg_at_1000_max": 0.45280397565939245, "nauc_ndcg_at_1000_std": 0.008694053070665663, "nauc_ndcg_at_100_diff1": 0.4651022132533749, "nauc_ndcg_at_100_max": 0.44752944736864253, "nauc_ndcg_at_100_std": 0.014536279874750873, "nauc_ndcg_at_10_diff1": 0.46973038934283456, "nauc_ndcg_at_10_max": 0.43947463568381123, "nauc_ndcg_at_10_std": -0.030194439183616174, "nauc_ndcg_at_1_diff1": 0.5586960705076397, "nauc_ndcg_at_1_max": 0.4705538236539276, "nauc_ndcg_at_1_std": -0.058784589924181264, "nauc_ndcg_at_20_diff1": 0.47711457240048044, "nauc_ndcg_at_20_max": 0.441598930338093, "nauc_ndcg_at_20_std": -0.022550640444449096, "nauc_ndcg_at_3_diff1": 0.4908716670181991, "nauc_ndcg_at_3_max": 0.4258518504997637, "nauc_ndcg_at_3_std": -0.05437443315310704, "nauc_ndcg_at_5_diff1": 0.4859565184409851, "nauc_ndcg_at_5_max": 0.4390595623034592, "nauc_ndcg_at_5_std": -0.040078077863090804, "nauc_precision_at_1000_diff1": 0.26713983690727916, "nauc_precision_at_1000_max": 0.6180240947682804, "nauc_precision_at_1000_std": 0.5043625625020973, "nauc_precision_at_100_diff1": 0.3595962428150849, "nauc_precision_at_100_max": 0.47169198757443165, "nauc_precision_at_100_std": 0.27103824330207266, "nauc_precision_at_10_diff1": 0.3953074527553725, "nauc_precision_at_10_max": 0.4309367686028743, "nauc_precision_at_10_std": 0.015925517579321397, "nauc_precision_at_1_diff1": 0.5586960705076397, "nauc_precision_at_1_max": 0.4705538236539276, "nauc_precision_at_1_std": -0.058784589924181264, "nauc_precision_at_20_diff1": 0.4278371209877518, "nauc_precision_at_20_max": 0.4460086480936678, "nauc_precision_at_20_std": 0.052184729598988976, "nauc_precision_at_3_diff1": 0.44962159757930276, "nauc_precision_at_3_max": 0.39509525767082015, "nauc_precision_at_3_std": -0.050449163336410925, "nauc_precision_at_5_diff1": 0.43724741510523296, "nauc_precision_at_5_max": 0.42402771217131174, "nauc_precision_at_5_std": -0.018671600272922646, "nauc_recall_at_1000_diff1": 0.267139836907279, "nauc_recall_at_1000_max": 0.6180240947682816, "nauc_recall_at_1000_std": 0.5043625625020984, "nauc_recall_at_100_diff1": 0.3595962428150849, "nauc_recall_at_100_max": 0.47169198757443176, "nauc_recall_at_100_std": 0.2710382433020728, "nauc_recall_at_10_diff1": 0.39530745275537255, "nauc_recall_at_10_max": 0.4309367686028742, "nauc_recall_at_10_std": 0.015925517579321348, "nauc_recall_at_1_diff1": 0.5586960705076397, "nauc_recall_at_1_max": 0.4705538236539276, "nauc_recall_at_1_std": -0.058784589924181264, "nauc_recall_at_20_diff1": 0.4278371209877519, "nauc_recall_at_20_max": 0.44600864809366825, "nauc_recall_at_20_std": 0.05218472959898892, "nauc_recall_at_3_diff1": 0.4496215975793028, "nauc_recall_at_3_max": 0.3950952576708202, "nauc_recall_at_3_std": -0.05044916333641102, "nauc_recall_at_5_diff1": 0.4372474151052326, "nauc_recall_at_5_max": 0.42402771217131147, "nauc_recall_at_5_std": -0.018671600272923, "ndcg_at_1": 0.225, "ndcg_at_10": 0.30869, "ndcg_at_100": 0.34748, "ndcg_at_1000": 0.37894, "ndcg_at_20": 0.32377, "ndcg_at_3": 0.27089, "ndcg_at_5": 0.29392, "precision_at_1": 0.225, "precision_at_10": 0.0405, "precision_at_100": 0.00595, "precision_at_1000": 0.00086, "precision_at_20": 0.02325, "precision_at_3": 0.10167, "precision_at_5": 0.072, "recall_at_1": 0.225, "recall_at_10": 0.405, "recall_at_100": 0.595, "recall_at_1000": 0.855, "recall_at_20": 0.465, "recall_at_3": 0.305, "recall_at_5": 0.36 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.29422, "map_at_1": 0.21, "map_at_10": 0.26295, "map_at_100": 0.26991, "map_at_1000": 0.27095, "map_at_20": 0.26708, "map_at_3": 0.24583, "map_at_5": 0.25508, "mrr_at_1": 0.21, "mrr_at_10": 0.2629464285714287, "mrr_at_100": 0.2699074844783189, "mrr_at_1000": 0.27094814561293573, "mrr_at_20": 0.26707917877616033, "mrr_at_3": 0.24583333333333335, "mrr_at_5": 0.25508333333333333, "nauc_map_at_1000_diff1": 0.5993055629941758, "nauc_map_at_1000_max": 0.5573641723030943, "nauc_map_at_1000_std": -0.028809539494729766, "nauc_map_at_100_diff1": 0.599754447437361, "nauc_map_at_100_max": 0.5573981154342734, "nauc_map_at_100_std": -0.02890453697300498, "nauc_map_at_10_diff1": 0.6023176407829586, "nauc_map_at_10_max": 0.5605663848880248, "nauc_map_at_10_std": -0.029242256541835446, "nauc_map_at_1_diff1": 0.6437597985652526, "nauc_map_at_1_max": 0.5890224397041822, "nauc_map_at_1_std": -0.06044626031323742, "nauc_map_at_20_diff1": 0.5994845168527906, "nauc_map_at_20_max": 0.5554448313082759, "nauc_map_at_20_std": -0.035068836554058784, "nauc_map_at_3_diff1": 0.6056887725170285, "nauc_map_at_3_max": 0.5655388339621944, "nauc_map_at_3_std": -0.04295974260959779, "nauc_map_at_5_diff1": 0.6090842844289199, "nauc_map_at_5_max": 0.5634194263131953, "nauc_map_at_5_std": -0.03926705649959931, "nauc_mrr_at_1000_diff1": 0.5993055629941758, "nauc_mrr_at_1000_max": 0.5573641723030943, "nauc_mrr_at_1000_std": -0.028809539494729766, "nauc_mrr_at_100_diff1": 0.599754447437361, "nauc_mrr_at_100_max": 0.5573981154342734, "nauc_mrr_at_100_std": -0.02890453697300498, "nauc_mrr_at_10_diff1": 0.6023176407829586, "nauc_mrr_at_10_max": 0.5605663848880248, "nauc_mrr_at_10_std": -0.029242256541835446, "nauc_mrr_at_1_diff1": 0.6437597985652526, "nauc_mrr_at_1_max": 0.5890224397041822, "nauc_mrr_at_1_std": -0.06044626031323742, "nauc_mrr_at_20_diff1": 0.5994845168527906, "nauc_mrr_at_20_max": 0.5554448313082759, "nauc_mrr_at_20_std": -0.035068836554058784, "nauc_mrr_at_3_diff1": 0.6056887725170285, "nauc_mrr_at_3_max": 0.5655388339621944, "nauc_mrr_at_3_std": -0.04295974260959779, "nauc_mrr_at_5_diff1": 0.6090842844289199, "nauc_mrr_at_5_max": 0.5634194263131953, "nauc_mrr_at_5_std": -0.03926705649959931, "nauc_ndcg_at_1000_diff1": 0.5699912603235107, "nauc_ndcg_at_1000_max": 0.5527261730583227, "nauc_ndcg_at_1000_std": 0.015365660275101606, "nauc_ndcg_at_100_diff1": 0.5802480477342637, "nauc_ndcg_at_100_max": 0.5492838888900472, "nauc_ndcg_at_100_std": 0.014018667337782901, "nauc_ndcg_at_10_diff1": 0.5855860402088727, "nauc_ndcg_at_10_max": 0.5495709356998074, "nauc_ndcg_at_10_std": -0.009903153000061426, "nauc_ndcg_at_1_diff1": 0.6437597985652526, "nauc_ndcg_at_1_max": 0.5890224397041822, "nauc_ndcg_at_1_std": -0.06044626031323742, "nauc_ndcg_at_20_diff1": 0.5743342832932331, "nauc_ndcg_at_20_max": 0.5334021632144429, "nauc_ndcg_at_20_std": -0.0292445969650835, "nauc_ndcg_at_3_diff1": 0.5941871898787882, "nauc_ndcg_at_3_max": 0.5584074268194473, "nauc_ndcg_at_3_std": -0.038877830654709296, "nauc_ndcg_at_5_diff1": 0.6001032735514831, "nauc_ndcg_at_5_max": 0.5546463616504773, "nauc_ndcg_at_5_std": -0.03270964849256897, "nauc_precision_at_1000_diff1": 0.3298034185484586, "nauc_precision_at_1000_max": 0.6153536378009128, "nauc_precision_at_1000_std": 0.41801151448259366, "nauc_precision_at_100_diff1": 0.5279182261912579, "nauc_precision_at_100_max": 0.5474934518207223, "nauc_precision_at_100_std": 0.21157732688365083, "nauc_precision_at_10_diff1": 0.5395469472814903, "nauc_precision_at_10_max": 0.5205718967953392, "nauc_precision_at_10_std": 0.04711825695354005, "nauc_precision_at_1_diff1": 0.6437597985652526, "nauc_precision_at_1_max": 0.5890224397041822, "nauc_precision_at_1_std": -0.06044626031323742, "nauc_precision_at_20_diff1": 0.4956242937110718, "nauc_precision_at_20_max": 0.46506005098420566, "nauc_precision_at_20_std": -0.02001918477832353, "nauc_precision_at_3_diff1": 0.5635604476442821, "nauc_precision_at_3_max": 0.5394072731978777, "nauc_precision_at_3_std": -0.02834898816862272, "nauc_precision_at_5_diff1": 0.5772732404093264, "nauc_precision_at_5_max": 0.531154333981898, "nauc_precision_at_5_std": -0.01529328171380355, "nauc_recall_at_1000_diff1": 0.3298034185484597, "nauc_recall_at_1000_max": 0.6153536378009135, "nauc_recall_at_1000_std": 0.4180115144825946, "nauc_recall_at_100_diff1": 0.5279182261912576, "nauc_recall_at_100_max": 0.547493451820722, "nauc_recall_at_100_std": 0.2115773268836503, "nauc_recall_at_10_diff1": 0.5395469472814901, "nauc_recall_at_10_max": 0.5205718967953392, "nauc_recall_at_10_std": 0.04711825695353978, "nauc_recall_at_1_diff1": 0.6437597985652526, "nauc_recall_at_1_max": 0.5890224397041822, "nauc_recall_at_1_std": -0.06044626031323742, "nauc_recall_at_20_diff1": 0.4956242937110719, "nauc_recall_at_20_max": 0.4650600509842053, "nauc_recall_at_20_std": -0.020019184778323842, "nauc_recall_at_3_diff1": 0.5635604476442824, "nauc_recall_at_3_max": 0.5394072731978776, "nauc_recall_at_3_std": -0.02834898816862281, "nauc_recall_at_5_diff1": 0.5772732404093258, "nauc_recall_at_5_max": 0.5311543339818973, "nauc_recall_at_5_std": -0.015293281713804032, "ndcg_at_1": 0.21, "ndcg_at_10": 0.29422, "ndcg_at_100": 0.33265, "ndcg_at_1000": 0.36357, "ndcg_at_20": 0.31017, "ndcg_at_3": 0.25839, "ndcg_at_5": 0.27496, "precision_at_1": 0.21, "precision_at_10": 0.0395, "precision_at_100": 0.00585, "precision_at_1000": 0.00084, "precision_at_20": 0.023, "precision_at_3": 0.09833, "precision_at_5": 0.067, "recall_at_1": 0.21, "recall_at_10": 0.395, "recall_at_100": 0.585, "recall_at_1000": 0.835, "recall_at_20": 0.46, "recall_at_3": 0.295, "recall_at_5": 0.335 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 2.5225961208343506, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.83507, "map_at_1": 0.71429, "map_at_10": 0.79848, "map_at_100": 0.8006, "map_at_1000": 0.8006, "map_at_20": 0.79848, "map_at_3": 0.78139, "map_at_5": 0.79307, "mrr_at_1": 0.7142857142857143, "mrr_at_10": 0.7984848484848486, "mrr_at_100": 0.8005995952099849, "mrr_at_1000": 0.8005995952099849, "mrr_at_20": 0.7984848484848486, "mrr_at_3": 0.7813852813852814, "mrr_at_5": 0.7930735930735933, "nauc_map_at_1000_diff1": 0.7913266849850282, "nauc_map_at_1000_max": 0.3344967478481731, "nauc_map_at_1000_std": -0.27987464893738734, "nauc_map_at_100_diff1": 0.7913266849850282, "nauc_map_at_100_max": 0.3344967478481731, "nauc_map_at_100_std": -0.27987464893738734, "nauc_map_at_10_diff1": 0.7926589773096495, "nauc_map_at_10_max": 0.3330035751571369, "nauc_map_at_10_std": -0.27943277460446947, "nauc_map_at_1_diff1": 0.7577173196038672, "nauc_map_at_1_max": 0.32284171451891813, "nauc_map_at_1_std": -0.20882769514998567, "nauc_map_at_20_diff1": 0.7926589773096495, "nauc_map_at_20_max": 0.3330035751571369, "nauc_map_at_20_std": -0.27943277460446947, "nauc_map_at_3_diff1": 0.8036211706145331, "nauc_map_at_3_max": 0.3491022360132139, "nauc_map_at_3_std": -0.29770470107793134, "nauc_map_at_5_diff1": 0.792365609522334, "nauc_map_at_5_max": 0.3310666097797035, "nauc_map_at_5_std": -0.2977794052487303, "nauc_mrr_at_1000_diff1": 0.7913266849850282, "nauc_mrr_at_1000_max": 0.3344967478481731, "nauc_mrr_at_1000_std": -0.27987464893738734, "nauc_mrr_at_100_diff1": 0.7913266849850282, "nauc_mrr_at_100_max": 0.3344967478481731, "nauc_mrr_at_100_std": -0.27987464893738734, "nauc_mrr_at_10_diff1": 0.7926589773096495, "nauc_mrr_at_10_max": 0.3330035751571369, "nauc_mrr_at_10_std": -0.27943277460446947, "nauc_mrr_at_1_diff1": 0.7577173196038672, "nauc_mrr_at_1_max": 0.32284171451891813, "nauc_mrr_at_1_std": -0.20882769514998567, "nauc_mrr_at_20_diff1": 0.7926589773096495, "nauc_mrr_at_20_max": 0.3330035751571369, "nauc_mrr_at_20_std": -0.27943277460446947, "nauc_mrr_at_3_diff1": 0.8036211706145331, "nauc_mrr_at_3_max": 0.3491022360132139, "nauc_mrr_at_3_std": -0.29770470107793134, "nauc_mrr_at_5_diff1": 0.792365609522334, "nauc_mrr_at_5_max": 0.3310666097797035, "nauc_mrr_at_5_std": -0.2977794052487303, "nauc_ndcg_at_1000_diff1": 0.7956021251421836, "nauc_ndcg_at_1000_max": 0.33379727067161213, "nauc_ndcg_at_1000_std": -0.2862451635342314, "nauc_ndcg_at_100_diff1": 0.7956021251421836, "nauc_ndcg_at_100_max": 0.33379727067161213, "nauc_ndcg_at_100_std": -0.2862451635342314, "nauc_ndcg_at_10_diff1": 0.8035444610764061, "nauc_ndcg_at_10_max": 0.3256076661295447, "nauc_ndcg_at_10_std": -0.2817943860852995, "nauc_ndcg_at_1_diff1": 0.7577173196038672, "nauc_ndcg_at_1_max": 0.32284171451891813, "nauc_ndcg_at_1_std": -0.20882769514998567, "nauc_ndcg_at_20_diff1": 0.8035444610764061, "nauc_ndcg_at_20_max": 0.3256076661295447, "nauc_ndcg_at_20_std": -0.2817943860852995, "nauc_ndcg_at_3_diff1": 0.8231850214230185, "nauc_ndcg_at_3_max": 0.35797191104736537, "nauc_ndcg_at_3_std": -0.3364824459168836, "nauc_ndcg_at_5_diff1": 0.8030605288101285, "nauc_ndcg_at_5_max": 0.32315494694121266, "nauc_ndcg_at_5_std": -0.33328124665004133, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.9015345268542192, "nauc_precision_at_10_max": 0.23077751617064066, "nauc_precision_at_10_std": -0.21914209780721652, "nauc_precision_at_1_diff1": 0.7577173196038672, "nauc_precision_at_1_max": 0.32284171451891813, "nauc_precision_at_1_std": -0.20882769514998567, "nauc_precision_at_20_diff1": 0.9015345268542192, "nauc_precision_at_20_max": 0.23077751617064066, "nauc_precision_at_20_std": -0.21914209780721652, "nauc_precision_at_3_diff1": 0.9036789848803191, "nauc_precision_at_3_max": 0.3934861268650965, "nauc_precision_at_3_std": -0.4964096640360838, "nauc_precision_at_5_diff1": 0.8613853520214367, "nauc_precision_at_5_max": 0.2666030629895003, "nauc_precision_at_5_std": -0.5325711292896321, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.9015345268542205, "nauc_recall_at_10_max": 0.2307775161706363, "nauc_recall_at_10_std": -0.2191420978072253, "nauc_recall_at_1_diff1": 0.7577173196038672, "nauc_recall_at_1_max": 0.32284171451891813, "nauc_recall_at_1_std": -0.20882769514998567, "nauc_recall_at_20_diff1": 0.9015345268542205, "nauc_recall_at_20_max": 0.2307775161706363, "nauc_recall_at_20_std": -0.2191420978072253, "nauc_recall_at_3_diff1": 0.903678984880319, "nauc_recall_at_3_max": 0.3934861268650947, "nauc_recall_at_3_std": -0.49640966403608566, "nauc_recall_at_5_diff1": 0.8613853520214378, "nauc_recall_at_5_max": 0.26660306298949915, "nauc_recall_at_5_std": -0.5325711292896324, "ndcg_at_1": 0.71429, "ndcg_at_10": 0.83507, "ndcg_at_100": 0.84618, "ndcg_at_1000": 0.84618, "ndcg_at_20": 0.83507, "ndcg_at_3": 0.80102, "ndcg_at_5": 0.82225, "precision_at_1": 0.71429, "precision_at_10": 0.09481, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.0474, "precision_at_3": 0.28571, "precision_at_5": 0.18182, "recall_at_1": 0.71429, "recall_at_10": 0.94805, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.94805, "recall_at_3": 0.85714, "recall_at_5": 0.90909 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 21.183428287506104, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.35086, "map_at_1": 0.19131, "map_at_10": 0.30434, "map_at_100": 0.32044, "map_at_1000": 0.3217, "map_at_20": 0.31239, "map_at_3": 0.27181, "map_at_5": 0.29452, "mrr_at_1": 0.27522935779816515, "mrr_at_10": 0.34570833940099976, "mrr_at_100": 0.35727638920899696, "mrr_at_1000": 0.35812819846900046, "mrr_at_20": 0.3513326196391708, "mrr_at_3": 0.32237512742099905, "mrr_at_5": 0.3391182466870539, "nauc_map_at_1000_diff1": 0.409891211890938, "nauc_map_at_1000_max": 0.16453956251285742, "nauc_map_at_1000_std": -0.3139775027453628, "nauc_map_at_100_diff1": 0.40949505819737153, "nauc_map_at_100_max": 0.16508576648509377, "nauc_map_at_100_std": -0.3127088715184433, "nauc_map_at_10_diff1": 0.4122547974430382, "nauc_map_at_10_max": 0.16044604520693573, "nauc_map_at_10_std": -0.3248597067207718, "nauc_map_at_1_diff1": 0.4777061091822854, "nauc_map_at_1_max": 0.14012396295028154, "nauc_map_at_1_std": -0.2933447558477882, "nauc_map_at_20_diff1": 0.4096657611840877, "nauc_map_at_20_max": 0.1596009991931055, "nauc_map_at_20_std": -0.32136636100620725, "nauc_map_at_3_diff1": 0.4150801755132676, "nauc_map_at_3_max": 0.16114985288044859, "nauc_map_at_3_std": -0.3152011065710757, "nauc_map_at_5_diff1": 0.4156042299065368, "nauc_map_at_5_max": 0.164822475562178, "nauc_map_at_5_std": -0.3253096219738663, "nauc_mrr_at_1000_diff1": 0.4316079477998367, "nauc_mrr_at_1000_max": 0.16386364595135064, "nauc_mrr_at_1000_std": -0.31489718856720234, "nauc_mrr_at_100_diff1": 0.4314284888707276, "nauc_mrr_at_100_max": 0.16450638693823644, "nauc_mrr_at_100_std": -0.3137283988869892, "nauc_mrr_at_10_diff1": 0.43309757520126724, "nauc_mrr_at_10_max": 0.16043575416951922, "nauc_mrr_at_10_std": -0.3229044396958187, "nauc_mrr_at_1_diff1": 0.45481694829443947, "nauc_mrr_at_1_max": 0.180971841227373, "nauc_mrr_at_1_std": -0.3205775185600898, "nauc_mrr_at_20_diff1": 0.4306733326980395, "nauc_mrr_at_20_max": 0.1603739795107345, "nauc_mrr_at_20_std": -0.31915671054108863, "nauc_mrr_at_3_diff1": 0.43581235824304093, "nauc_mrr_at_3_max": 0.1525340587607215, "nauc_mrr_at_3_std": -0.32962878843671045, "nauc_mrr_at_5_diff1": 0.43567216072404585, "nauc_mrr_at_5_max": 0.15921852607655793, "nauc_mrr_at_5_std": -0.32828041586705575, "nauc_ndcg_at_1000_diff1": 0.4087053309824719, "nauc_ndcg_at_1000_max": 0.17254236463813988, "nauc_ndcg_at_1000_std": -0.29251465841312, "nauc_ndcg_at_100_diff1": 0.4002654119015821, "nauc_ndcg_at_100_max": 0.18760731852733362, "nauc_ndcg_at_100_std": -0.25818039099702117, "nauc_ndcg_at_10_diff1": 0.4074582908497202, "nauc_ndcg_at_10_max": 0.15382112712853638, "nauc_ndcg_at_10_std": -0.32581525769692254, "nauc_ndcg_at_1_diff1": 0.46062551861494544, "nauc_ndcg_at_1_max": 0.18478273585670998, "nauc_ndcg_at_1_std": -0.31807342680384515, "nauc_ndcg_at_20_diff1": 0.3987930474204499, "nauc_ndcg_at_20_max": 0.15167313866012083, "nauc_ndcg_at_20_std": -0.3123073905402383, "nauc_ndcg_at_3_diff1": 0.413138499471295, "nauc_ndcg_at_3_max": 0.14968205337514273, "nauc_ndcg_at_3_std": -0.32356853747445136, "nauc_ndcg_at_5_diff1": 0.41559541269795314, "nauc_ndcg_at_5_max": 0.16058447547298413, "nauc_ndcg_at_5_std": -0.3306809176053113, "nauc_precision_at_1000_diff1": 0.005598514217803497, "nauc_precision_at_1000_max": 0.12201578109620512, "nauc_precision_at_1000_std": 0.04235933539624416, "nauc_precision_at_100_diff1": 0.09573341817465238, "nauc_precision_at_100_max": 0.21966018517316385, "nauc_precision_at_100_std": 0.08492736005360908, "nauc_precision_at_10_diff1": 0.23415224862463332, "nauc_precision_at_10_max": 0.1210617698100394, "nauc_precision_at_10_std": -0.27548985963557165, "nauc_precision_at_1_diff1": 0.46062551861494544, "nauc_precision_at_1_max": 0.18478273585670998, "nauc_precision_at_1_std": -0.31807342680384515, "nauc_precision_at_20_diff1": 0.19211125561102455, "nauc_precision_at_20_max": 0.11098938573217373, "nauc_precision_at_20_std": -0.21514662243840546, "nauc_precision_at_3_diff1": 0.2854559172268683, "nauc_precision_at_3_max": 0.15258475902272714, "nauc_precision_at_3_std": -0.2985137253538266, "nauc_precision_at_5_diff1": 0.2669222443359308, "nauc_precision_at_5_max": 0.1500651160680117, "nauc_precision_at_5_std": -0.30143876723721863, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.2862413184497751, "nauc_recall_at_100_max": 0.3338603298540222, "nauc_recall_at_100_std": 0.12383628514709578, "nauc_recall_at_10_diff1": 0.3591712788207327, "nauc_recall_at_10_max": 0.13294003401397117, "nauc_recall_at_10_std": -0.31282126791818005, "nauc_recall_at_1_diff1": 0.4777061091822854, "nauc_recall_at_1_max": 0.14012396295028154, "nauc_recall_at_1_std": -0.2933447558477882, "nauc_recall_at_20_diff1": 0.3242252495515925, "nauc_recall_at_20_max": 0.12006091569134453, "nauc_recall_at_20_std": -0.2637785006914098, "nauc_recall_at_3_diff1": 0.3849361371197937, "nauc_recall_at_3_max": 0.1359168382947383, "nauc_recall_at_3_std": -0.31194448337435726, "nauc_recall_at_5_diff1": 0.3844633841942806, "nauc_recall_at_5_max": 0.1538609262812138, "nauc_recall_at_5_std": -0.3249312935156017, "ndcg_at_1": 0.2737, "ndcg_at_10": 0.35086, "ndcg_at_100": 0.42802, "ndcg_at_1000": 0.45685, "ndcg_at_20": 0.37512, "ndcg_at_3": 0.29972, "ndcg_at_5": 0.33117, "precision_at_1": 0.2737, "precision_at_10": 0.07125, "precision_at_100": 0.01257, "precision_at_1000": 0.00156, "precision_at_20": 0.04228, "precision_at_3": 0.16718, "precision_at_5": 0.12416, "recall_at_1": 0.19131, "recall_at_10": 0.4476, "recall_at_100": 0.7923, "recall_at_1000": 1.0, "recall_at_20": 0.53259, "recall_at_3": 0.31814, "recall_at_5": 0.39422 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.36377, "map_at_1": 0.17026, "map_at_10": 0.30586, "map_at_100": 0.32484, "map_at_1000": 0.32634, "map_at_20": 0.31556, "map_at_3": 0.26655, "map_at_5": 0.28822, "mrr_at_1": 0.3012232415902141, "mrr_at_10": 0.3803650308237463, "mrr_at_100": 0.3913429533203558, "mrr_at_1000": 0.39204175444015227, "mrr_at_20": 0.38581026176568256, "mrr_at_3": 0.36009174311926617, "mrr_at_5": 0.3716360856269113, "nauc_map_at_1000_diff1": 0.3905129088274741, "nauc_map_at_1000_max": 0.09151544153933133, "nauc_map_at_1000_std": -0.3897538814357438, "nauc_map_at_100_diff1": 0.3907039036342101, "nauc_map_at_100_max": 0.09073947613087816, "nauc_map_at_100_std": -0.3906553158476923, "nauc_map_at_10_diff1": 0.3899537801863117, "nauc_map_at_10_max": 0.08718295461061652, "nauc_map_at_10_std": -0.3958163496438521, "nauc_map_at_1_diff1": 0.46049287918661735, "nauc_map_at_1_max": 0.0630133239894579, "nauc_map_at_1_std": -0.3497742685415477, "nauc_map_at_20_diff1": 0.3905049564577334, "nauc_map_at_20_max": 0.09162655579285119, "nauc_map_at_20_std": -0.39226698525830733, "nauc_map_at_3_diff1": 0.40906145738501326, "nauc_map_at_3_max": 0.07561830491702498, "nauc_map_at_3_std": -0.3918087853272699, "nauc_map_at_5_diff1": 0.3993190233282131, "nauc_map_at_5_max": 0.08423375728472689, "nauc_map_at_5_std": -0.39102988017824986, "nauc_mrr_at_1000_diff1": 0.35249685487189764, "nauc_mrr_at_1000_max": 0.06464677251048417, "nauc_mrr_at_1000_std": -0.3958787711297941, "nauc_mrr_at_100_diff1": 0.35241828188347774, "nauc_mrr_at_100_max": 0.0647195675869247, "nauc_mrr_at_100_std": -0.3956532654295093, "nauc_mrr_at_10_diff1": 0.35029761789993663, "nauc_mrr_at_10_max": 0.06085785807185011, "nauc_mrr_at_10_std": -0.4025441401674854, "nauc_mrr_at_1_diff1": 0.4103962669017795, "nauc_mrr_at_1_max": 0.09122270133294437, "nauc_mrr_at_1_std": -0.397300936878487, "nauc_mrr_at_20_diff1": 0.35087000183203443, "nauc_mrr_at_20_max": 0.06466336598668623, "nauc_mrr_at_20_std": -0.396970413603308, "nauc_mrr_at_3_diff1": 0.36002736433393084, "nauc_mrr_at_3_max": 0.058919259461077185, "nauc_mrr_at_3_std": -0.40384169099115996, "nauc_mrr_at_5_diff1": 0.35202572569014046, "nauc_mrr_at_5_max": 0.05923611779394499, "nauc_mrr_at_5_std": -0.4032668057214314, "nauc_ndcg_at_1000_diff1": 0.3629001370839508, "nauc_ndcg_at_1000_max": 0.09126996346619579, "nauc_ndcg_at_1000_std": -0.37359818020832514, "nauc_ndcg_at_100_diff1": 0.3657815332529199, "nauc_ndcg_at_100_max": 0.08821375764553004, "nauc_ndcg_at_100_std": -0.3717296673154076, "nauc_ndcg_at_10_diff1": 0.35722238704057396, "nauc_ndcg_at_10_max": 0.07119677725654398, "nauc_ndcg_at_10_std": -0.4031827765071559, "nauc_ndcg_at_1_diff1": 0.4103962669017795, "nauc_ndcg_at_1_max": 0.09122270133294437, "nauc_ndcg_at_1_std": -0.397300936878487, "nauc_ndcg_at_20_diff1": 0.3572772000121436, "nauc_ndcg_at_20_max": 0.08467338613153831, "nauc_ndcg_at_20_std": -0.38846316389609437, "nauc_ndcg_at_3_diff1": 0.38032648713998807, "nauc_ndcg_at_3_max": 0.06941830849493003, "nauc_ndcg_at_3_std": -0.40083307028190324, "nauc_ndcg_at_5_diff1": 0.3706386370369462, "nauc_ndcg_at_5_max": 0.07058132504063835, "nauc_ndcg_at_5_std": -0.39576709249460673, "nauc_precision_at_1000_diff1": -0.05655782430294636, "nauc_precision_at_1000_max": 0.13439956993472651, "nauc_precision_at_1000_std": 0.03825648662313364, "nauc_precision_at_100_diff1": 0.05320036231388014, "nauc_precision_at_100_max": 0.12599490994320076, "nauc_precision_at_100_std": -0.06557644002992093, "nauc_precision_at_10_diff1": 0.15543747030263136, "nauc_precision_at_10_max": 0.09143402417922405, "nauc_precision_at_10_std": -0.2973547392633464, "nauc_precision_at_1_diff1": 0.4103962669017795, "nauc_precision_at_1_max": 0.09122270133294437, "nauc_precision_at_1_std": -0.397300936878487, "nauc_precision_at_20_diff1": 0.13970539353492695, "nauc_precision_at_20_max": 0.11306823178518817, "nauc_precision_at_20_std": -0.23946049278152726, "nauc_precision_at_3_diff1": 0.25118756576971546, "nauc_precision_at_3_max": 0.08726898876587896, "nauc_precision_at_3_std": -0.35813694351384023, "nauc_precision_at_5_diff1": 0.20874812786297975, "nauc_precision_at_5_max": 0.09592288168702671, "nauc_precision_at_5_std": -0.32413314057359777, "nauc_recall_at_1000_diff1": 0.5377292301225426, "nauc_recall_at_1000_max": 0.08617668209469644, "nauc_recall_at_1000_std": -0.6137837304264496, "nauc_recall_at_100_diff1": 0.33559922640241197, "nauc_recall_at_100_max": 0.09917249026842827, "nauc_recall_at_100_std": -0.2242024437567541, "nauc_recall_at_10_diff1": 0.2863014008696317, "nauc_recall_at_10_max": 0.041133587343860675, "nauc_recall_at_10_std": -0.3917937947225005, "nauc_recall_at_1_diff1": 0.46049287918661735, "nauc_recall_at_1_max": 0.0630133239894579, "nauc_recall_at_1_std": -0.3497742685415477, "nauc_recall_at_20_diff1": 0.27946785579178257, "nauc_recall_at_20_max": 0.08117566142227187, "nauc_recall_at_20_std": -0.3406835436160669, "nauc_recall_at_3_diff1": 0.3456054830033666, "nauc_recall_at_3_max": 0.035682358079581346, "nauc_recall_at_3_std": -0.3777249942139196, "nauc_recall_at_5_diff1": 0.3131933271355397, "nauc_recall_at_5_max": 0.04456728949924791, "nauc_recall_at_5_std": -0.3693165343322579, "ndcg_at_1": 0.30122, "ndcg_at_10": 0.36377, "ndcg_at_100": 0.44631, "ndcg_at_1000": 0.47442, "ndcg_at_20": 0.3904, "ndcg_at_3": 0.32092, "ndcg_at_5": 0.33334, "precision_at_1": 0.30122, "precision_at_10": 0.0945, "precision_at_100": 0.01602, "precision_at_1000": 0.00194, "precision_at_20": 0.05627, "precision_at_3": 0.21254, "precision_at_5": 0.15413, "recall_at_1": 0.17026, "recall_at_10": 0.45178, "recall_at_100": 0.80499, "recall_at_1000": 0.99554, "recall_at_20": 0.53833, "recall_at_3": 0.32067, "recall_at_5": 0.37704 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.30566, "map_at_1": 0.14889, "map_at_10": 0.2533, "map_at_100": 0.26858, "map_at_1000": 0.27012, "map_at_20": 0.2598, "map_at_3": 0.21998, "map_at_5": 0.23516, "mrr_at_1": 0.21661237785016288, "mrr_at_10": 0.29075345121762064, "mrr_at_100": 0.302032412104649, "mrr_at_1000": 0.3029382678911658, "mrr_at_20": 0.2959832386412645, "mrr_at_3": 0.26628664495114013, "mrr_at_5": 0.27923452768729645, "nauc_map_at_1000_diff1": 0.2754364492779979, "nauc_map_at_1000_max": 0.14501680207074708, "nauc_map_at_1000_std": -0.2045503695323753, "nauc_map_at_100_diff1": 0.27468520019967235, "nauc_map_at_100_max": 0.1449756191021319, "nauc_map_at_100_std": -0.2041052774261539, "nauc_map_at_10_diff1": 0.28147767660133116, "nauc_map_at_10_max": 0.14025745921071164, "nauc_map_at_10_std": -0.21372035506613338, "nauc_map_at_1_diff1": 0.37355624071723426, "nauc_map_at_1_max": 0.15577040920214252, "nauc_map_at_1_std": -0.17169604535252767, "nauc_map_at_20_diff1": 0.27836961510751596, "nauc_map_at_20_max": 0.14085538102145206, "nauc_map_at_20_std": -0.21238875224210338, "nauc_map_at_3_diff1": 0.30405261799344346, "nauc_map_at_3_max": 0.15057309668512775, "nauc_map_at_3_std": -0.20396464056220748, "nauc_map_at_5_diff1": 0.28049454701333115, "nauc_map_at_5_max": 0.14003192566298497, "nauc_map_at_5_std": -0.21110639749549373, "nauc_mrr_at_1000_diff1": 0.2984814662606854, "nauc_mrr_at_1000_max": 0.16666015956025307, "nauc_mrr_at_1000_std": -0.18122636527094793, "nauc_mrr_at_100_diff1": 0.29792570356379233, "nauc_mrr_at_100_max": 0.166961439195054, "nauc_mrr_at_100_std": -0.18041092111235443, "nauc_mrr_at_10_diff1": 0.30116251421854034, "nauc_mrr_at_10_max": 0.1637199501267214, "nauc_mrr_at_10_std": -0.1874035341015473, "nauc_mrr_at_1_diff1": 0.33740901044062355, "nauc_mrr_at_1_max": 0.1999944810544008, "nauc_mrr_at_1_std": -0.1596080201600802, "nauc_mrr_at_20_diff1": 0.3004351087223235, "nauc_mrr_at_20_max": 0.16574095356682053, "nauc_mrr_at_20_std": -0.18385409738665423, "nauc_mrr_at_3_diff1": 0.3145750231821114, "nauc_mrr_at_3_max": 0.16624498810306568, "nauc_mrr_at_3_std": -0.19056963230635668, "nauc_mrr_at_5_diff1": 0.3031707905169204, "nauc_mrr_at_5_max": 0.15864107353931567, "nauc_mrr_at_5_std": -0.1917669677434899, "nauc_ndcg_at_1000_diff1": 0.2638915547403895, "nauc_ndcg_at_1000_max": 0.15893667003710962, "nauc_ndcg_at_1000_std": -0.17555103553711757, "nauc_ndcg_at_100_diff1": 0.2444918613900452, "nauc_ndcg_at_100_max": 0.16317746440265982, "nauc_ndcg_at_100_std": -0.1534943916878785, "nauc_ndcg_at_10_diff1": 0.27135860426735636, "nauc_ndcg_at_10_max": 0.1336012494083595, "nauc_ndcg_at_10_std": -0.21470931348310615, "nauc_ndcg_at_1_diff1": 0.33740901044062355, "nauc_ndcg_at_1_max": 0.1999944810544008, "nauc_ndcg_at_1_std": -0.1596080201600802, "nauc_ndcg_at_20_diff1": 0.26369265299606637, "nauc_ndcg_at_20_max": 0.13752019711062136, "nauc_ndcg_at_20_std": -0.20626370482023434, "nauc_ndcg_at_3_diff1": 0.29881336605653613, "nauc_ndcg_at_3_max": 0.1482670195777456, "nauc_ndcg_at_3_std": -0.2040467634245727, "nauc_ndcg_at_5_diff1": 0.26977399346089515, "nauc_ndcg_at_5_max": 0.12880930683718117, "nauc_ndcg_at_5_std": -0.21464303645223115, "nauc_precision_at_1000_diff1": -0.035880393010250144, "nauc_precision_at_1000_max": 0.1872749399058316, "nauc_precision_at_1000_std": 0.183770376736487, "nauc_precision_at_100_diff1": -0.0090438550124689, "nauc_precision_at_100_max": 0.21951059823863653, "nauc_precision_at_100_std": 0.1654453563845961, "nauc_precision_at_10_diff1": 0.12395431875178056, "nauc_precision_at_10_max": 0.10197447301235081, "nauc_precision_at_10_std": -0.16913261463696172, "nauc_precision_at_1_diff1": 0.33740901044062355, "nauc_precision_at_1_max": 0.1999944810544008, "nauc_precision_at_1_std": -0.1596080201600802, "nauc_precision_at_20_diff1": 0.10141506216433606, "nauc_precision_at_20_max": 0.11950733862274829, "nauc_precision_at_20_std": -0.1242682898861257, "nauc_precision_at_3_diff1": 0.19338903376280286, "nauc_precision_at_3_max": 0.1517632901942953, "nauc_precision_at_3_std": -0.18603356922253142, "nauc_precision_at_5_diff1": 0.13213350660356132, "nauc_precision_at_5_max": 0.11840418939206036, "nauc_precision_at_5_std": -0.19022391159285162, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.047343750173015633, "nauc_recall_at_100_max": 0.19953634957126565, "nauc_recall_at_100_std": 0.07688256827536674, "nauc_recall_at_10_diff1": 0.22177387526628714, "nauc_recall_at_10_max": 0.094492493751116, "nauc_recall_at_10_std": -0.22657379096798275, "nauc_recall_at_1_diff1": 0.37355624071723426, "nauc_recall_at_1_max": 0.15577040920214252, "nauc_recall_at_1_std": -0.17169604535252767, "nauc_recall_at_20_diff1": 0.19755904312802172, "nauc_recall_at_20_max": 0.09992574025954884, "nauc_recall_at_20_std": -0.2072258063780811, "nauc_recall_at_3_diff1": 0.2748226355661856, "nauc_recall_at_3_max": 0.11562233256202345, "nauc_recall_at_3_std": -0.21550214097011122, "nauc_recall_at_5_diff1": 0.2133193695719311, "nauc_recall_at_5_max": 0.07666242316327561, "nauc_recall_at_5_std": -0.22846840896747536, "ndcg_at_1": 0.21661, "ndcg_at_10": 0.30566, "ndcg_at_100": 0.38118, "ndcg_at_1000": 0.41551, "ndcg_at_20": 0.32625, "ndcg_at_3": 0.24937, "ndcg_at_5": 0.26993, "precision_at_1": 0.21661, "precision_at_10": 0.06954, "precision_at_100": 0.01288, "precision_at_1000": 0.00167, "precision_at_20": 0.04112, "precision_at_3": 0.14495, "precision_at_5": 0.10456, "recall_at_1": 0.14889, "recall_at_10": 0.41997, "recall_at_100": 0.75291, "recall_at_1000": 1.0, "recall_at_20": 0.4909, "recall_at_3": 0.27454, "recall_at_5": 0.32448 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/model_meta.json ================================================ {"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "loader": null} ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "task_name": "AutoRAGRetrieval", "mteb_version": "1.29.16", "scores": { "test": [ { "ndcg_at_1": 0.65789, "ndcg_at_3": 0.78801, "ndcg_at_5": 0.79556, "ndcg_at_10": 0.80682, "ndcg_at_20": 0.81613, "ndcg_at_100": 0.82083, "ndcg_at_1000": 0.82083, "map_at_1": 0.65789, "map_at_3": 0.75439, "map_at_5": 0.75877, "map_at_10": 0.76335, "map_at_20": 0.76614, "map_at_100": 0.76671, "map_at_1000": 0.76671, "recall_at_1": 0.65789, "recall_at_3": 0.88596, "recall_at_5": 0.90351, "recall_at_10": 0.9386, "recall_at_20": 0.97368, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.65789, "precision_at_3": 0.29532, "precision_at_5": 0.1807, "precision_at_10": 0.09386, "precision_at_20": 0.04868, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.657895, "mrr_at_3": 0.754386, "mrr_at_5": 0.758772, "mrr_at_10": 0.763349, "mrr_at_20": 0.766137, "mrr_at_100": 0.766708, "mrr_at_1000": 0.766708, "nauc_ndcg_at_1_max": 0.144952, "nauc_ndcg_at_1_std": -0.468953, "nauc_ndcg_at_1_diff1": 0.703742, "nauc_ndcg_at_3_max": 0.174457, "nauc_ndcg_at_3_std": -0.609373, "nauc_ndcg_at_3_diff1": 0.661707, "nauc_ndcg_at_5_max": 0.14815, "nauc_ndcg_at_5_std": -0.652516, "nauc_ndcg_at_5_diff1": 0.671674, "nauc_ndcg_at_10_max": 0.19037, "nauc_ndcg_at_10_std": -0.639461, "nauc_ndcg_at_10_diff1": 0.681606, "nauc_ndcg_at_20_max": 0.177141, "nauc_ndcg_at_20_std": -0.629436, "nauc_ndcg_at_20_diff1": 0.684969, "nauc_ndcg_at_100_max": 0.161606, "nauc_ndcg_at_100_std": -0.601734, "nauc_ndcg_at_100_diff1": 0.684327, "nauc_ndcg_at_1000_max": 0.161606, "nauc_ndcg_at_1000_std": -0.601734, "nauc_ndcg_at_1000_diff1": 0.684327, "nauc_map_at_1_max": 0.144952, "nauc_map_at_1_std": -0.468953, "nauc_map_at_1_diff1": 0.703742, "nauc_map_at_3_max": 0.15807, "nauc_map_at_3_std": -0.570541, "nauc_map_at_3_diff1": 0.675715, "nauc_map_at_5_max": 0.144572, "nauc_map_at_5_std": -0.591471, "nauc_map_at_5_diff1": 0.680973, "nauc_map_at_10_max": 0.158889, "nauc_map_at_10_std": -0.586532, "nauc_map_at_10_diff1": 0.68437, "nauc_map_at_20_max": 0.155476, "nauc_map_at_20_std": -0.583429, "nauc_map_at_20_diff1": 0.685566, "nauc_map_at_100_max": 0.154046, "nauc_map_at_100_std": -0.580807, "nauc_map_at_100_diff1": 0.68558, "nauc_map_at_1000_max": 0.154046, "nauc_map_at_1000_std": -0.580807, "nauc_map_at_1000_diff1": 0.68558, "nauc_recall_at_1_max": 0.144952, "nauc_recall_at_1_std": -0.468953, "nauc_recall_at_1_diff1": 0.703742, "nauc_recall_at_3_max": 0.269469, "nauc_recall_at_3_std": -0.816798, "nauc_recall_at_3_diff1": 0.584712, "nauc_recall_at_5_max": 0.170742, "nauc_recall_at_5_std": -1.036153, "nauc_recall_at_5_diff1": 0.615636, "nauc_recall_at_10_max": 0.54137, "nauc_recall_at_10_std": -1.128869, "nauc_recall_at_10_diff1": 0.671434, "nauc_recall_at_20_max": 0.699035, "nauc_recall_at_20_std": -1.553327, "nauc_recall_at_20_diff1": 0.715652, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.144952, "nauc_precision_at_1_std": -0.468953, "nauc_precision_at_1_diff1": 0.703742, "nauc_precision_at_3_max": 0.269469, "nauc_precision_at_3_std": -0.816798, "nauc_precision_at_3_diff1": 0.584712, "nauc_precision_at_5_max": 0.170742, "nauc_precision_at_5_std": -1.036153, "nauc_precision_at_5_diff1": 0.615636, "nauc_precision_at_10_max": 0.54137, "nauc_precision_at_10_std": -1.128869, "nauc_precision_at_10_diff1": 0.671434, "nauc_precision_at_20_max": 0.699035, "nauc_precision_at_20_std": -1.553327, "nauc_precision_at_20_diff1": 0.715652, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_1000_diff1": 1.0, "nauc_mrr_at_1_max": 0.144952, "nauc_mrr_at_1_std": -0.468953, "nauc_mrr_at_1_diff1": 0.703742, "nauc_mrr_at_3_max": 0.15807, "nauc_mrr_at_3_std": -0.570541, "nauc_mrr_at_3_diff1": 0.675715, "nauc_mrr_at_5_max": 0.144572, "nauc_mrr_at_5_std": -0.591471, "nauc_mrr_at_5_diff1": 0.680973, "nauc_mrr_at_10_max": 0.158889, "nauc_mrr_at_10_std": -0.586532, "nauc_mrr_at_10_diff1": 0.68437, "nauc_mrr_at_20_max": 0.155476, "nauc_mrr_at_20_std": -0.583429, "nauc_mrr_at_20_diff1": 0.685566, "nauc_mrr_at_100_max": 0.154046, "nauc_mrr_at_100_std": -0.580807, "nauc_mrr_at_100_diff1": 0.68558, "nauc_mrr_at_1000_max": 0.154046, "nauc_mrr_at_1000_std": -0.580807, "nauc_mrr_at_1000_diff1": 0.68558, "main_score": 0.80682, "hf_subset": "default", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 10.152276277542114, "kg_co2_emissions": null } ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "task_name": "BelebeleRetrieval", "mteb_version": "1.29.16", "scores": { "test": [ { "ndcg_at_1": 0.9, "ndcg_at_3": 0.92729, "ndcg_at_5": 0.93236, "ndcg_at_10": 0.93636, "ndcg_at_20": 0.93941, "ndcg_at_100": 0.94156, "ndcg_at_1000": 0.94246, "map_at_1": 0.9, "map_at_3": 0.92093, "map_at_5": 0.92376, "map_at_10": 0.92544, "map_at_20": 0.92625, "map_at_100": 0.92659, "map_at_1000": 0.92663, "recall_at_1": 0.9, "recall_at_3": 0.94556, "recall_at_5": 0.95778, "recall_at_10": 0.97, "recall_at_20": 0.98222, "recall_at_100": 0.99333, "recall_at_1000": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.31519, "precision_at_5": 0.19156, "precision_at_10": 0.097, "precision_at_20": 0.04911, "precision_at_100": 0.00993, "precision_at_1000": 0.001, "mrr_at_1": 0.9, "mrr_at_3": 0.920926, "mrr_at_5": 0.923759, "mrr_at_10": 0.925442, "mrr_at_20": 0.926254, "mrr_at_100": 0.926592, "mrr_at_1000": 0.926632, "nauc_ndcg_at_1_max": 0.849813, "nauc_ndcg_at_1_std": 0.119157, "nauc_ndcg_at_1_diff1": 0.900093, "nauc_ndcg_at_3_max": 0.872149, "nauc_ndcg_at_3_std": 0.075853, "nauc_ndcg_at_3_diff1": 0.912054, "nauc_ndcg_at_5_max": 0.870113, "nauc_ndcg_at_5_std": 0.096455, "nauc_ndcg_at_5_diff1": 0.907313, "nauc_ndcg_at_10_max": 0.865967, "nauc_ndcg_at_10_std": 0.086424, "nauc_ndcg_at_10_diff1": 0.908127, "nauc_ndcg_at_20_max": 0.865356, "nauc_ndcg_at_20_std": 0.083029, "nauc_ndcg_at_20_diff1": 0.910442, "nauc_ndcg_at_100_max": 0.86684, "nauc_ndcg_at_100_std": 0.105116, "nauc_ndcg_at_100_diff1": 0.909392, "nauc_ndcg_at_1000_max": 0.86514, "nauc_ndcg_at_1000_std": 0.099406, "nauc_ndcg_at_1000_diff1": 0.908351, "nauc_map_at_1_max": 0.849813, "nauc_map_at_1_std": 0.119157, "nauc_map_at_1_diff1": 0.900093, "nauc_map_at_3_max": 0.866128, "nauc_map_at_3_std": 0.090474, "nauc_map_at_3_diff1": 0.908821, "nauc_map_at_5_max": 0.864939, "nauc_map_at_5_std": 0.101507, "nauc_map_at_5_diff1": 0.906386, "nauc_map_at_10_max": 0.863324, "nauc_map_at_10_std": 0.097936, "nauc_map_at_10_diff1": 0.906683, "nauc_map_at_20_max": 0.863212, "nauc_map_at_20_std": 0.097465, "nauc_map_at_20_diff1": 0.907289, "nauc_map_at_100_max": 0.863306, "nauc_map_at_100_std": 0.100578, "nauc_map_at_100_diff1": 0.907099, "nauc_map_at_1000_max": 0.863247, "nauc_map_at_1000_std": 0.100452, "nauc_map_at_1000_diff1": 0.907064, "nauc_recall_at_1_max": 0.849813, "nauc_recall_at_1_std": 0.119157, "nauc_recall_at_1_diff1": 0.900093, "nauc_recall_at_3_max": 0.896911, "nauc_recall_at_3_std": 0.01372, "nauc_recall_at_3_diff1": 0.925351, "nauc_recall_at_5_max": 0.895879, "nauc_recall_at_5_std": 0.073284, "nauc_recall_at_5_diff1": 0.910622, "nauc_recall_at_10_max": 0.879656, "nauc_recall_at_10_std": -5.2e-05, "nauc_recall_at_10_diff1": 0.916917, "nauc_recall_at_20_max": 0.87891, "nauc_recall_at_20_std": -0.111578, "nauc_recall_at_20_diff1": 0.95098, "nauc_recall_at_100_max": 0.978214, "nauc_recall_at_100_std": 0.512605, "nauc_recall_at_100_diff1": 0.978214, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.849813, "nauc_precision_at_1_std": 0.119157, "nauc_precision_at_1_diff1": 0.900093, "nauc_precision_at_3_max": 0.896911, "nauc_precision_at_3_std": 0.01372, "nauc_precision_at_3_diff1": 0.925351, "nauc_precision_at_5_max": 0.895879, "nauc_precision_at_5_std": 0.073284, "nauc_precision_at_5_diff1": 0.910622, "nauc_precision_at_10_max": 0.879656, "nauc_precision_at_10_std": -5.2e-05, "nauc_precision_at_10_diff1": 0.916917, "nauc_precision_at_20_max": 0.87891, "nauc_precision_at_20_std": -0.111578, "nauc_precision_at_20_diff1": 0.95098, "nauc_precision_at_100_max": 0.978214, "nauc_precision_at_100_std": 0.512605, "nauc_precision_at_100_diff1": 0.978214, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.849813, "nauc_mrr_at_1_std": 0.119157, "nauc_mrr_at_1_diff1": 0.900093, "nauc_mrr_at_3_max": 0.866128, "nauc_mrr_at_3_std": 0.090474, "nauc_mrr_at_3_diff1": 0.908821, "nauc_mrr_at_5_max": 0.864939, "nauc_mrr_at_5_std": 0.101507, "nauc_mrr_at_5_diff1": 0.906386, "nauc_mrr_at_10_max": 0.863324, "nauc_mrr_at_10_std": 0.097936, "nauc_mrr_at_10_diff1": 0.906683, "nauc_mrr_at_20_max": 0.863212, "nauc_mrr_at_20_std": 0.097465, "nauc_mrr_at_20_diff1": 0.907289, "nauc_mrr_at_100_max": 0.863306, "nauc_mrr_at_100_std": 0.100578, "nauc_mrr_at_100_diff1": 0.907099, "nauc_mrr_at_1000_max": 0.863247, "nauc_mrr_at_1000_std": 0.100452, "nauc_mrr_at_1000_diff1": 0.907064, "main_score": 0.93636, "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ] }, { "ndcg_at_1": 0.77444, "ndcg_at_3": 0.83469, "ndcg_at_5": 0.84971, "ndcg_at_10": 0.8598, "ndcg_at_20": 0.8654, "ndcg_at_100": 0.87031, "ndcg_at_1000": 0.87122, "map_at_1": 0.77444, "map_at_3": 0.81981, "map_at_5": 0.82809, "map_at_10": 0.83228, "map_at_20": 0.83381, "map_at_100": 0.83454, "map_at_1000": 0.83459, "recall_at_1": 0.77444, "recall_at_3": 0.87778, "recall_at_5": 0.91444, "recall_at_10": 0.94556, "recall_at_20": 0.96778, "recall_at_100": 0.99333, "recall_at_1000": 1.0, "precision_at_1": 0.77444, "precision_at_3": 0.29259, "precision_at_5": 0.18289, "precision_at_10": 0.09456, "precision_at_20": 0.04839, "precision_at_100": 0.00993, "precision_at_1000": 0.001, "mrr_at_1": 0.774444, "mrr_at_3": 0.819815, "mrr_at_5": 0.828093, "mrr_at_10": 0.832275, "mrr_at_20": 0.833805, "mrr_at_100": 0.834541, "mrr_at_1000": 0.834586, "nauc_ndcg_at_1_max": 0.709926, "nauc_ndcg_at_1_std": -0.063143, "nauc_ndcg_at_1_diff1": 0.863108, "nauc_ndcg_at_3_max": 0.720473, "nauc_ndcg_at_3_std": -0.080147, "nauc_ndcg_at_3_diff1": 0.841613, "nauc_ndcg_at_5_max": 0.723408, "nauc_ndcg_at_5_std": -0.062139, "nauc_ndcg_at_5_diff1": 0.842072, "nauc_ndcg_at_10_max": 0.730953, "nauc_ndcg_at_10_std": -0.019305, "nauc_ndcg_at_10_diff1": 0.845659, "nauc_ndcg_at_20_max": 0.7317, "nauc_ndcg_at_20_std": -0.026528, "nauc_ndcg_at_20_diff1": 0.847865, "nauc_ndcg_at_100_max": 0.726646, "nauc_ndcg_at_100_std": -0.040923, "nauc_ndcg_at_100_diff1": 0.847999, "nauc_ndcg_at_1000_max": 0.725425, "nauc_ndcg_at_1000_std": -0.045003, "nauc_ndcg_at_1000_diff1": 0.848429, "nauc_map_at_1_max": 0.709926, "nauc_map_at_1_std": -0.063143, "nauc_map_at_1_diff1": 0.863108, "nauc_map_at_3_max": 0.719983, "nauc_map_at_3_std": -0.074201, "nauc_map_at_3_diff1": 0.848359, "nauc_map_at_5_max": 0.720959, "nauc_map_at_5_std": -0.065162, "nauc_map_at_5_diff1": 0.848573, "nauc_map_at_10_max": 0.723392, "nauc_map_at_10_std": -0.049768, "nauc_map_at_10_diff1": 0.849886, "nauc_map_at_20_max": 0.723456, "nauc_map_at_20_std": -0.051621, "nauc_map_at_20_diff1": 0.850372, "nauc_map_at_100_max": 0.722814, "nauc_map_at_100_std": -0.0534, "nauc_map_at_100_diff1": 0.850366, "nauc_map_at_1000_max": 0.722767, "nauc_map_at_1000_std": -0.05353, "nauc_map_at_1000_diff1": 0.850381, "nauc_recall_at_1_max": 0.709926, "nauc_recall_at_1_std": -0.063143, "nauc_recall_at_1_diff1": 0.863108, "nauc_recall_at_3_max": 0.720942, "nauc_recall_at_3_std": -0.104817, "nauc_recall_at_3_diff1": 0.813999, "nauc_recall_at_5_max": 0.736325, "nauc_recall_at_5_std": -0.041635, "nauc_recall_at_5_diff1": 0.807505, "nauc_recall_at_10_max": 0.801787, "nauc_recall_at_10_std": 0.280674, "nauc_recall_at_10_diff1": 0.816126, "nauc_recall_at_20_max": 0.863164, "nauc_recall_at_20_std": 0.373161, "nauc_recall_at_20_diff1": 0.831611, "nauc_recall_at_100_max": 0.888344, "nauc_recall_at_100_std": 0.527, "nauc_recall_at_100_diff1": 0.789527, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.709926, "nauc_precision_at_1_std": -0.063143, "nauc_precision_at_1_diff1": 0.863108, "nauc_precision_at_3_max": 0.720942, "nauc_precision_at_3_std": -0.104817, "nauc_precision_at_3_diff1": 0.813999, "nauc_precision_at_5_max": 0.736325, "nauc_precision_at_5_std": -0.041635, "nauc_precision_at_5_diff1": 0.807505, "nauc_precision_at_10_max": 0.801787, "nauc_precision_at_10_std": 0.280674, "nauc_precision_at_10_diff1": 0.816126, "nauc_precision_at_20_max": 0.863164, "nauc_precision_at_20_std": 0.373161, "nauc_precision_at_20_diff1": 0.831611, "nauc_precision_at_100_max": 0.888344, "nauc_precision_at_100_std": 0.527, "nauc_precision_at_100_diff1": 0.789527, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.709926, "nauc_mrr_at_1_std": -0.063143, "nauc_mrr_at_1_diff1": 0.863108, "nauc_mrr_at_3_max": 0.719983, "nauc_mrr_at_3_std": -0.074201, "nauc_mrr_at_3_diff1": 0.848359, "nauc_mrr_at_5_max": 0.720959, "nauc_mrr_at_5_std": -0.065162, "nauc_mrr_at_5_diff1": 0.848573, "nauc_mrr_at_10_max": 0.723392, "nauc_mrr_at_10_std": -0.049768, "nauc_mrr_at_10_diff1": 0.849886, "nauc_mrr_at_20_max": 0.723456, "nauc_mrr_at_20_std": -0.051621, "nauc_mrr_at_20_diff1": 0.850372, "nauc_mrr_at_100_max": 0.722814, "nauc_mrr_at_100_std": -0.0534, "nauc_mrr_at_100_diff1": 0.850366, "nauc_mrr_at_1000_max": 0.722767, "nauc_mrr_at_1000_std": -0.05353, "nauc_mrr_at_1000_diff1": 0.850381, "main_score": 0.8598, "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ] }, { "ndcg_at_1": 0.71, "ndcg_at_3": 0.77028, "ndcg_at_5": 0.78702, "ndcg_at_10": 0.80552, "ndcg_at_20": 0.81313, "ndcg_at_100": 0.82143, "ndcg_at_1000": 0.82368, "map_at_1": 0.71, "map_at_3": 0.75574, "map_at_5": 0.76519, "map_at_10": 0.77293, "map_at_20": 0.77502, "map_at_100": 0.77618, "map_at_1000": 0.77629, "recall_at_1": 0.71, "recall_at_3": 0.81222, "recall_at_5": 0.85222, "recall_at_10": 0.90889, "recall_at_20": 0.93889, "recall_at_100": 0.98333, "recall_at_1000": 1.0, "precision_at_1": 0.71, "precision_at_3": 0.27074, "precision_at_5": 0.17044, "precision_at_10": 0.09089, "precision_at_20": 0.04694, "precision_at_100": 0.00983, "precision_at_1000": 0.001, "mrr_at_1": 0.71, "mrr_at_3": 0.755741, "mrr_at_5": 0.765185, "mrr_at_10": 0.772925, "mrr_at_20": 0.77502, "mrr_at_100": 0.776182, "mrr_at_1000": 0.776288, "nauc_ndcg_at_1_max": 0.741047, "nauc_ndcg_at_1_std": 0.150175, "nauc_ndcg_at_1_diff1": 0.874735, "nauc_ndcg_at_3_max": 0.743184, "nauc_ndcg_at_3_std": 0.158375, "nauc_ndcg_at_3_diff1": 0.848813, "nauc_ndcg_at_5_max": 0.752586, "nauc_ndcg_at_5_std": 0.187284, "nauc_ndcg_at_5_diff1": 0.84275, "nauc_ndcg_at_10_max": 0.746801, "nauc_ndcg_at_10_std": 0.195985, "nauc_ndcg_at_10_diff1": 0.841376, "nauc_ndcg_at_20_max": 0.74242, "nauc_ndcg_at_20_std": 0.18473, "nauc_ndcg_at_20_diff1": 0.842535, "nauc_ndcg_at_100_max": 0.745244, "nauc_ndcg_at_100_std": 0.183086, "nauc_ndcg_at_100_diff1": 0.847303, "nauc_ndcg_at_1000_max": 0.74542, "nauc_ndcg_at_1000_std": 0.178041, "nauc_ndcg_at_1000_diff1": 0.849251, "nauc_map_at_1_max": 0.741047, "nauc_map_at_1_std": 0.150175, "nauc_map_at_1_diff1": 0.874735, "nauc_map_at_3_max": 0.742529, "nauc_map_at_3_std": 0.15468, "nauc_map_at_3_diff1": 0.855357, "nauc_map_at_5_max": 0.747374, "nauc_map_at_5_std": 0.169622, "nauc_map_at_5_diff1": 0.852636, "nauc_map_at_10_max": 0.745154, "nauc_map_at_10_std": 0.17275, "nauc_map_at_10_diff1": 0.852492, "nauc_map_at_20_max": 0.744114, "nauc_map_at_20_std": 0.169849, "nauc_map_at_20_diff1": 0.852877, "nauc_map_at_100_max": 0.744707, "nauc_map_at_100_std": 0.169854, "nauc_map_at_100_diff1": 0.85356, "nauc_map_at_1000_max": 0.744725, "nauc_map_at_1000_std": 0.169696, "nauc_map_at_1000_diff1": 0.853632, "nauc_recall_at_1_max": 0.741047, "nauc_recall_at_1_std": 0.150175, "nauc_recall_at_1_diff1": 0.874735, "nauc_recall_at_3_max": 0.745556, "nauc_recall_at_3_std": 0.1723, "nauc_recall_at_3_diff1": 0.825776, "nauc_recall_at_5_max": 0.776768, "nauc_recall_at_5_std": 0.269229, "nauc_recall_at_5_diff1": 0.799352, "nauc_recall_at_10_max": 0.755352, "nauc_recall_at_10_std": 0.360046, "nauc_recall_at_10_diff1": 0.766904, "nauc_recall_at_20_max": 0.711858, "nauc_recall_at_20_std": 0.318852, "nauc_recall_at_20_diff1": 0.742917, "nauc_recall_at_100_max": 0.73906, "nauc_recall_at_100_std": 0.555836, "nauc_recall_at_100_diff1": 0.709679, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.741047, "nauc_precision_at_1_std": 0.150175, "nauc_precision_at_1_diff1": 0.874735, "nauc_precision_at_3_max": 0.745556, "nauc_precision_at_3_std": 0.1723, "nauc_precision_at_3_diff1": 0.825776, "nauc_precision_at_5_max": 0.776768, "nauc_precision_at_5_std": 0.269229, "nauc_precision_at_5_diff1": 0.799352, "nauc_precision_at_10_max": 0.755352, "nauc_precision_at_10_std": 0.360046, "nauc_precision_at_10_diff1": 0.766904, "nauc_precision_at_20_max": 0.711858, "nauc_precision_at_20_std": 0.318852, "nauc_precision_at_20_diff1": 0.742917, "nauc_precision_at_100_max": 0.73906, "nauc_precision_at_100_std": 0.555836, "nauc_precision_at_100_diff1": 0.709679, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.741047, "nauc_mrr_at_1_std": 0.150175, "nauc_mrr_at_1_diff1": 0.874735, "nauc_mrr_at_3_max": 0.742529, "nauc_mrr_at_3_std": 0.15468, "nauc_mrr_at_3_diff1": 0.855357, "nauc_mrr_at_5_max": 0.747374, "nauc_mrr_at_5_std": 0.169622, "nauc_mrr_at_5_diff1": 0.852636, "nauc_mrr_at_10_max": 0.745154, "nauc_mrr_at_10_std": 0.17275, "nauc_mrr_at_10_diff1": 0.852492, "nauc_mrr_at_20_max": 0.744114, "nauc_mrr_at_20_std": 0.169849, "nauc_mrr_at_20_diff1": 0.852877, "nauc_mrr_at_100_max": 0.744707, "nauc_mrr_at_100_std": 0.169854, "nauc_mrr_at_100_diff1": 0.85356, "nauc_mrr_at_1000_max": 0.744725, "nauc_mrr_at_1000_std": 0.169696, "nauc_mrr_at_1000_diff1": 0.853632, "main_score": 0.80552, "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ] } ] }, "evaluation_time": 19.255128145217896, "kg_co2_emissions": null } ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "task_name": "Ko-StrategyQA", "mteb_version": "1.29.16", "scores": { "dev": [ { "ndcg_at_1": 0.71453, "ndcg_at_3": 0.7184, "ndcg_at_5": 0.74457, "ndcg_at_10": 0.76325, "ndcg_at_20": 0.77435, "ndcg_at_100": 0.78386, "ndcg_at_1000": 0.79186, "map_at_1": 0.46761, "map_at_3": 0.67454, "map_at_5": 0.70272, "map_at_10": 0.71519, "map_at_20": 0.71987, "map_at_100": 0.72181, "map_at_1000": 0.72217, "recall_at_1": 0.46761, "recall_at_3": 0.72928, "recall_at_5": 0.78289, "recall_at_10": 0.82686, "recall_at_20": 0.86076, "recall_at_100": 0.90259, "recall_at_1000": 0.9544, "precision_at_1": 0.71453, "precision_at_3": 0.42736, "precision_at_5": 0.28378, "precision_at_10": 0.15253, "precision_at_20": 0.08015, "precision_at_100": 0.01691, "precision_at_1000": 0.00181, "mrr_at_1": 0.714527, "mrr_at_3": 0.774775, "mrr_at_5": 0.777477, "mrr_at_10": 0.780593, "mrr_at_20": 0.782165, "mrr_at_100": 0.782891, "mrr_at_1000": 0.783037, "nauc_ndcg_at_1_max": 0.657745, "nauc_ndcg_at_1_std": 0.24173, "nauc_ndcg_at_1_diff1": 0.689846, "nauc_ndcg_at_3_max": 0.639985, "nauc_ndcg_at_3_std": 0.223257, "nauc_ndcg_at_3_diff1": 0.576491, "nauc_ndcg_at_5_max": 0.677781, "nauc_ndcg_at_5_std": 0.266772, "nauc_ndcg_at_5_diff1": 0.564678, "nauc_ndcg_at_10_max": 0.68748, "nauc_ndcg_at_10_std": 0.301639, "nauc_ndcg_at_10_diff1": 0.571674, "nauc_ndcg_at_20_max": 0.705281, "nauc_ndcg_at_20_std": 0.324945, "nauc_ndcg_at_20_diff1": 0.580892, "nauc_ndcg_at_100_max": 0.704981, "nauc_ndcg_at_100_std": 0.342294, "nauc_ndcg_at_100_diff1": 0.578293, "nauc_ndcg_at_1000_max": 0.699871, "nauc_ndcg_at_1000_std": 0.326877, "nauc_ndcg_at_1000_diff1": 0.582325, "nauc_map_at_1_max": 0.321776, "nauc_map_at_1_std": 0.051511, "nauc_map_at_1_diff1": 0.615717, "nauc_map_at_3_max": 0.591045, "nauc_map_at_3_std": 0.169109, "nauc_map_at_3_diff1": 0.566227, "nauc_map_at_5_max": 0.628691, "nauc_map_at_5_std": 0.209505, "nauc_map_at_5_diff1": 0.554182, "nauc_map_at_10_max": 0.634033, "nauc_map_at_10_std": 0.228864, "nauc_map_at_10_diff1": 0.558702, "nauc_map_at_20_max": 0.641918, "nauc_map_at_20_std": 0.237956, "nauc_map_at_20_diff1": 0.561968, "nauc_map_at_100_max": 0.642257, "nauc_map_at_100_std": 0.241074, "nauc_map_at_100_diff1": 0.561667, "nauc_map_at_1000_max": 0.642144, "nauc_map_at_1000_std": 0.240601, "nauc_map_at_1000_diff1": 0.561874, "nauc_recall_at_1_max": 0.321776, "nauc_recall_at_1_std": 0.051511, "nauc_recall_at_1_diff1": 0.615717, "nauc_recall_at_3_max": 0.61726, "nauc_recall_at_3_std": 0.21254, "nauc_recall_at_3_diff1": 0.513592, "nauc_recall_at_5_max": 0.700999, "nauc_recall_at_5_std": 0.30856, "nauc_recall_at_5_diff1": 0.484074, "nauc_recall_at_10_max": 0.736903, "nauc_recall_at_10_std": 0.419002, "nauc_recall_at_10_diff1": 0.498531, "nauc_recall_at_20_max": 0.822719, "nauc_recall_at_20_std": 0.540586, "nauc_recall_at_20_diff1": 0.529592, "nauc_recall_at_100_max": 0.862766, "nauc_recall_at_100_std": 0.775853, "nauc_recall_at_100_diff1": 0.482599, "nauc_recall_at_1000_max": 0.918329, "nauc_recall_at_1000_std": 0.846873, "nauc_recall_at_1000_diff1": 0.480587, "nauc_precision_at_1_max": 0.657745, "nauc_precision_at_1_std": 0.24173, "nauc_precision_at_1_diff1": 0.689846, "nauc_precision_at_3_max": 0.482042, "nauc_precision_at_3_std": 0.267328, "nauc_precision_at_3_diff1": 0.057941, "nauc_precision_at_5_max": 0.405916, "nauc_precision_at_5_std": 0.275096, "nauc_precision_at_5_diff1": -0.036951, "nauc_precision_at_10_max": 0.33671, "nauc_precision_at_10_std": 0.294959, "nauc_precision_at_10_diff1": -0.091127, "nauc_precision_at_20_max": 0.314331, "nauc_precision_at_20_std": 0.316038, "nauc_precision_at_20_diff1": -0.124284, "nauc_precision_at_100_max": 0.23692, "nauc_precision_at_100_std": 0.318947, "nauc_precision_at_100_diff1": -0.191179, "nauc_precision_at_1000_max": 0.107935, "nauc_precision_at_1000_std": 0.212494, "nauc_precision_at_1000_diff1": -0.265746, "nauc_mrr_at_1_max": 0.657745, "nauc_mrr_at_1_std": 0.24173, "nauc_mrr_at_1_diff1": 0.689846, "nauc_mrr_at_3_max": 0.729292, "nauc_mrr_at_3_std": 0.334929, "nauc_mrr_at_3_diff1": 0.661563, "nauc_mrr_at_5_max": 0.731308, "nauc_mrr_at_5_std": 0.337845, "nauc_mrr_at_5_diff1": 0.661708, "nauc_mrr_at_10_max": 0.731278, "nauc_mrr_at_10_std": 0.339699, "nauc_mrr_at_10_diff1": 0.663405, "nauc_mrr_at_20_max": 0.73118, "nauc_mrr_at_20_std": 0.340714, "nauc_mrr_at_20_diff1": 0.664546, "nauc_mrr_at_100_max": 0.730669, "nauc_mrr_at_100_std": 0.340809, "nauc_mrr_at_100_diff1": 0.664665, "nauc_mrr_at_1000_max": 0.730492, "nauc_mrr_at_1000_std": 0.340435, "nauc_mrr_at_1000_diff1": 0.664655, "main_score": 0.76325, "hf_subset": "default", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 57.423654556274414, "kg_co2_emissions": null } ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "task_name": "MIRACLRetrieval", "mteb_version": "1.29.16", "scores": { "dev": [ { "ndcg_at_1": 0.61033, "ndcg_at_3": 0.6176, "ndcg_at_5": 0.6276, "ndcg_at_10": 0.65913, "ndcg_at_20": 0.69053, "ndcg_at_100": 0.72143, "ndcg_at_1000": 0.72654, "map_at_1": 0.38197, "map_at_3": 0.50465, "map_at_5": 0.53735, "map_at_10": 0.56822, "map_at_20": 0.58548, "map_at_100": 0.59466, "map_at_1000": 0.59528, "recall_at_1": 0.38197, "recall_at_3": 0.58499, "recall_at_5": 0.65526, "recall_at_10": 0.75056, "recall_at_20": 0.83774, "recall_at_100": 0.95188, "recall_at_1000": 0.97786, "precision_at_1": 0.61033, "precision_at_3": 0.36776, "precision_at_5": 0.26854, "precision_at_10": 0.16901, "precision_at_20": 0.1, "precision_at_100": 0.02423, "precision_at_1000": 0.00252, "mrr_at_1": 0.610329, "mrr_at_3": 0.711268, "mrr_at_5": 0.718545, "mrr_at_10": 0.72552, "mrr_at_20": 0.727103, "mrr_at_100": 0.727978, "mrr_at_1000": 0.727988, "nauc_ndcg_at_1_max": 0.363021, "nauc_ndcg_at_1_std": -0.042296, "nauc_ndcg_at_1_diff1": 0.492822, "nauc_ndcg_at_3_max": 0.358674, "nauc_ndcg_at_3_std": -0.048445, "nauc_ndcg_at_3_diff1": 0.480832, "nauc_ndcg_at_5_max": 0.369315, "nauc_ndcg_at_5_std": -0.019819, "nauc_ndcg_at_5_diff1": 0.475952, "nauc_ndcg_at_10_max": 0.398008, "nauc_ndcg_at_10_std": -0.008035, "nauc_ndcg_at_10_diff1": 0.487936, "nauc_ndcg_at_20_max": 0.412476, "nauc_ndcg_at_20_std": 0.024583, "nauc_ndcg_at_20_diff1": 0.489312, "nauc_ndcg_at_100_max": 0.412105, "nauc_ndcg_at_100_std": 0.023433, "nauc_ndcg_at_100_diff1": 0.49975, "nauc_ndcg_at_1000_max": 0.413061, "nauc_ndcg_at_1000_std": 0.023001, "nauc_ndcg_at_1000_diff1": 0.497511, "nauc_map_at_1_max": 0.214492, "nauc_map_at_1_std": -0.122707, "nauc_map_at_1_diff1": 0.520518, "nauc_map_at_3_max": 0.296177, "nauc_map_at_3_std": -0.090703, "nauc_map_at_3_diff1": 0.47893, "nauc_map_at_5_max": 0.321156, "nauc_map_at_5_std": -0.053627, "nauc_map_at_5_diff1": 0.470241, "nauc_map_at_10_max": 0.354251, "nauc_map_at_10_std": -0.043471, "nauc_map_at_10_diff1": 0.487821, "nauc_map_at_20_max": 0.364801, "nauc_map_at_20_std": -0.029409, "nauc_map_at_20_diff1": 0.489186, "nauc_map_at_100_max": 0.366346, "nauc_map_at_100_std": -0.028111, "nauc_map_at_100_diff1": 0.491536, "nauc_map_at_1000_max": 0.366562, "nauc_map_at_1000_std": -0.028481, "nauc_map_at_1000_diff1": 0.491601, "nauc_recall_at_1_max": 0.214492, "nauc_recall_at_1_std": -0.122707, "nauc_recall_at_1_diff1": 0.520518, "nauc_recall_at_3_max": 0.26531, "nauc_recall_at_3_std": -0.068894, "nauc_recall_at_3_diff1": 0.409745, "nauc_recall_at_5_max": 0.297952, "nauc_recall_at_5_std": 0.001566, "nauc_recall_at_5_diff1": 0.370023, "nauc_recall_at_10_max": 0.388384, "nauc_recall_at_10_std": 0.0103, "nauc_recall_at_10_diff1": 0.407716, "nauc_recall_at_20_max": 0.440723, "nauc_recall_at_20_std": 0.171722, "nauc_recall_at_20_diff1": 0.378589, "nauc_recall_at_100_max": 0.545394, "nauc_recall_at_100_std": 0.322864, "nauc_recall_at_100_diff1": 0.512669, "nauc_recall_at_1000_max": 0.857152, "nauc_recall_at_1000_std": 0.743431, "nauc_recall_at_1000_diff1": 0.52794, "nauc_precision_at_1_max": 0.363021, "nauc_precision_at_1_std": -0.042296, "nauc_precision_at_1_diff1": 0.492822, "nauc_precision_at_3_max": 0.316402, "nauc_precision_at_3_std": 0.113752, "nauc_precision_at_3_diff1": 0.081769, "nauc_precision_at_5_max": 0.267986, "nauc_precision_at_5_std": 0.178829, "nauc_precision_at_5_diff1": -0.004231, "nauc_precision_at_10_max": 0.241916, "nauc_precision_at_10_std": 0.180147, "nauc_precision_at_10_diff1": -0.030231, "nauc_precision_at_20_max": 0.173029, "nauc_precision_at_20_std": 0.182248, "nauc_precision_at_20_diff1": -0.082592, "nauc_precision_at_100_max": 0.098192, "nauc_precision_at_100_std": 0.14408, "nauc_precision_at_100_diff1": -0.108657, "nauc_precision_at_1000_max": 0.087758, "nauc_precision_at_1000_std": 0.132117, "nauc_precision_at_1000_diff1": -0.116775, "nauc_mrr_at_1_max": 0.363021, "nauc_mrr_at_1_std": -0.042296, "nauc_mrr_at_1_diff1": 0.492822, "nauc_mrr_at_3_max": 0.40441, "nauc_mrr_at_3_std": 0.011383, "nauc_mrr_at_3_diff1": 0.46785, "nauc_mrr_at_5_max": 0.411774, "nauc_mrr_at_5_std": 0.015938, "nauc_mrr_at_5_diff1": 0.47189, "nauc_mrr_at_10_max": 0.407763, "nauc_mrr_at_10_std": 0.012721, "nauc_mrr_at_10_diff1": 0.477408, "nauc_mrr_at_20_max": 0.408228, "nauc_mrr_at_20_std": 0.015341, "nauc_mrr_at_20_diff1": 0.476169, "nauc_mrr_at_100_max": 0.407503, "nauc_mrr_at_100_std": 0.013473, "nauc_mrr_at_100_diff1": 0.47738, "nauc_mrr_at_1000_max": 0.407515, "nauc_mrr_at_1000_std": 0.013498, "nauc_mrr_at_1000_diff1": 0.477357, "main_score": 0.65913, "hf_subset": "ko", "languages": [ "kor-Kore" ] } ] }, "evaluation_time": 5944.149179697037, "kg_co2_emissions": null } ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "task_name": "MrTidyRetrieval", "mteb_version": "1.29.16", "scores": { "test": [ { "ndcg_at_1": 0.41568, "ndcg_at_3": 0.48461, "ndcg_at_5": 0.51236, "ndcg_at_10": 0.53766, "ndcg_at_20": 0.55826, "ndcg_at_100": 0.58428, "ndcg_at_1000": 0.5934, "map_at_1": 0.38717, "map_at_3": 0.45691, "map_at_5": 0.47439, "map_at_10": 0.48529, "map_at_20": 0.49112, "map_at_100": 0.49529, "map_at_1000": 0.49568, "recall_at_1": 0.38717, "recall_at_3": 0.53761, "recall_at_5": 0.60016, "recall_at_10": 0.67379, "recall_at_20": 0.75218, "recall_at_100": 0.88361, "recall_at_1000": 0.95131, "precision_at_1": 0.41568, "precision_at_3": 0.19477, "precision_at_5": 0.13302, "precision_at_10": 0.07577, "precision_at_20": 0.04276, "precision_at_100": 0.01024, "precision_at_1000": 0.00111, "mrr_at_1": 0.415677, "mrr_at_3": 0.485748, "mrr_at_5": 0.498931, "mrr_at_10": 0.509528, "mrr_at_20": 0.514774, "mrr_at_100": 0.517754, "mrr_at_1000": 0.518063, "nauc_ndcg_at_1_max": 0.36563, "nauc_ndcg_at_1_std": 0.021405, "nauc_ndcg_at_1_diff1": 0.545735, "nauc_ndcg_at_3_max": 0.363778, "nauc_ndcg_at_3_std": 0.075458, "nauc_ndcg_at_3_diff1": 0.467389, "nauc_ndcg_at_5_max": 0.35744, "nauc_ndcg_at_5_std": 0.064434, "nauc_ndcg_at_5_diff1": 0.46136, "nauc_ndcg_at_10_max": 0.359559, "nauc_ndcg_at_10_std": 0.069018, "nauc_ndcg_at_10_diff1": 0.464143, "nauc_ndcg_at_20_max": 0.352116, "nauc_ndcg_at_20_std": 0.063247, "nauc_ndcg_at_20_diff1": 0.457219, "nauc_ndcg_at_100_max": 0.364589, "nauc_ndcg_at_100_std": 0.066101, "nauc_ndcg_at_100_diff1": 0.465926, "nauc_ndcg_at_1000_max": 0.366221, "nauc_ndcg_at_1000_std": 0.065985, "nauc_ndcg_at_1000_diff1": 0.465447, "nauc_map_at_1_max": 0.341404, "nauc_map_at_1_std": 0.003898, "nauc_map_at_1_diff1": 0.528226, "nauc_map_at_3_max": 0.354432, "nauc_map_at_3_std": 0.055171, "nauc_map_at_3_diff1": 0.477927, "nauc_map_at_5_max": 0.351993, "nauc_map_at_5_std": 0.049569, "nauc_map_at_5_diff1": 0.476226, "nauc_map_at_10_max": 0.352941, "nauc_map_at_10_std": 0.050982, "nauc_map_at_10_diff1": 0.477124, "nauc_map_at_20_max": 0.35123, "nauc_map_at_20_std": 0.050284, "nauc_map_at_20_diff1": 0.475396, "nauc_map_at_100_max": 0.353237, "nauc_map_at_100_std": 0.050806, "nauc_map_at_100_diff1": 0.477174, "nauc_map_at_1000_max": 0.353336, "nauc_map_at_1000_std": 0.050726, "nauc_map_at_1000_diff1": 0.47718, "nauc_recall_at_1_max": 0.341404, "nauc_recall_at_1_std": 0.003898, "nauc_recall_at_1_diff1": 0.528226, "nauc_recall_at_3_max": 0.361431, "nauc_recall_at_3_std": 0.121148, "nauc_recall_at_3_diff1": 0.410598, "nauc_recall_at_5_max": 0.346582, "nauc_recall_at_5_std": 0.098528, "nauc_recall_at_5_diff1": 0.396035, "nauc_recall_at_10_max": 0.347597, "nauc_recall_at_10_std": 0.116493, "nauc_recall_at_10_diff1": 0.393829, "nauc_recall_at_20_max": 0.307695, "nauc_recall_at_20_std": 0.089611, "nauc_recall_at_20_diff1": 0.348414, "nauc_recall_at_100_max": 0.438337, "nauc_recall_at_100_std": 0.154177, "nauc_recall_at_100_diff1": 0.372848, "nauc_recall_at_1000_max": 0.618793, "nauc_recall_at_1000_std": 0.310567, "nauc_recall_at_1000_diff1": 0.254789, "nauc_precision_at_1_max": 0.36563, "nauc_precision_at_1_std": 0.021405, "nauc_precision_at_1_diff1": 0.545735, "nauc_precision_at_3_max": 0.368948, "nauc_precision_at_3_std": 0.13057, "nauc_precision_at_3_diff1": 0.392224, "nauc_precision_at_5_max": 0.316819, "nauc_precision_at_5_std": 0.093421, "nauc_precision_at_5_diff1": 0.326096, "nauc_precision_at_10_max": 0.292185, "nauc_precision_at_10_std": 0.111568, "nauc_precision_at_10_diff1": 0.280641, "nauc_precision_at_20_max": 0.210837, "nauc_precision_at_20_std": 0.077051, "nauc_precision_at_20_diff1": 0.177257, "nauc_precision_at_100_max": 0.142966, "nauc_precision_at_100_std": 0.079245, "nauc_precision_at_100_diff1": 0.034753, "nauc_precision_at_1000_max": 0.069815, "nauc_precision_at_1000_std": 0.064027, "nauc_precision_at_1000_diff1": -0.110983, "nauc_mrr_at_1_max": 0.36563, "nauc_mrr_at_1_std": 0.021405, "nauc_mrr_at_1_diff1": 0.545735, "nauc_mrr_at_3_max": 0.377567, "nauc_mrr_at_3_std": 0.069935, "nauc_mrr_at_3_diff1": 0.494994, "nauc_mrr_at_5_max": 0.372936, "nauc_mrr_at_5_std": 0.06194, "nauc_mrr_at_5_diff1": 0.489104, "nauc_mrr_at_10_max": 0.37477, "nauc_mrr_at_10_std": 0.064485, "nauc_mrr_at_10_diff1": 0.492345, "nauc_mrr_at_20_max": 0.372642, "nauc_mrr_at_20_std": 0.063059, "nauc_mrr_at_20_diff1": 0.490381, "nauc_mrr_at_100_max": 0.373562, "nauc_mrr_at_100_std": 0.062576, "nauc_mrr_at_100_diff1": 0.491209, "nauc_mrr_at_1000_max": 0.373513, "nauc_mrr_at_1000_std": 0.06247, "nauc_mrr_at_1000_diff1": 0.49126, "main_score": 0.53766, "hf_subset": "korean", "languages": [ "kor-Kore" ] } ] }, "evaluation_time": 5873.947786092758, "kg_co2_emissions": null } ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "task_name": "MultiLongDocRetrieval", "mteb_version": "1.29.16", "scores": { "dev": [ { "ndcg_at_1": 0.185, "ndcg_at_3": 0.24601, "ndcg_at_5": 0.26021, "ndcg_at_10": 0.27135, "ndcg_at_20": 0.28472, "ndcg_at_100": 0.31724, "ndcg_at_1000": 0.34325, "map_at_1": 0.185, "map_at_3": 0.23083, "map_at_5": 0.23858, "map_at_10": 0.24308, "map_at_20": 0.24646, "map_at_100": 0.25034, "map_at_1000": 0.25104, "recall_at_1": 0.185, "recall_at_3": 0.29, "recall_at_5": 0.325, "recall_at_10": 0.36, "recall_at_20": 0.415, "recall_at_100": 0.6, "recall_at_1000": 0.82, "precision_at_1": 0.185, "precision_at_3": 0.09667, "precision_at_5": 0.065, "precision_at_10": 0.036, "precision_at_20": 0.02075, "precision_at_100": 0.006, "precision_at_1000": 0.00082, "mrr_at_1": 0.185, "mrr_at_3": 0.230833, "mrr_at_5": 0.238583, "mrr_at_10": 0.243081, "mrr_at_20": 0.246464, "mrr_at_100": 0.250338, "mrr_at_1000": 0.251036, "nauc_ndcg_at_1_max": 0.527168, "nauc_ndcg_at_1_std": 0.014784, "nauc_ndcg_at_1_diff1": 0.559594, "nauc_ndcg_at_3_max": 0.48434, "nauc_ndcg_at_3_std": 0.091035, "nauc_ndcg_at_3_diff1": 0.472706, "nauc_ndcg_at_5_max": 0.449814, "nauc_ndcg_at_5_std": 0.076969, "nauc_ndcg_at_5_diff1": 0.443832, "nauc_ndcg_at_10_max": 0.447785, "nauc_ndcg_at_10_std": 0.08679, "nauc_ndcg_at_10_diff1": 0.44382, "nauc_ndcg_at_20_max": 0.448409, "nauc_ndcg_at_20_std": 0.091679, "nauc_ndcg_at_20_diff1": 0.440796, "nauc_ndcg_at_100_max": 0.462103, "nauc_ndcg_at_100_std": 0.121956, "nauc_ndcg_at_100_diff1": 0.44629, "nauc_ndcg_at_1000_max": 0.462386, "nauc_ndcg_at_1000_std": 0.118884, "nauc_ndcg_at_1000_diff1": 0.443739, "nauc_map_at_1_max": 0.527168, "nauc_map_at_1_std": 0.014784, "nauc_map_at_1_diff1": 0.559594, "nauc_map_at_3_max": 0.493459, "nauc_map_at_3_std": 0.071993, "nauc_map_at_3_diff1": 0.491309, "nauc_map_at_5_max": 0.473685, "nauc_map_at_5_std": 0.063892, "nauc_map_at_5_diff1": 0.475104, "nauc_map_at_10_max": 0.47256, "nauc_map_at_10_std": 0.067593, "nauc_map_at_10_diff1": 0.474762, "nauc_map_at_20_max": 0.472079, "nauc_map_at_20_std": 0.06829, "nauc_map_at_20_diff1": 0.473401, "nauc_map_at_100_max": 0.473787, "nauc_map_at_100_std": 0.07125, "nauc_map_at_100_diff1": 0.473491, "nauc_map_at_1000_max": 0.473862, "nauc_map_at_1000_std": 0.071282, "nauc_map_at_1000_diff1": 0.473528, "nauc_recall_at_1_max": 0.527168, "nauc_recall_at_1_std": 0.014784, "nauc_recall_at_1_diff1": 0.559594, "nauc_recall_at_3_max": 0.461011, "nauc_recall_at_3_std": 0.140798, "nauc_recall_at_3_diff1": 0.425079, "nauc_recall_at_5_max": 0.385312, "nauc_recall_at_5_std": 0.109065, "nauc_recall_at_5_diff1": 0.361069, "nauc_recall_at_10_max": 0.38162, "nauc_recall_at_10_std": 0.138134, "nauc_recall_at_10_diff1": 0.363486, "nauc_recall_at_20_max": 0.389019, "nauc_recall_at_20_std": 0.159428, "nauc_recall_at_20_diff1": 0.357227, "nauc_recall_at_100_max": 0.45797, "nauc_recall_at_100_std": 0.349295, "nauc_recall_at_100_diff1": 0.384298, "nauc_recall_at_1000_max": 0.455144, "nauc_recall_at_1000_std": 0.481083, "nauc_recall_at_1000_diff1": 0.291524, "nauc_precision_at_1_max": 0.527168, "nauc_precision_at_1_std": 0.014784, "nauc_precision_at_1_diff1": 0.559594, "nauc_precision_at_3_max": 0.461011, "nauc_precision_at_3_std": 0.140798, "nauc_precision_at_3_diff1": 0.425079, "nauc_precision_at_5_max": 0.385312, "nauc_precision_at_5_std": 0.109065, "nauc_precision_at_5_diff1": 0.361069, "nauc_precision_at_10_max": 0.38162, "nauc_precision_at_10_std": 0.138134, "nauc_precision_at_10_diff1": 0.363486, "nauc_precision_at_20_max": 0.389019, "nauc_precision_at_20_std": 0.159428, "nauc_precision_at_20_diff1": 0.357227, "nauc_precision_at_100_max": 0.45797, "nauc_precision_at_100_std": 0.349295, "nauc_precision_at_100_diff1": 0.384298, "nauc_precision_at_1000_max": 0.455144, "nauc_precision_at_1000_std": 0.481083, "nauc_precision_at_1000_diff1": 0.291524, "nauc_mrr_at_1_max": 0.527168, "nauc_mrr_at_1_std": 0.014784, "nauc_mrr_at_1_diff1": 0.559594, "nauc_mrr_at_3_max": 0.493459, "nauc_mrr_at_3_std": 0.071993, "nauc_mrr_at_3_diff1": 0.491309, "nauc_mrr_at_5_max": 0.473685, "nauc_mrr_at_5_std": 0.063892, "nauc_mrr_at_5_diff1": 0.475104, "nauc_mrr_at_10_max": 0.47256, "nauc_mrr_at_10_std": 0.067593, "nauc_mrr_at_10_diff1": 0.474762, "nauc_mrr_at_20_max": 0.472079, "nauc_mrr_at_20_std": 0.06829, "nauc_mrr_at_20_diff1": 0.473401, "nauc_mrr_at_100_max": 0.473787, "nauc_mrr_at_100_std": 0.07125, "nauc_mrr_at_100_diff1": 0.473491, "nauc_mrr_at_1000_max": 0.473862, "nauc_mrr_at_1000_std": 0.071282, "nauc_mrr_at_1000_diff1": 0.473528, "main_score": 0.27135, "hf_subset": "ko", "languages": [ "kor-Hang" ] } ], "test": [ { "ndcg_at_1": 0.215, "ndcg_at_3": 0.24327, "ndcg_at_5": 0.25813, "ndcg_at_10": 0.27145, "ndcg_at_20": 0.2863, "ndcg_at_100": 0.31275, "ndcg_at_1000": 0.34383, "map_at_1": 0.215, "map_at_3": 0.23583, "map_at_5": 0.24433, "map_at_10": 0.25006, "map_at_20": 0.25398, "map_at_100": 0.25714, "map_at_1000": 0.25812, "recall_at_1": 0.215, "recall_at_3": 0.265, "recall_at_5": 0.3, "recall_at_10": 0.34, "recall_at_20": 0.4, "recall_at_100": 0.55, "recall_at_1000": 0.805, "precision_at_1": 0.215, "precision_at_3": 0.08833, "precision_at_5": 0.06, "precision_at_10": 0.034, "precision_at_20": 0.02, "precision_at_100": 0.0055, "precision_at_1000": 0.00081, "mrr_at_1": 0.215, "mrr_at_3": 0.235833, "mrr_at_5": 0.244333, "mrr_at_10": 0.250062, "mrr_at_20": 0.253979, "mrr_at_100": 0.257143, "mrr_at_1000": 0.258122, "nauc_ndcg_at_1_max": 0.549596, "nauc_ndcg_at_1_std": -0.036517, "nauc_ndcg_at_1_diff1": 0.62056, "nauc_ndcg_at_3_max": 0.549458, "nauc_ndcg_at_3_std": -0.004973, "nauc_ndcg_at_3_diff1": 0.606656, "nauc_ndcg_at_5_max": 0.523696, "nauc_ndcg_at_5_std": 0.009265, "nauc_ndcg_at_5_diff1": 0.585378, "nauc_ndcg_at_10_max": 0.517249, "nauc_ndcg_at_10_std": 0.020401, "nauc_ndcg_at_10_diff1": 0.572666, "nauc_ndcg_at_20_max": 0.501538, "nauc_ndcg_at_20_std": 0.021863, "nauc_ndcg_at_20_diff1": 0.547941, "nauc_ndcg_at_100_max": 0.50442, "nauc_ndcg_at_100_std": 0.035896, "nauc_ndcg_at_100_diff1": 0.54428, "nauc_ndcg_at_1000_max": 0.516494, "nauc_ndcg_at_1000_std": 0.056268, "nauc_ndcg_at_1000_diff1": 0.54551, "nauc_map_at_1_max": 0.549596, "nauc_map_at_1_std": -0.036517, "nauc_map_at_1_diff1": 0.62056, "nauc_map_at_3_max": 0.549431, "nauc_map_at_3_std": -0.012275, "nauc_map_at_3_diff1": 0.610549, "nauc_map_at_5_max": 0.534007, "nauc_map_at_5_std": -0.004376, "nauc_map_at_5_diff1": 0.598306, "nauc_map_at_10_max": 0.530521, "nauc_map_at_10_std": -0.001206, "nauc_map_at_10_diff1": 0.592643, "nauc_map_at_20_max": 0.525733, "nauc_map_at_20_std": -0.000354, "nauc_map_at_20_diff1": 0.585909, "nauc_map_at_100_max": 0.525779, "nauc_map_at_100_std": 0.001575, "nauc_map_at_100_diff1": 0.584504, "nauc_map_at_1000_max": 0.526529, "nauc_map_at_1000_std": 0.002524, "nauc_map_at_1000_diff1": 0.584661, "nauc_recall_at_1_max": 0.549596, "nauc_recall_at_1_std": -0.036517, "nauc_recall_at_1_diff1": 0.62056, "nauc_recall_at_3_max": 0.549555, "nauc_recall_at_3_std": 0.015032, "nauc_recall_at_3_diff1": 0.595725, "nauc_recall_at_5_max": 0.49419, "nauc_recall_at_5_std": 0.047257, "nauc_recall_at_5_diff1": 0.548534, "nauc_recall_at_10_max": 0.47967, "nauc_recall_at_10_std": 0.085714, "nauc_recall_at_10_diff1": 0.513991, "nauc_recall_at_20_max": 0.426757, "nauc_recall_at_20_std": 0.086657, "nauc_recall_at_20_diff1": 0.42529, "nauc_recall_at_100_max": 0.441129, "nauc_recall_at_100_std": 0.16625, "nauc_recall_at_100_diff1": 0.410012, "nauc_recall_at_1000_max": 0.517694, "nauc_recall_at_1000_std": 0.512887, "nauc_recall_at_1000_diff1": 0.314148, "nauc_precision_at_1_max": 0.549596, "nauc_precision_at_1_std": -0.036517, "nauc_precision_at_1_diff1": 0.62056, "nauc_precision_at_3_max": 0.549555, "nauc_precision_at_3_std": 0.015032, "nauc_precision_at_3_diff1": 0.595725, "nauc_precision_at_5_max": 0.49419, "nauc_precision_at_5_std": 0.047257, "nauc_precision_at_5_diff1": 0.548534, "nauc_precision_at_10_max": 0.47967, "nauc_precision_at_10_std": 0.085714, "nauc_precision_at_10_diff1": 0.513991, "nauc_precision_at_20_max": 0.426757, "nauc_precision_at_20_std": 0.086657, "nauc_precision_at_20_diff1": 0.42529, "nauc_precision_at_100_max": 0.441129, "nauc_precision_at_100_std": 0.16625, "nauc_precision_at_100_diff1": 0.410012, "nauc_precision_at_1000_max": 0.517694, "nauc_precision_at_1000_std": 0.512887, "nauc_precision_at_1000_diff1": 0.314148, "nauc_mrr_at_1_max": 0.549596, "nauc_mrr_at_1_std": -0.036517, "nauc_mrr_at_1_diff1": 0.62056, "nauc_mrr_at_3_max": 0.549431, "nauc_mrr_at_3_std": -0.012275, "nauc_mrr_at_3_diff1": 0.610549, "nauc_mrr_at_5_max": 0.534007, "nauc_mrr_at_5_std": -0.004376, "nauc_mrr_at_5_diff1": 0.598306, "nauc_mrr_at_10_max": 0.530521, "nauc_mrr_at_10_std": -0.001206, "nauc_mrr_at_10_diff1": 0.592643, "nauc_mrr_at_20_max": 0.525733, "nauc_mrr_at_20_std": -0.000354, "nauc_mrr_at_20_diff1": 0.585909, "nauc_mrr_at_100_max": 0.525779, "nauc_mrr_at_100_std": 0.001575, "nauc_mrr_at_100_diff1": 0.584504, "nauc_mrr_at_1000_max": 0.526529, "nauc_mrr_at_1000_std": 0.002524, "nauc_mrr_at_1000_diff1": 0.584661, "main_score": 0.27145, "hf_subset": "ko", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 220.92831134796143, "kg_co2_emissions": null } ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "task_name": "PublicHealthQA", "mteb_version": "1.29.16", "scores": { "test": [ { "ndcg_at_1": 0.62338, "ndcg_at_3": 0.74629, "ndcg_at_5": 0.76752, "ndcg_at_10": 0.78448, "ndcg_at_20": 0.79802, "ndcg_at_100": 0.80336, "ndcg_at_1000": 0.80336, "map_at_1": 0.62338, "map_at_3": 0.72078, "map_at_5": 0.73247, "map_at_10": 0.73955, "map_at_20": 0.74349, "map_at_100": 0.74443, "map_at_1000": 0.74443, "recall_at_1": 0.62338, "recall_at_3": 0.81818, "recall_at_5": 0.87013, "recall_at_10": 0.92208, "recall_at_20": 0.97403, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.62338, "precision_at_3": 0.27273, "precision_at_5": 0.17403, "precision_at_10": 0.09221, "precision_at_20": 0.0487, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.623377, "mrr_at_3": 0.720779, "mrr_at_5": 0.732468, "mrr_at_10": 0.739554, "mrr_at_20": 0.743491, "mrr_at_100": 0.744426, "mrr_at_1000": 0.744426, "nauc_ndcg_at_1_max": 0.320914, "nauc_ndcg_at_1_std": 0.06798, "nauc_ndcg_at_1_diff1": 0.545833, "nauc_ndcg_at_3_max": 0.304722, "nauc_ndcg_at_3_std": 0.1048, "nauc_ndcg_at_3_diff1": 0.504437, "nauc_ndcg_at_5_max": 0.28857, "nauc_ndcg_at_5_std": 0.063545, "nauc_ndcg_at_5_diff1": 0.51012, "nauc_ndcg_at_10_max": 0.296526, "nauc_ndcg_at_10_std": 0.114659, "nauc_ndcg_at_10_diff1": 0.483728, "nauc_ndcg_at_20_max": 0.317526, "nauc_ndcg_at_20_std": 0.112382, "nauc_ndcg_at_20_diff1": 0.515798, "nauc_ndcg_at_100_max": 0.306529, "nauc_ndcg_at_100_std": 0.094793, "nauc_ndcg_at_100_diff1": 0.513444, "nauc_ndcg_at_1000_max": 0.306529, "nauc_ndcg_at_1000_std": 0.094793, "nauc_ndcg_at_1000_diff1": 0.513444, "nauc_map_at_1_max": 0.320914, "nauc_map_at_1_std": 0.06798, "nauc_map_at_1_diff1": 0.545833, "nauc_map_at_3_max": 0.309003, "nauc_map_at_3_std": 0.095065, "nauc_map_at_3_diff1": 0.515382, "nauc_map_at_5_max": 0.301849, "nauc_map_at_5_std": 0.074899, "nauc_map_at_5_diff1": 0.518527, "nauc_map_at_10_max": 0.305767, "nauc_map_at_10_std": 0.094267, "nauc_map_at_10_diff1": 0.50991, "nauc_map_at_20_max": 0.309326, "nauc_map_at_20_std": 0.09249, "nauc_map_at_20_diff1": 0.518194, "nauc_map_at_100_max": 0.308006, "nauc_map_at_100_std": 0.090224, "nauc_map_at_100_diff1": 0.517606, "nauc_map_at_1000_max": 0.308006, "nauc_map_at_1000_std": 0.090224, "nauc_map_at_1000_diff1": 0.517606, "nauc_recall_at_1_max": 0.320914, "nauc_recall_at_1_std": 0.06798, "nauc_recall_at_1_diff1": 0.545833, "nauc_recall_at_3_max": 0.288097, "nauc_recall_at_3_std": 0.142604, "nauc_recall_at_3_diff1": 0.461934, "nauc_recall_at_5_max": 0.215277, "nauc_recall_at_5_std": -0.00758, "nauc_recall_at_5_diff1": 0.469782, "nauc_recall_at_10_max": 0.217403, "nauc_recall_at_10_std": 0.304819, "nauc_recall_at_10_diff1": 0.246522, "nauc_recall_at_20_max": 0.678958, "nauc_recall_at_20_std": 0.678958, "nauc_recall_at_20_diff1": 0.562819, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.320914, "nauc_precision_at_1_std": 0.06798, "nauc_precision_at_1_diff1": 0.545833, "nauc_precision_at_3_max": 0.288097, "nauc_precision_at_3_std": 0.142604, "nauc_precision_at_3_diff1": 0.461934, "nauc_precision_at_5_max": 0.215277, "nauc_precision_at_5_std": -0.00758, "nauc_precision_at_5_diff1": 0.469782, "nauc_precision_at_10_max": 0.217403, "nauc_precision_at_10_std": 0.304819, "nauc_precision_at_10_diff1": 0.246522, "nauc_precision_at_20_max": 0.678958, "nauc_precision_at_20_std": 0.678958, "nauc_precision_at_20_diff1": 0.562819, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_1000_diff1": NaN, "nauc_mrr_at_1_max": 0.320914, "nauc_mrr_at_1_std": 0.06798, "nauc_mrr_at_1_diff1": 0.545833, "nauc_mrr_at_3_max": 0.309003, "nauc_mrr_at_3_std": 0.095065, "nauc_mrr_at_3_diff1": 0.515382, "nauc_mrr_at_5_max": 0.301849, "nauc_mrr_at_5_std": 0.074899, "nauc_mrr_at_5_diff1": 0.518527, "nauc_mrr_at_10_max": 0.305767, "nauc_mrr_at_10_std": 0.094267, "nauc_mrr_at_10_diff1": 0.50991, "nauc_mrr_at_20_max": 0.309326, "nauc_mrr_at_20_std": 0.09249, "nauc_mrr_at_20_diff1": 0.518194, "nauc_mrr_at_100_max": 0.308006, "nauc_mrr_at_100_std": 0.090224, "nauc_mrr_at_100_diff1": 0.517606, "nauc_mrr_at_1000_max": 0.308006, "nauc_mrr_at_1000_std": 0.090224, "nauc_mrr_at_1000_diff1": 0.517606, "main_score": 0.78448, "hf_subset": "korean", "languages": [ "kor-Hang" ] } ] }, "evaluation_time": 1.3850245475769043, "kg_co2_emissions": null } ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "task_name": "XPQARetrieval", "mteb_version": "1.29.16", "scores": { "test": [ { "ndcg_at_1": 0.30581, "ndcg_at_3": 0.32337, "ndcg_at_5": 0.34775, "ndcg_at_10": 0.36871, "ndcg_at_20": 0.39062, "ndcg_at_100": 0.43898, "ndcg_at_1000": 0.47042, "map_at_1": 0.21654, "map_at_3": 0.2965, "map_at_5": 0.31775, "map_at_10": 0.3276, "map_at_20": 0.33433, "map_at_100": 0.34197, "map_at_1000": 0.34329, "recall_at_1": 0.21654, "recall_at_3": 0.33871, "recall_at_5": 0.39748, "recall_at_10": 0.45464, "recall_at_20": 0.53305, "recall_at_100": 0.76542, "recall_at_1000": 1.0, "precision_at_1": 0.30581, "precision_at_3": 0.18094, "precision_at_5": 0.12936, "precision_at_10": 0.07446, "precision_at_20": 0.0432, "precision_at_100": 0.01252, "precision_at_1000": 0.00156, "mrr_at_1": 0.30581, "mrr_at_3": 0.344801, "mrr_at_5": 0.355275, "mrr_at_10": 0.363192, "mrr_at_20": 0.368504, "mrr_at_100": 0.373922, "mrr_at_1000": 0.374789, "nauc_ndcg_at_1_max": 0.311288, "nauc_ndcg_at_1_std": -0.283077, "nauc_ndcg_at_1_diff1": 0.505347, "nauc_ndcg_at_3_max": 0.31655, "nauc_ndcg_at_3_std": -0.270178, "nauc_ndcg_at_3_diff1": 0.453532, "nauc_ndcg_at_5_max": 0.305679, "nauc_ndcg_at_5_std": -0.275801, "nauc_ndcg_at_5_diff1": 0.436661, "nauc_ndcg_at_10_max": 0.3056, "nauc_ndcg_at_10_std": -0.270147, "nauc_ndcg_at_10_diff1": 0.435218, "nauc_ndcg_at_20_max": 0.306377, "nauc_ndcg_at_20_std": -0.255502, "nauc_ndcg_at_20_diff1": 0.426827, "nauc_ndcg_at_100_max": 0.316141, "nauc_ndcg_at_100_std": -0.224757, "nauc_ndcg_at_100_diff1": 0.414833, "nauc_ndcg_at_1000_max": 0.313036, "nauc_ndcg_at_1000_std": -0.246118, "nauc_ndcg_at_1000_diff1": 0.43138, "nauc_map_at_1_max": 0.235078, "nauc_map_at_1_std": -0.271015, "nauc_map_at_1_diff1": 0.532184, "nauc_map_at_3_max": 0.311095, "nauc_map_at_3_std": -0.269109, "nauc_map_at_3_diff1": 0.457754, "nauc_map_at_5_max": 0.303814, "nauc_map_at_5_std": -0.277169, "nauc_map_at_5_diff1": 0.435833, "nauc_map_at_10_max": 0.304025, "nauc_map_at_10_std": -0.275743, "nauc_map_at_10_diff1": 0.436994, "nauc_map_at_20_max": 0.304141, "nauc_map_at_20_std": -0.271632, "nauc_map_at_20_diff1": 0.433932, "nauc_map_at_100_max": 0.305175, "nauc_map_at_100_std": -0.266555, "nauc_map_at_100_diff1": 0.432242, "nauc_map_at_1000_max": 0.3051, "nauc_map_at_1000_std": -0.267058, "nauc_map_at_1000_diff1": 0.432696, "nauc_recall_at_1_max": 0.235078, "nauc_recall_at_1_std": -0.271015, "nauc_recall_at_1_diff1": 0.532184, "nauc_recall_at_3_max": 0.313823, "nauc_recall_at_3_std": -0.249996, "nauc_recall_at_3_diff1": 0.427477, "nauc_recall_at_5_max": 0.28534, "nauc_recall_at_5_std": -0.268175, "nauc_recall_at_5_diff1": 0.388201, "nauc_recall_at_10_max": 0.286909, "nauc_recall_at_10_std": -0.248995, "nauc_recall_at_10_diff1": 0.379877, "nauc_recall_at_20_max": 0.286162, "nauc_recall_at_20_std": -0.189555, "nauc_recall_at_20_diff1": 0.34246, "nauc_recall_at_100_max": 0.345446, "nauc_recall_at_100_std": 0.039232, "nauc_recall_at_100_diff1": 0.22118, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.311288, "nauc_precision_at_1_std": -0.283077, "nauc_precision_at_1_diff1": 0.505347, "nauc_precision_at_3_max": 0.335789, "nauc_precision_at_3_std": -0.21226, "nauc_precision_at_3_diff1": 0.27896, "nauc_precision_at_5_max": 0.286018, "nauc_precision_at_5_std": -0.211069, "nauc_precision_at_5_diff1": 0.2114, "nauc_precision_at_10_max": 0.267447, "nauc_precision_at_10_std": -0.188898, "nauc_precision_at_10_diff1": 0.206173, "nauc_precision_at_20_max": 0.243744, "nauc_precision_at_20_std": -0.144214, "nauc_precision_at_20_diff1": 0.163522, "nauc_precision_at_100_max": 0.214885, "nauc_precision_at_100_std": 0.027884, "nauc_precision_at_100_diff1": 0.03346, "nauc_precision_at_1000_max": 0.140024, "nauc_precision_at_1000_std": 0.039212, "nauc_precision_at_1000_diff1": -0.029463, "nauc_mrr_at_1_max": 0.311288, "nauc_mrr_at_1_std": -0.283077, "nauc_mrr_at_1_diff1": 0.505347, "nauc_mrr_at_3_max": 0.325613, "nauc_mrr_at_3_std": -0.272578, "nauc_mrr_at_3_diff1": 0.485305, "nauc_mrr_at_5_max": 0.319542, "nauc_mrr_at_5_std": -0.273768, "nauc_mrr_at_5_diff1": 0.480104, "nauc_mrr_at_10_max": 0.317487, "nauc_mrr_at_10_std": -0.271725, "nauc_mrr_at_10_diff1": 0.476175, "nauc_mrr_at_20_max": 0.31808, "nauc_mrr_at_20_std": -0.267332, "nauc_mrr_at_20_diff1": 0.474981, "nauc_mrr_at_100_max": 0.319132, "nauc_mrr_at_100_std": -0.264791, "nauc_mrr_at_100_diff1": 0.474002, "nauc_mrr_at_1000_max": 0.319001, "nauc_mrr_at_1000_std": -0.265319, "nauc_mrr_at_1000_diff1": 0.474376, "main_score": 0.36871, "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ] }, { "ndcg_at_1": 0.28746, "ndcg_at_3": 0.2976, "ndcg_at_5": 0.31584, "ndcg_at_10": 0.34545, "ndcg_at_20": 0.37121, "ndcg_at_100": 0.42469, "ndcg_at_1000": 0.45866, "map_at_1": 0.1656, "map_at_3": 0.24906, "map_at_5": 0.27422, "map_at_10": 0.28984, "map_at_20": 0.29906, "map_at_100": 0.30812, "map_at_1000": 0.30997, "recall_at_1": 0.1656, "recall_at_3": 0.2936, "recall_at_5": 0.35798, "recall_at_10": 0.43349, "recall_at_20": 0.51575, "recall_at_100": 0.76636, "recall_at_1000": 0.99809, "precision_at_1": 0.28746, "precision_at_3": 0.1947, "precision_at_5": 0.14771, "precision_at_10": 0.09021, "precision_at_20": 0.05413, "precision_at_100": 0.01554, "precision_at_1000": 0.00195, "mrr_at_1": 0.287462, "mrr_at_3": 0.335372, "mrr_at_5": 0.35158, "mrr_at_10": 0.360888, "mrr_at_20": 0.36642, "mrr_at_100": 0.371447, "mrr_at_1000": 0.372251, "nauc_ndcg_at_1_max": 0.201657, "nauc_ndcg_at_1_std": -0.2879, "nauc_ndcg_at_1_diff1": 0.442656, "nauc_ndcg_at_3_max": 0.192234, "nauc_ndcg_at_3_std": -0.290577, "nauc_ndcg_at_3_diff1": 0.402866, "nauc_ndcg_at_5_max": 0.19155, "nauc_ndcg_at_5_std": -0.286866, "nauc_ndcg_at_5_diff1": 0.400679, "nauc_ndcg_at_10_max": 0.177678, "nauc_ndcg_at_10_std": -0.291851, "nauc_ndcg_at_10_diff1": 0.395469, "nauc_ndcg_at_20_max": 0.165234, "nauc_ndcg_at_20_std": -0.291409, "nauc_ndcg_at_20_diff1": 0.378526, "nauc_ndcg_at_100_max": 0.179826, "nauc_ndcg_at_100_std": -0.260967, "nauc_ndcg_at_100_diff1": 0.376394, "nauc_ndcg_at_1000_max": 0.187783, "nauc_ndcg_at_1000_std": -0.267931, "nauc_ndcg_at_1000_diff1": 0.388467, "nauc_map_at_1_max": 0.174807, "nauc_map_at_1_std": -0.245718, "nauc_map_at_1_diff1": 0.457417, "nauc_map_at_3_max": 0.194672, "nauc_map_at_3_std": -0.276974, "nauc_map_at_3_diff1": 0.423815, "nauc_map_at_5_max": 0.197, "nauc_map_at_5_std": -0.281495, "nauc_map_at_5_diff1": 0.410867, "nauc_map_at_10_max": 0.190462, "nauc_map_at_10_std": -0.286135, "nauc_map_at_10_diff1": 0.406367, "nauc_map_at_20_max": 0.184592, "nauc_map_at_20_std": -0.287948, "nauc_map_at_20_diff1": 0.399485, "nauc_map_at_100_max": 0.186309, "nauc_map_at_100_std": -0.283896, "nauc_map_at_100_diff1": 0.398831, "nauc_map_at_1000_max": 0.187486, "nauc_map_at_1000_std": -0.28301, "nauc_map_at_1000_diff1": 0.399449, "nauc_recall_at_1_max": 0.174807, "nauc_recall_at_1_std": -0.245718, "nauc_recall_at_1_diff1": 0.457417, "nauc_recall_at_3_max": 0.17536, "nauc_recall_at_3_std": -0.276148, "nauc_recall_at_3_diff1": 0.383754, "nauc_recall_at_5_max": 0.168221, "nauc_recall_at_5_std": -0.274878, "nauc_recall_at_5_diff1": 0.355527, "nauc_recall_at_10_max": 0.135118, "nauc_recall_at_10_std": -0.281284, "nauc_recall_at_10_diff1": 0.348563, "nauc_recall_at_20_max": 0.097836, "nauc_recall_at_20_std": -0.275298, "nauc_recall_at_20_diff1": 0.289646, "nauc_recall_at_100_max": 0.144121, "nauc_recall_at_100_std": -0.076736, "nauc_recall_at_100_diff1": 0.223493, "nauc_recall_at_1000_max": 0.687293, "nauc_recall_at_1000_std": 0.824538, "nauc_recall_at_1000_diff1": 0.6613, "nauc_precision_at_1_max": 0.201657, "nauc_precision_at_1_std": -0.2879, "nauc_precision_at_1_diff1": 0.442656, "nauc_precision_at_3_max": 0.186142, "nauc_precision_at_3_std": -0.267155, "nauc_precision_at_3_diff1": 0.293118, "nauc_precision_at_5_max": 0.16958, "nauc_precision_at_5_std": -0.2512, "nauc_precision_at_5_diff1": 0.240926, "nauc_precision_at_10_max": 0.121913, "nauc_precision_at_10_std": -0.239312, "nauc_precision_at_10_diff1": 0.197129, "nauc_precision_at_20_max": 0.07668, "nauc_precision_at_20_std": -0.218574, "nauc_precision_at_20_diff1": 0.140458, "nauc_precision_at_100_max": 0.087452, "nauc_precision_at_100_std": -0.059537, "nauc_precision_at_100_diff1": 0.054791, "nauc_precision_at_1000_max": 0.080563, "nauc_precision_at_1000_std": 0.000892, "nauc_precision_at_1000_diff1": -0.002318, "nauc_mrr_at_1_max": 0.201657, "nauc_mrr_at_1_std": -0.2879, "nauc_mrr_at_1_diff1": 0.442656, "nauc_mrr_at_3_max": 0.195831, "nauc_mrr_at_3_std": -0.294736, "nauc_mrr_at_3_diff1": 0.419949, "nauc_mrr_at_5_max": 0.189667, "nauc_mrr_at_5_std": -0.29252, "nauc_mrr_at_5_diff1": 0.411453, "nauc_mrr_at_10_max": 0.18534, "nauc_mrr_at_10_std": -0.291873, "nauc_mrr_at_10_diff1": 0.410579, "nauc_mrr_at_20_max": 0.185276, "nauc_mrr_at_20_std": -0.287898, "nauc_mrr_at_20_diff1": 0.407642, "nauc_mrr_at_100_max": 0.187201, "nauc_mrr_at_100_std": -0.285521, "nauc_mrr_at_100_diff1": 0.408829, "nauc_mrr_at_1000_max": 0.187446, "nauc_mrr_at_1000_std": -0.285824, "nauc_mrr_at_1000_diff1": 0.409163, "main_score": 0.34545, "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ] }, { "ndcg_at_1": 0.25081, "ndcg_at_3": 0.27258, "ndcg_at_5": 0.2871, "ndcg_at_10": 0.31205, "ndcg_at_20": 0.33519, "ndcg_at_100": 0.3934, "ndcg_at_1000": 0.42727, "map_at_1": 0.16895, "map_at_3": 0.24394, "map_at_5": 0.25794, "map_at_10": 0.27103, "map_at_20": 0.27809, "map_at_100": 0.28706, "map_at_1000": 0.28864, "recall_at_1": 0.16895, "recall_at_3": 0.28555, "recall_at_5": 0.32541, "recall_at_10": 0.39173, "recall_at_20": 0.47433, "recall_at_100": 0.75536, "recall_at_1000": 1.0, "precision_at_1": 0.25081, "precision_at_3": 0.16124, "precision_at_5": 0.11238, "precision_at_10": 0.06906, "precision_at_20": 0.04096, "precision_at_100": 0.01303, "precision_at_1000": 0.00167, "mrr_at_1": 0.250814, "mrr_at_3": 0.294245, "mrr_at_5": 0.301982, "mrr_at_10": 0.310341, "mrr_at_20": 0.316275, "mrr_at_100": 0.322775, "mrr_at_1000": 0.323634, "nauc_ndcg_at_1_max": 0.193886, "nauc_ndcg_at_1_std": -0.216627, "nauc_ndcg_at_1_diff1": 0.441826, "nauc_ndcg_at_3_max": 0.180387, "nauc_ndcg_at_3_std": -0.213751, "nauc_ndcg_at_3_diff1": 0.387516, "nauc_ndcg_at_5_max": 0.159271, "nauc_ndcg_at_5_std": -0.22786, "nauc_ndcg_at_5_diff1": 0.366407, "nauc_ndcg_at_10_max": 0.165582, "nauc_ndcg_at_10_std": -0.225034, "nauc_ndcg_at_10_diff1": 0.349482, "nauc_ndcg_at_20_max": 0.161333, "nauc_ndcg_at_20_std": -0.22264, "nauc_ndcg_at_20_diff1": 0.352346, "nauc_ndcg_at_100_max": 0.174417, "nauc_ndcg_at_100_std": -0.175529, "nauc_ndcg_at_100_diff1": 0.334731, "nauc_ndcg_at_1000_max": 0.175213, "nauc_ndcg_at_1000_std": -0.196456, "nauc_ndcg_at_1000_diff1": 0.357803, "nauc_map_at_1_max": 0.163012, "nauc_map_at_1_std": -0.229248, "nauc_map_at_1_diff1": 0.481955, "nauc_map_at_3_max": 0.180761, "nauc_map_at_3_std": -0.219693, "nauc_map_at_3_diff1": 0.39315, "nauc_map_at_5_max": 0.167584, "nauc_map_at_5_std": -0.225807, "nauc_map_at_5_diff1": 0.375512, "nauc_map_at_10_max": 0.167727, "nauc_map_at_10_std": -0.227501, "nauc_map_at_10_diff1": 0.366203, "nauc_map_at_20_max": 0.165379, "nauc_map_at_20_std": -0.227643, "nauc_map_at_20_diff1": 0.36711, "nauc_map_at_100_max": 0.168346, "nauc_map_at_100_std": -0.219738, "nauc_map_at_100_diff1": 0.365347, "nauc_map_at_1000_max": 0.168504, "nauc_map_at_1000_std": -0.220301, "nauc_map_at_1000_diff1": 0.366307, "nauc_recall_at_1_max": 0.163012, "nauc_recall_at_1_std": -0.229248, "nauc_recall_at_1_diff1": 0.481955, "nauc_recall_at_3_max": 0.178663, "nauc_recall_at_3_std": -0.212073, "nauc_recall_at_3_diff1": 0.363339, "nauc_recall_at_5_max": 0.132254, "nauc_recall_at_5_std": -0.236296, "nauc_recall_at_5_diff1": 0.315634, "nauc_recall_at_10_max": 0.153427, "nauc_recall_at_10_std": -0.220275, "nauc_recall_at_10_diff1": 0.269116, "nauc_recall_at_20_max": 0.136551, "nauc_recall_at_20_std": -0.210697, "nauc_recall_at_20_diff1": 0.268921, "nauc_recall_at_100_max": 0.166553, "nauc_recall_at_100_std": 0.06243, "nauc_recall_at_100_diff1": 0.091275, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_1000_diff1": NaN, "nauc_precision_at_1_max": 0.193886, "nauc_precision_at_1_std": -0.216627, "nauc_precision_at_1_diff1": 0.441826, "nauc_precision_at_3_max": 0.163269, "nauc_precision_at_3_std": -0.170411, "nauc_precision_at_3_diff1": 0.240167, "nauc_precision_at_5_max": 0.113688, "nauc_precision_at_5_std": -0.183426, "nauc_precision_at_5_diff1": 0.189589, "nauc_precision_at_10_max": 0.100432, "nauc_precision_at_10_std": -0.17552, "nauc_precision_at_10_diff1": 0.142262, "nauc_precision_at_20_max": 0.095465, "nauc_precision_at_20_std": -0.145022, "nauc_precision_at_20_diff1": 0.14824, "nauc_precision_at_100_max": 0.129026, "nauc_precision_at_100_std": 0.104406, "nauc_precision_at_100_diff1": 0.027278, "nauc_precision_at_1000_max": 0.094267, "nauc_precision_at_1000_std": 0.105362, "nauc_precision_at_1000_diff1": 0.008359, "nauc_mrr_at_1_max": 0.193886, "nauc_mrr_at_1_std": -0.216627, "nauc_mrr_at_1_diff1": 0.441826, "nauc_mrr_at_3_max": 0.187154, "nauc_mrr_at_3_std": -0.208072, "nauc_mrr_at_3_diff1": 0.416461, "nauc_mrr_at_5_max": 0.174889, "nauc_mrr_at_5_std": -0.216175, "nauc_mrr_at_5_diff1": 0.404943, "nauc_mrr_at_10_max": 0.180038, "nauc_mrr_at_10_std": -0.211804, "nauc_mrr_at_10_diff1": 0.399066, "nauc_mrr_at_20_max": 0.179579, "nauc_mrr_at_20_std": -0.210213, "nauc_mrr_at_20_diff1": 0.399724, "nauc_mrr_at_100_max": 0.180684, "nauc_mrr_at_100_std": -0.206719, "nauc_mrr_at_100_diff1": 0.39797, "nauc_mrr_at_1000_max": 0.180764, "nauc_mrr_at_1000_std": -0.207127, "nauc_mrr_at_1000_diff1": 0.398566, "main_score": 0.31205, "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ] } ] }, "evaluation_time": 17.949707984924316, "kg_co2_emissions": null } ================================================ FILE: eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/model_meta.json ================================================ {"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instructions": null, "training_datasets": null, "adapted_from": null, "superseded_by": null, "loader": null} ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 26.646844148635864, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.76466, "map_at_1": 0.57895, "map_at_10": 0.70816, "map_at_100": 0.71079, "map_at_1000": 0.71088, "map_at_20": 0.71012, "map_at_3": 0.68275, "map_at_5": 0.69942, "mrr_at_1": 0.5789473684210527, "mrr_at_10": 0.7081592871066557, "mrr_at_100": 0.7107949944358983, "mrr_at_1000": 0.7108785366247037, "mrr_at_20": 0.7101188285398814, "mrr_at_3": 0.682748538011696, "mrr_at_5": 0.6994152046783627, "nauc_map_at_1000_diff1": 0.5829110062545787, "nauc_map_at_1000_max": 0.23128052346477607, "nauc_map_at_1000_std": -0.20377647286138453, "nauc_map_at_100_diff1": 0.5830135479066073, "nauc_map_at_100_max": 0.23150875202930604, "nauc_map_at_100_std": -0.20365994369781237, "nauc_map_at_10_diff1": 0.5802843326592293, "nauc_map_at_10_max": 0.2343691605851438, "nauc_map_at_10_std": -0.1995251028748617, "nauc_map_at_1_diff1": 0.6403438739047393, "nauc_map_at_1_max": 0.18225916909648981, "nauc_map_at_1_std": -0.2271351677125876, "nauc_map_at_20_diff1": 0.5828885713014177, "nauc_map_at_20_max": 0.23233109061892526, "nauc_map_at_20_std": -0.20391468794273773, "nauc_map_at_3_diff1": 0.5900444084004162, "nauc_map_at_3_max": 0.24110925672649372, "nauc_map_at_3_std": -0.21439877098670698, "nauc_map_at_5_diff1": 0.5821939194952693, "nauc_map_at_5_max": 0.242817217238973, "nauc_map_at_5_std": -0.18968439733720696, "nauc_mrr_at_1000_diff1": 0.5829110062545787, "nauc_mrr_at_1000_max": 0.23128052346477607, "nauc_mrr_at_1000_std": -0.20377647286138453, "nauc_mrr_at_100_diff1": 0.5830135479066073, "nauc_mrr_at_100_max": 0.23150875202930604, "nauc_mrr_at_100_std": -0.20365994369781237, "nauc_mrr_at_10_diff1": 0.5802843326592293, "nauc_mrr_at_10_max": 0.2343691605851438, "nauc_mrr_at_10_std": -0.1995251028748617, "nauc_mrr_at_1_diff1": 0.6403438739047393, "nauc_mrr_at_1_max": 0.18225916909648981, "nauc_mrr_at_1_std": -0.2271351677125876, "nauc_mrr_at_20_diff1": 0.5828885713014177, "nauc_mrr_at_20_max": 0.23233109061892526, "nauc_mrr_at_20_std": -0.20391468794273773, "nauc_mrr_at_3_diff1": 0.5900444084004162, "nauc_mrr_at_3_max": 0.24110925672649372, "nauc_mrr_at_3_std": -0.21439877098670698, "nauc_mrr_at_5_diff1": 0.5821939194952693, "nauc_mrr_at_5_max": 0.242817217238973, "nauc_mrr_at_5_std": -0.18968439733720696, "nauc_ndcg_at_1000_diff1": 0.5773268804066995, "nauc_ndcg_at_1000_max": 0.23923313794532927, "nauc_ndcg_at_1000_std": -0.20329298222158235, "nauc_ndcg_at_100_diff1": 0.5794181753883131, "nauc_ndcg_at_100_max": 0.24374202633049696, "nauc_ndcg_at_100_std": -0.20096525193591755, "nauc_ndcg_at_10_diff1": 0.5648083467695035, "nauc_ndcg_at_10_max": 0.2565211201237364, "nauc_ndcg_at_10_std": -0.19112254941280304, "nauc_ndcg_at_1_diff1": 0.6403438739047393, "nauc_ndcg_at_1_max": 0.18225916909648981, "nauc_ndcg_at_1_std": -0.2271351677125876, "nauc_ndcg_at_20_diff1": 0.5747514020950384, "nauc_ndcg_at_20_max": 0.2474030248483696, "nauc_ndcg_at_20_std": -0.21037902951574675, "nauc_ndcg_at_3_diff1": 0.5860471149700772, "nauc_ndcg_at_3_max": 0.2666193142967102, "nauc_ndcg_at_3_std": -0.2149612280213397, "nauc_ndcg_at_5_diff1": 0.5699609677327039, "nauc_ndcg_at_5_max": 0.2753338721736746, "nauc_ndcg_at_5_std": -0.16581140344599296, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 0.8697886921759251, "nauc_precision_at_100_max": 0.8697886921759251, "nauc_precision_at_100_std": 0.12223365206034731, "nauc_precision_at_10_diff1": 0.4054838122463558, "nauc_precision_at_10_max": 0.4950344150961882, "nauc_precision_at_10_std": -0.11514909457473732, "nauc_precision_at_1_diff1": 0.6403438739047393, "nauc_precision_at_1_max": 0.18225916909648981, "nauc_precision_at_1_std": -0.2271351677125876, "nauc_precision_at_20_diff1": 0.48194467595441853, "nauc_precision_at_20_max": 0.4745484821827859, "nauc_precision_at_20_std": -0.4601411718520565, "nauc_precision_at_3_diff1": 0.5767799568769227, "nauc_precision_at_3_max": 0.37242262131016984, "nauc_precision_at_3_std": -0.21979199837598418, "nauc_precision_at_5_diff1": 0.5048896679723024, "nauc_precision_at_5_max": 0.472750814625999, "nauc_precision_at_5_std": -0.016834198269562995, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8697886921759079, "nauc_recall_at_100_max": 0.8697886921759079, "nauc_recall_at_100_std": 0.12223365206031499, "nauc_recall_at_10_diff1": 0.40548381224635466, "nauc_recall_at_10_max": 0.4950344150961911, "nauc_recall_at_10_std": -0.11514909457473714, "nauc_recall_at_1_diff1": 0.6403438739047393, "nauc_recall_at_1_max": 0.18225916909648981, "nauc_recall_at_1_std": -0.2271351677125876, "nauc_recall_at_20_diff1": 0.48194467595441987, "nauc_recall_at_20_max": 0.4745484821827929, "nauc_recall_at_20_std": -0.4601411718520569, "nauc_recall_at_3_diff1": 0.5767799568769236, "nauc_recall_at_3_max": 0.3724226213101706, "nauc_recall_at_3_std": -0.21979199837598373, "nauc_recall_at_5_diff1": 0.5048896679723038, "nauc_recall_at_5_max": 0.47275081462600177, "nauc_recall_at_5_std": -0.016834198269564362, "ndcg_at_1": 0.57895, "ndcg_at_10": 0.76466, "ndcg_at_100": 0.77633, "ndcg_at_1000": 0.77763, "ndcg_at_20": 0.77147, "ndcg_at_3": 0.7148, "ndcg_at_5": 0.74426, "precision_at_1": 0.57895, "precision_at_10": 0.09386, "precision_at_100": 0.00991, "precision_at_1000": 0.001, "precision_at_20": 0.04825, "precision_at_3": 0.26901, "precision_at_5": 0.17544, "recall_at_1": 0.57895, "recall_at_10": 0.9386, "recall_at_100": 0.99123, "recall_at_1000": 1.0, "recall_at_20": 0.96491, "recall_at_3": 0.80702, "recall_at_5": 0.87719 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 104.64979100227356, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.89451, "map_at_1": 0.82778, "map_at_10": 0.87456, "map_at_100": 0.8758, "map_at_1000": 0.87589, "map_at_20": 0.87534, "map_at_3": 0.86685, "map_at_5": 0.87235, "mrr_at_1": 0.8277777777777777, "mrr_at_10": 0.8745582010582011, "mrr_at_100": 0.8758046640392401, "mrr_at_1000": 0.8758850873355876, "mrr_at_20": 0.8753374166444343, "mrr_at_3": 0.8668518518518518, "mrr_at_5": 0.8723518518518518, "nauc_map_at_1000_diff1": 0.8765941147751475, "nauc_map_at_1000_max": 0.8184670320296146, "nauc_map_at_1000_std": 0.2043466216556563, "nauc_map_at_100_diff1": 0.8765424901088339, "nauc_map_at_100_max": 0.81855492741636, "nauc_map_at_100_std": 0.2045661771941467, "nauc_map_at_10_diff1": 0.8765135889872123, "nauc_map_at_10_max": 0.8193676890799481, "nauc_map_at_10_std": 0.20616727802609794, "nauc_map_at_1_diff1": 0.8867658774417635, "nauc_map_at_1_max": 0.7924731525445328, "nauc_map_at_1_std": 0.14594181192441288, "nauc_map_at_20_diff1": 0.8768109145959556, "nauc_map_at_20_max": 0.8189936498500454, "nauc_map_at_20_std": 0.20548817848269876, "nauc_map_at_3_diff1": 0.8721401111097621, "nauc_map_at_3_max": 0.8190431660209794, "nauc_map_at_3_std": 0.20189887851198635, "nauc_map_at_5_diff1": 0.8747765806150434, "nauc_map_at_5_max": 0.8201125877199104, "nauc_map_at_5_std": 0.20757675285082214, "nauc_mrr_at_1000_diff1": 0.8765941147751475, "nauc_mrr_at_1000_max": 0.8184670320296146, "nauc_mrr_at_1000_std": 0.2043466216556563, "nauc_mrr_at_100_diff1": 0.8765424901088339, "nauc_mrr_at_100_max": 0.81855492741636, "nauc_mrr_at_100_std": 0.2045661771941467, "nauc_mrr_at_10_diff1": 0.8765135889872123, "nauc_mrr_at_10_max": 0.8193676890799481, "nauc_mrr_at_10_std": 0.20616727802609794, "nauc_mrr_at_1_diff1": 0.8867658774417635, "nauc_mrr_at_1_max": 0.7924731525445328, "nauc_mrr_at_1_std": 0.14594181192441288, "nauc_mrr_at_20_diff1": 0.8768109145959556, "nauc_mrr_at_20_max": 0.8189936498500454, "nauc_mrr_at_20_std": 0.20548817848269876, "nauc_mrr_at_3_diff1": 0.8721401111097621, "nauc_mrr_at_3_max": 0.8190431660209794, "nauc_mrr_at_3_std": 0.20189887851198635, "nauc_mrr_at_5_diff1": 0.8747765806150434, "nauc_mrr_at_5_max": 0.8201125877199104, "nauc_mrr_at_5_std": 0.20757675285082214, "nauc_ndcg_at_1000_diff1": 0.8760407250178142, "nauc_ndcg_at_1000_max": 0.8237066948911186, "nauc_ndcg_at_1000_std": 0.21778313960840487, "nauc_ndcg_at_100_diff1": 0.8749002952635064, "nauc_ndcg_at_100_max": 0.8260964950874432, "nauc_ndcg_at_100_std": 0.2242878788311883, "nauc_ndcg_at_10_diff1": 0.8764469804941419, "nauc_ndcg_at_10_max": 0.8310922283782205, "nauc_ndcg_at_10_std": 0.23470651073965554, "nauc_ndcg_at_1_diff1": 0.8867658774417635, "nauc_ndcg_at_1_max": 0.7924731525445328, "nauc_ndcg_at_1_std": 0.14594181192441288, "nauc_ndcg_at_20_diff1": 0.8777498779087349, "nauc_ndcg_at_20_max": 0.8298172101274326, "nauc_ndcg_at_20_std": 0.23235065354034032, "nauc_ndcg_at_3_diff1": 0.8667275120085678, "nauc_ndcg_at_3_max": 0.8294622491545487, "nauc_ndcg_at_3_std": 0.22511113369692567, "nauc_ndcg_at_5_diff1": 0.8714639073447893, "nauc_ndcg_at_5_max": 0.8322960824684759, "nauc_ndcg_at_5_std": 0.2372072419718233, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8194833488951155, "nauc_precision_at_100_max": 0.9519530034235867, "nauc_precision_at_100_std": 0.5737628384687296, "nauc_precision_at_10_diff1": 0.8835200746965407, "nauc_precision_at_10_max": 0.9216269841269835, "nauc_precision_at_10_std": 0.4553804855275415, "nauc_precision_at_1_diff1": 0.8867658774417635, "nauc_precision_at_1_max": 0.7924731525445328, "nauc_precision_at_1_std": 0.14594181192441288, "nauc_precision_at_20_diff1": 0.9011048863990015, "nauc_precision_at_20_max": 0.9370525988173025, "nauc_precision_at_20_std": 0.5000466853408015, "nauc_precision_at_3_diff1": 0.8434812521499847, "nauc_precision_at_3_max": 0.8745147181679728, "nauc_precision_at_3_std": 0.3261032483168725, "nauc_precision_at_5_diff1": 0.853959765724471, "nauc_precision_at_5_max": 0.902427637721753, "nauc_precision_at_5_std": 0.4090569561157763, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8194833488951093, "nauc_recall_at_100_max": 0.9519530034235866, "nauc_recall_at_100_std": 0.5737628384687221, "nauc_recall_at_10_diff1": 0.8835200746965439, "nauc_recall_at_10_max": 0.921626984126984, "nauc_recall_at_10_std": 0.4553804855275454, "nauc_recall_at_1_diff1": 0.8867658774417635, "nauc_recall_at_1_max": 0.7924731525445328, "nauc_recall_at_1_std": 0.14594181192441288, "nauc_recall_at_20_diff1": 0.9011048863990032, "nauc_recall_at_20_max": 0.9370525988173043, "nauc_recall_at_20_std": 0.5000466853408011, "nauc_recall_at_3_diff1": 0.8434812521499828, "nauc_recall_at_3_max": 0.8745147181679692, "nauc_recall_at_3_std": 0.32610324831686943, "nauc_recall_at_5_diff1": 0.8539597657244727, "nauc_recall_at_5_max": 0.9024276377217549, "nauc_recall_at_5_std": 0.40905695611578, "ndcg_at_1": 0.82778, "ndcg_at_10": 0.89451, "ndcg_at_100": 0.90095, "ndcg_at_1000": 0.90274, "ndcg_at_20": 0.89733, "ndcg_at_3": 0.87938, "ndcg_at_5": 0.88913, "precision_at_1": 0.82778, "precision_at_10": 0.09556, "precision_at_100": 0.00987, "precision_at_1000": 0.001, "precision_at_20": 0.04833, "precision_at_3": 0.30519, "precision_at_5": 0.18778, "recall_at_1": 0.82778, "recall_at_10": 0.95556, "recall_at_100": 0.98667, "recall_at_1000": 1.0, "recall_at_20": 0.96667, "recall_at_3": 0.91556, "recall_at_5": 0.93889 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.94404, "map_at_1": 0.89667, "map_at_10": 0.9308, "map_at_100": 0.93143, "map_at_1000": 0.93144, "map_at_20": 0.93123, "map_at_3": 0.92722, "map_at_5": 0.92894, "mrr_at_1": 0.8966666666666666, "mrr_at_10": 0.9308002645502649, "mrr_at_100": 0.9314284409958359, "mrr_at_1000": 0.9314385805367853, "mrr_at_20": 0.931229114452799, "mrr_at_3": 0.9272222222222225, "mrr_at_5": 0.9289444444444448, "nauc_map_at_1000_diff1": 0.8944097187683182, "nauc_map_at_1000_max": 0.7331593396576956, "nauc_map_at_1000_std": -0.17657754769293518, "nauc_map_at_100_diff1": 0.8944253819595299, "nauc_map_at_100_max": 0.7331989226215538, "nauc_map_at_100_std": -0.17640301497183783, "nauc_map_at_10_diff1": 0.8942036768623971, "nauc_map_at_10_max": 0.7351280873510182, "nauc_map_at_10_std": -0.17113751257430934, "nauc_map_at_1_diff1": 0.8900769285058929, "nauc_map_at_1_max": 0.6828387929383056, "nauc_map_at_1_std": -0.2375759958863897, "nauc_map_at_20_diff1": 0.8944467039706967, "nauc_map_at_20_max": 0.7338846356141927, "nauc_map_at_20_std": -0.17387853872569373, "nauc_map_at_3_diff1": 0.8951668441985208, "nauc_map_at_3_max": 0.7406033223478056, "nauc_map_at_3_std": -0.1749987489365976, "nauc_map_at_5_diff1": 0.8938483367738029, "nauc_map_at_5_max": 0.7395307392770073, "nauc_map_at_5_std": -0.17639382868741973, "nauc_mrr_at_1000_diff1": 0.8944097187683182, "nauc_mrr_at_1000_max": 0.7331593396576956, "nauc_mrr_at_1000_std": -0.17657754769293518, "nauc_mrr_at_100_diff1": 0.8944253819595299, "nauc_mrr_at_100_max": 0.7331989226215538, "nauc_mrr_at_100_std": -0.17640301497183783, "nauc_mrr_at_10_diff1": 0.8942036768623971, "nauc_mrr_at_10_max": 0.7351280873510182, "nauc_mrr_at_10_std": -0.17113751257430934, "nauc_mrr_at_1_diff1": 0.8900769285058929, "nauc_mrr_at_1_max": 0.6828387929383056, "nauc_mrr_at_1_std": -0.2375759958863897, "nauc_mrr_at_20_diff1": 0.8944467039706967, "nauc_mrr_at_20_max": 0.7338846356141927, "nauc_mrr_at_20_std": -0.17387853872569373, "nauc_mrr_at_3_diff1": 0.8951668441985208, "nauc_mrr_at_3_max": 0.7406033223478056, "nauc_mrr_at_3_std": -0.1749987489365976, "nauc_mrr_at_5_diff1": 0.8938483367738029, "nauc_mrr_at_5_max": 0.7395307392770073, "nauc_mrr_at_5_std": -0.17639382868741973, "nauc_ndcg_at_1000_diff1": 0.8951841037723368, "nauc_ndcg_at_1000_max": 0.7409071064713759, "nauc_ndcg_at_1000_std": -0.1606713452509553, "nauc_ndcg_at_100_diff1": 0.8957385508571588, "nauc_ndcg_at_100_max": 0.7422776361780857, "nauc_ndcg_at_100_std": -0.15453171503288304, "nauc_ndcg_at_10_diff1": 0.8951195079697556, "nauc_ndcg_at_10_max": 0.7526671853587154, "nauc_ndcg_at_10_std": -0.12627422949906705, "nauc_ndcg_at_1_diff1": 0.8900769285058929, "nauc_ndcg_at_1_max": 0.6828387929383056, "nauc_ndcg_at_1_std": -0.2375759958863897, "nauc_ndcg_at_20_diff1": 0.8960069380183776, "nauc_ndcg_at_20_max": 0.74806717367277, "nauc_ndcg_at_20_std": -0.13433623091149807, "nauc_ndcg_at_3_diff1": 0.8967126881150357, "nauc_ndcg_at_3_max": 0.7652745835504532, "nauc_ndcg_at_3_std": -0.14427405393147222, "nauc_ndcg_at_5_diff1": 0.8941084626720966, "nauc_ndcg_at_5_max": 0.7639066051670461, "nauc_ndcg_at_5_std": -0.14605075393547182, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.9054621848739591, "nauc_precision_at_10_max": 0.950980392156856, "nauc_precision_at_10_std": 0.4800253434707128, "nauc_precision_at_1_diff1": 0.8900769285058929, "nauc_precision_at_1_max": 0.6828387929383056, "nauc_precision_at_1_std": -0.2375759958863897, "nauc_precision_at_20_diff1": 0.9273783587509046, "nauc_precision_at_20_max": 0.9691358024691454, "nauc_precision_at_20_std": 0.6805166511048747, "nauc_precision_at_3_diff1": 0.9047069808315497, "nauc_precision_at_3_max": 0.8992283187784935, "nauc_precision_at_3_std": 0.025086505190320282, "nauc_precision_at_5_diff1": 0.8945257115191743, "nauc_precision_at_5_max": 0.9255628177196821, "nauc_precision_at_5_std": 0.05842583947158553, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 1.0, "nauc_recall_at_100_max": 1.0, "nauc_recall_at_100_std": 1.0, "nauc_recall_at_10_diff1": 0.9054621848739518, "nauc_recall_at_10_max": 0.9509803921568608, "nauc_recall_at_10_std": 0.4800253434707287, "nauc_recall_at_1_diff1": 0.8900769285058929, "nauc_recall_at_1_max": 0.6828387929383056, "nauc_recall_at_1_std": -0.2375759958863897, "nauc_recall_at_20_diff1": 0.9273783587509028, "nauc_recall_at_20_max": 0.9691358024691298, "nauc_recall_at_20_std": 0.6805166511048744, "nauc_recall_at_3_diff1": 0.9047069808315451, "nauc_recall_at_3_max": 0.8992283187784907, "nauc_recall_at_3_std": 0.025086505190313194, "nauc_recall_at_5_diff1": 0.8945257115191748, "nauc_recall_at_5_max": 0.9255628177196803, "nauc_recall_at_5_std": 0.05842583947159392, "ndcg_at_1": 0.89667, "ndcg_at_10": 0.94404, "ndcg_at_100": 0.94693, "ndcg_at_1000": 0.94721, "ndcg_at_20": 0.94549, "ndcg_at_3": 0.93628, "ndcg_at_5": 0.93944, "precision_at_1": 0.89667, "precision_at_10": 0.09844, "precision_at_100": 0.00998, "precision_at_1000": 0.001, "precision_at_20": 0.0495, "precision_at_3": 0.32074, "precision_at_5": 0.194, "recall_at_1": 0.89667, "recall_at_10": 0.98444, "recall_at_100": 0.99778, "recall_at_1000": 1.0, "recall_at_20": 0.99, "recall_at_3": 0.96222, "recall_at_5": 0.97 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.83939, "map_at_1": 0.74556, "map_at_10": 0.81086, "map_at_100": 0.81348, "map_at_1000": 0.81361, "map_at_20": 0.81281, "map_at_3": 0.80037, "map_at_5": 0.80576, "mrr_at_1": 0.7455555555555555, "mrr_at_10": 0.8108615520282187, "mrr_at_100": 0.8134795094402523, "mrr_at_1000": 0.8136063404487494, "mrr_at_20": 0.8128069114742231, "mrr_at_3": 0.8003703703703703, "mrr_at_5": 0.8057592592592592, "nauc_map_at_1000_diff1": 0.8236854639603854, "nauc_map_at_1000_max": 0.8103952303909684, "nauc_map_at_1000_std": 0.2586868987678583, "nauc_map_at_100_diff1": 0.8236446486633416, "nauc_map_at_100_max": 0.8104796888389018, "nauc_map_at_100_std": 0.2589135923444036, "nauc_map_at_10_diff1": 0.8225884486067362, "nauc_map_at_10_max": 0.8114661410076641, "nauc_map_at_10_std": 0.2619132283873032, "nauc_map_at_1_diff1": 0.8410043951458513, "nauc_map_at_1_max": 0.7818407338540867, "nauc_map_at_1_std": 0.19217127541767628, "nauc_map_at_20_diff1": 0.8234074332253382, "nauc_map_at_20_max": 0.8107245343183193, "nauc_map_at_20_std": 0.25984670164966256, "nauc_map_at_3_diff1": 0.8216450589475749, "nauc_map_at_3_max": 0.8133226584904062, "nauc_map_at_3_std": 0.26377009759337877, "nauc_map_at_5_diff1": 0.8209266148182254, "nauc_map_at_5_max": 0.8117429469022233, "nauc_map_at_5_std": 0.26320383986602525, "nauc_mrr_at_1000_diff1": 0.8236854639603854, "nauc_mrr_at_1000_max": 0.8103952303909684, "nauc_mrr_at_1000_std": 0.2586868987678583, "nauc_mrr_at_100_diff1": 0.8236446486633416, "nauc_mrr_at_100_max": 0.8104796888389018, "nauc_mrr_at_100_std": 0.2589135923444036, "nauc_mrr_at_10_diff1": 0.8225884486067362, "nauc_mrr_at_10_max": 0.8114661410076641, "nauc_mrr_at_10_std": 0.2619132283873032, "nauc_mrr_at_1_diff1": 0.8410043951458513, "nauc_mrr_at_1_max": 0.7818407338540867, "nauc_mrr_at_1_std": 0.19217127541767628, "nauc_mrr_at_20_diff1": 0.8234074332253382, "nauc_mrr_at_20_max": 0.8107245343183193, "nauc_mrr_at_20_std": 0.25984670164966256, "nauc_mrr_at_3_diff1": 0.8216450589475749, "nauc_mrr_at_3_max": 0.8133226584904062, "nauc_mrr_at_3_std": 0.26377009759337877, "nauc_mrr_at_5_diff1": 0.8209266148182254, "nauc_mrr_at_5_max": 0.8117429469022233, "nauc_mrr_at_5_std": 0.26320383986602525, "nauc_ndcg_at_1000_diff1": 0.8216579324360656, "nauc_ndcg_at_1000_max": 0.8151412460616603, "nauc_ndcg_at_1000_std": 0.27193364821124083, "nauc_ndcg_at_100_diff1": 0.8210987189079783, "nauc_ndcg_at_100_max": 0.8173944243567918, "nauc_ndcg_at_100_std": 0.27868650248356985, "nauc_ndcg_at_10_diff1": 0.8170767326320595, "nauc_ndcg_at_10_max": 0.8224982340728226, "nauc_ndcg_at_10_std": 0.2932402399225149, "nauc_ndcg_at_1_diff1": 0.8410043951458513, "nauc_ndcg_at_1_max": 0.7818407338540867, "nauc_ndcg_at_1_std": 0.19217127541767628, "nauc_ndcg_at_20_diff1": 0.8196429827542367, "nauc_ndcg_at_20_max": 0.8195488316353458, "nauc_ndcg_at_20_std": 0.2861480223727952, "nauc_ndcg_at_3_diff1": 0.8145823402723746, "nauc_ndcg_at_3_max": 0.8244369456509973, "nauc_ndcg_at_3_std": 0.29220631014785814, "nauc_ndcg_at_5_diff1": 0.8128094206384131, "nauc_ndcg_at_5_max": 0.8218584503476292, "nauc_ndcg_at_5_std": 0.29312522084572695, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8000570598609865, "nauc_precision_at_100_max": 0.9307500778089097, "nauc_precision_at_100_std": 0.627632534495279, "nauc_precision_at_10_diff1": 0.7837175895999401, "nauc_precision_at_10_max": 0.8981972276089916, "nauc_precision_at_10_std": 0.5126912303382867, "nauc_precision_at_1_diff1": 0.8410043951458513, "nauc_precision_at_1_max": 0.7818407338540867, "nauc_precision_at_1_std": 0.19217127541767628, "nauc_precision_at_20_diff1": 0.7910204732299405, "nauc_precision_at_20_max": 0.9047619047619067, "nauc_precision_at_20_std": 0.5567625424153366, "nauc_precision_at_3_diff1": 0.7862129987129995, "nauc_precision_at_3_max": 0.8689028314028325, "nauc_precision_at_3_std": 0.40753700128700165, "nauc_precision_at_5_diff1": 0.7746852599528464, "nauc_precision_at_5_max": 0.8679428075098536, "nauc_precision_at_5_std": 0.4342314997756957, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8000570598609825, "nauc_recall_at_100_max": 0.9307500778089021, "nauc_recall_at_100_std": 0.6276325344952765, "nauc_recall_at_10_diff1": 0.7837175895999435, "nauc_recall_at_10_max": 0.8981972276089929, "nauc_recall_at_10_std": 0.512691230338289, "nauc_recall_at_1_diff1": 0.8410043951458513, "nauc_recall_at_1_max": 0.7818407338540867, "nauc_recall_at_1_std": 0.19217127541767628, "nauc_recall_at_20_diff1": 0.7910204732299413, "nauc_recall_at_20_max": 0.9047619047619062, "nauc_recall_at_20_std": 0.5567625424153411, "nauc_recall_at_3_diff1": 0.786212998712998, "nauc_recall_at_3_max": 0.8689028314028308, "nauc_recall_at_3_std": 0.4075370012870014, "nauc_recall_at_5_diff1": 0.7746852599528486, "nauc_recall_at_5_max": 0.8679428075098551, "nauc_recall_at_5_std": 0.4342314997756967, "ndcg_at_1": 0.74556, "ndcg_at_10": 0.83939, "ndcg_at_100": 0.85105, "ndcg_at_1000": 0.85376, "ndcg_at_20": 0.84625, "ndcg_at_3": 0.81746, "ndcg_at_5": 0.82712, "precision_at_1": 0.74556, "precision_at_10": 0.09278, "precision_at_100": 0.0098, "precision_at_1000": 0.001, "precision_at_20": 0.04772, "precision_at_3": 0.28889, "precision_at_5": 0.178, "recall_at_1": 0.74556, "recall_at_10": 0.92778, "recall_at_100": 0.98, "recall_at_1000": 1.0, "recall_at_20": 0.95444, "recall_at_3": 0.86667, "recall_at_5": 0.89 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 237.30947017669678, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.73634, "map_at_1": 0.44241, "map_at_10": 0.68383, "map_at_100": 0.69176, "map_at_1000": 0.6921, "map_at_20": 0.68904, "map_at_3": 0.64254, "map_at_5": 0.6684, "mrr_at_1": 0.6790540540540541, "mrr_at_10": 0.7532972704847704, "mrr_at_100": 0.7559985262633544, "mrr_at_1000": 0.7561420435710626, "mrr_at_20": 0.7550215149020293, "mrr_at_3": 0.7446509009009006, "mrr_at_5": 0.7494650900900897, "nauc_map_at_1000_diff1": 0.48024490354329186, "nauc_map_at_1000_max": 0.6577756188539895, "nauc_map_at_1000_std": 0.32397669434789156, "nauc_map_at_100_diff1": 0.4804109136170325, "nauc_map_at_100_max": 0.6580401855215885, "nauc_map_at_100_std": 0.324251519616049, "nauc_map_at_10_diff1": 0.4781401379135908, "nauc_map_at_10_max": 0.6568089319286831, "nauc_map_at_10_std": 0.3202292904413023, "nauc_map_at_1_diff1": 0.51999296982784, "nauc_map_at_1_max": 0.34504071442645023, "nauc_map_at_1_std": 0.08913155871900409, "nauc_map_at_20_diff1": 0.4801684760313508, "nauc_map_at_20_max": 0.6579232930060366, "nauc_map_at_20_std": 0.32348525742939516, "nauc_map_at_3_diff1": 0.4528236603136616, "nauc_map_at_3_max": 0.6080448272990129, "nauc_map_at_3_std": 0.2662375486373062, "nauc_map_at_5_diff1": 0.46138854487159403, "nauc_map_at_5_max": 0.644121088475665, "nauc_map_at_5_std": 0.3124756372747152, "nauc_mrr_at_1000_diff1": 0.54167194486031, "nauc_mrr_at_1000_max": 0.6946953972618817, "nauc_mrr_at_1000_std": 0.3866057057930993, "nauc_mrr_at_100_diff1": 0.5417526786003602, "nauc_mrr_at_100_max": 0.6948396323300571, "nauc_mrr_at_100_std": 0.3868498594318191, "nauc_mrr_at_10_diff1": 0.5419284774153654, "nauc_mrr_at_10_max": 0.6964656597669708, "nauc_mrr_at_10_std": 0.3867261217686278, "nauc_mrr_at_1_diff1": 0.5547727290978133, "nauc_mrr_at_1_max": 0.6386574505999577, "nauc_mrr_at_1_std": 0.29287479172122255, "nauc_mrr_at_20_diff1": 0.5410970500118761, "nauc_mrr_at_20_max": 0.6952034358130702, "nauc_mrr_at_20_std": 0.3869955866876459, "nauc_mrr_at_3_diff1": 0.5372790416884039, "nauc_mrr_at_3_max": 0.6956157270245855, "nauc_mrr_at_3_std": 0.38559048071434454, "nauc_mrr_at_5_diff1": 0.5384152692038623, "nauc_mrr_at_5_max": 0.69596351576861, "nauc_mrr_at_5_std": 0.38785563897547254, "nauc_ndcg_at_1000_diff1": 0.5044511740728997, "nauc_ndcg_at_1000_max": 0.696478013741544, "nauc_ndcg_at_1000_std": 0.3854855467736299, "nauc_ndcg_at_100_diff1": 0.5099500981488786, "nauc_ndcg_at_100_max": 0.7039875042608843, "nauc_ndcg_at_100_std": 0.39397458509915845, "nauc_ndcg_at_10_diff1": 0.5010657696272983, "nauc_ndcg_at_10_max": 0.7010178968298936, "nauc_ndcg_at_10_std": 0.37961734234385697, "nauc_ndcg_at_1_diff1": 0.5547727290978133, "nauc_ndcg_at_1_max": 0.6386574505999577, "nauc_ndcg_at_1_std": 0.29287479172122255, "nauc_ndcg_at_20_diff1": 0.5063214529426214, "nauc_ndcg_at_20_max": 0.703223309216109, "nauc_ndcg_at_20_std": 0.3884921467593283, "nauc_ndcg_at_3_diff1": 0.4665841947069789, "nauc_ndcg_at_3_max": 0.6486065928362671, "nauc_ndcg_at_3_std": 0.31098809930840826, "nauc_ndcg_at_5_diff1": 0.47257352117214096, "nauc_ndcg_at_5_max": 0.6798015549552966, "nauc_ndcg_at_5_std": 0.36385777849580075, "nauc_precision_at_1000_diff1": -0.2045792141194065, "nauc_precision_at_1000_max": 0.0923983166590144, "nauc_precision_at_1000_std": 0.1842917601255491, "nauc_precision_at_100_diff1": -0.0924778240068036, "nauc_precision_at_100_max": 0.23945740578209762, "nauc_precision_at_100_std": 0.2772321794780336, "nauc_precision_at_10_diff1": 0.0050384572423236235, "nauc_precision_at_10_max": 0.38808155226369717, "nauc_precision_at_10_std": 0.32186634654595453, "nauc_precision_at_1_diff1": 0.5547727290978133, "nauc_precision_at_1_max": 0.6386574505999577, "nauc_precision_at_1_std": 0.29287479172122255, "nauc_precision_at_20_diff1": -0.027283037477734473, "nauc_precision_at_20_max": 0.331546778853098, "nauc_precision_at_20_std": 0.3063002908816235, "nauc_precision_at_3_diff1": 0.057265296872990146, "nauc_precision_at_3_max": 0.5059780250293879, "nauc_precision_at_3_std": 0.3348233691716301, "nauc_precision_at_5_diff1": 0.02009426255892043, "nauc_precision_at_5_max": 0.4538078963381503, "nauc_precision_at_5_std": 0.35551411831086677, "nauc_recall_at_1000_diff1": 0.5593920012054591, "nauc_recall_at_1000_max": 0.9484961822611118, "nauc_recall_at_1000_std": 0.9145594097008884, "nauc_recall_at_100_diff1": 0.6054992834066972, "nauc_recall_at_100_max": 0.8929036099365333, "nauc_recall_at_100_std": 0.7177467917566535, "nauc_recall_at_10_diff1": 0.5075914378203094, "nauc_recall_at_10_max": 0.7804190681705211, "nauc_recall_at_10_std": 0.49436746198645015, "nauc_recall_at_1_diff1": 0.51999296982784, "nauc_recall_at_1_max": 0.34504071442645023, "nauc_recall_at_1_std": 0.08913155871900409, "nauc_recall_at_20_diff1": 0.5361802043962446, "nauc_recall_at_20_max": 0.8061942008149131, "nauc_recall_at_20_std": 0.555787928974621, "nauc_recall_at_3_diff1": 0.41912542875681236, "nauc_recall_at_3_max": 0.633651990506452, "nauc_recall_at_3_std": 0.3153751225453728, "nauc_recall_at_5_diff1": 0.426538357846737, "nauc_recall_at_5_max": 0.7052808185418215, "nauc_recall_at_5_std": 0.4257365974276217, "ndcg_at_1": 0.67905, "ndcg_at_10": 0.73634, "ndcg_at_100": 0.7627, "ndcg_at_1000": 0.77106, "ndcg_at_20": 0.74878, "ndcg_at_3": 0.68791, "ndcg_at_5": 0.7133, "precision_at_1": 0.67905, "precision_at_10": 0.15, "precision_at_100": 0.01711, "precision_at_1000": 0.00184, "precision_at_20": 0.07931, "precision_at_3": 0.41273, "precision_at_5": 0.27399, "recall_at_1": 0.44241, "recall_at_10": 0.80961, "recall_at_100": 0.91154, "recall_at_1000": 0.9656, "recall_at_20": 0.84899, "recall_at_3": 0.70058, "recall_at_5": 0.75421 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 33137.345321416855, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.56248, "map_at_1": 0.30948, "map_at_10": 0.47448, "map_at_100": 0.49609, "map_at_1000": 0.49721, "map_at_20": 0.48632, "map_at_3": 0.41323, "map_at_5": 0.44685, "mrr_at_1": 0.5117370892018779, "mrr_at_10": 0.6263469707131678, "mrr_at_100": 0.6310493015293891, "mrr_at_1000": 0.6312807995001647, "mrr_at_20": 0.6296541877527791, "mrr_at_3": 0.6056338028169013, "mrr_at_5": 0.6187793427230046, "nauc_map_at_1000_diff1": 0.37925146547858096, "nauc_map_at_1000_max": 0.38393750763748724, "nauc_map_at_1000_std": 0.05891865912238271, "nauc_map_at_100_diff1": 0.3782379567688696, "nauc_map_at_100_max": 0.38322503194927765, "nauc_map_at_100_std": 0.058831474657309604, "nauc_map_at_10_diff1": 0.3832710216827202, "nauc_map_at_10_max": 0.36683267089602234, "nauc_map_at_10_std": 0.026892103842806724, "nauc_map_at_1_diff1": 0.49365461114311804, "nauc_map_at_1_max": 0.2918514591021837, "nauc_map_at_1_std": -0.11208835818015112, "nauc_map_at_20_diff1": 0.3793428340027727, "nauc_map_at_20_max": 0.3748696182549113, "nauc_map_at_20_std": 0.04339019773594362, "nauc_map_at_3_diff1": 0.4134825237458013, "nauc_map_at_3_max": 0.33924523392135614, "nauc_map_at_3_std": -0.03084896014209, "nauc_map_at_5_diff1": 0.3992244042521568, "nauc_map_at_5_max": 0.3502362451364744, "nauc_map_at_5_std": -0.0008844652182879082, "nauc_mrr_at_1000_diff1": 0.3804494072179413, "nauc_mrr_at_1000_max": 0.41181085719579424, "nauc_mrr_at_1000_std": 0.13615036332775504, "nauc_mrr_at_100_diff1": 0.3802312256800871, "nauc_mrr_at_100_max": 0.41152305903347564, "nauc_mrr_at_100_std": 0.1365010579879352, "nauc_mrr_at_10_diff1": 0.3798182269100851, "nauc_mrr_at_10_max": 0.41366216007296513, "nauc_mrr_at_10_std": 0.1389683805203404, "nauc_mrr_at_1_diff1": 0.4160609388879719, "nauc_mrr_at_1_max": 0.3532899641057627, "nauc_mrr_at_1_std": 0.05289132915299127, "nauc_mrr_at_20_diff1": 0.37803605444131233, "nauc_mrr_at_20_max": 0.4109375741828441, "nauc_mrr_at_20_std": 0.13732524262993256, "nauc_mrr_at_3_diff1": 0.3824567401276728, "nauc_mrr_at_3_max": 0.42021258755510976, "nauc_mrr_at_3_std": 0.12955773749855876, "nauc_mrr_at_5_diff1": 0.3789704117435803, "nauc_mrr_at_5_max": 0.4147745485772556, "nauc_mrr_at_5_std": 0.1369112019461578, "nauc_ndcg_at_1000_diff1": 0.363053513038303, "nauc_ndcg_at_1000_max": 0.42128006908240295, "nauc_ndcg_at_1000_std": 0.13584194128028712, "nauc_ndcg_at_100_diff1": 0.3506382906727237, "nauc_ndcg_at_100_max": 0.4154933578112467, "nauc_ndcg_at_100_std": 0.1419311999381482, "nauc_ndcg_at_10_diff1": 0.35725727935664, "nauc_ndcg_at_10_max": 0.39020719251422, "nauc_ndcg_at_10_std": 0.07699604833791614, "nauc_ndcg_at_1_diff1": 0.4160609388879719, "nauc_ndcg_at_1_max": 0.3532899641057627, "nauc_ndcg_at_1_std": 0.05289132915299127, "nauc_ndcg_at_20_diff1": 0.34602119552486355, "nauc_ndcg_at_20_max": 0.396721647626583, "nauc_ndcg_at_20_std": 0.09923537325308451, "nauc_ndcg_at_3_diff1": 0.3763835032280149, "nauc_ndcg_at_3_max": 0.4000639447674701, "nauc_ndcg_at_3_std": 0.0734424840936636, "nauc_ndcg_at_5_diff1": 0.37576210381027525, "nauc_ndcg_at_5_max": 0.38922798447671997, "nauc_ndcg_at_5_std": 0.06607350859818849, "nauc_precision_at_1000_diff1": -0.15659288685601278, "nauc_precision_at_1000_max": 0.10073399800798498, "nauc_precision_at_1000_std": 0.3600674256961004, "nauc_precision_at_100_diff1": -0.17475131446533904, "nauc_precision_at_100_max": 0.14409497116554254, "nauc_precision_at_100_std": 0.41957566181019906, "nauc_precision_at_10_diff1": -0.13137530202946976, "nauc_precision_at_10_max": 0.24963100519082526, "nauc_precision_at_10_std": 0.3958546008338007, "nauc_precision_at_1_diff1": 0.4160609388879719, "nauc_precision_at_1_max": 0.3532899641057627, "nauc_precision_at_1_std": 0.05289132915299127, "nauc_precision_at_20_diff1": -0.16766718583593815, "nauc_precision_at_20_max": 0.21212193458523845, "nauc_precision_at_20_std": 0.40767885355800093, "nauc_precision_at_3_diff1": 0.028779165141450597, "nauc_precision_at_3_max": 0.34986561938560257, "nauc_precision_at_3_std": 0.3142930493368277, "nauc_precision_at_5_diff1": -0.062490258776151913, "nauc_precision_at_5_max": 0.2838887776811703, "nauc_precision_at_5_std": 0.3754445793992626, "nauc_recall_at_1000_diff1": 0.2625174161462087, "nauc_recall_at_1000_max": 0.5058751981466968, "nauc_recall_at_1000_std": 0.41365934629491147, "nauc_recall_at_100_diff1": 0.19090263080722517, "nauc_recall_at_100_max": 0.37715710310087497, "nauc_recall_at_100_std": 0.3279595994770744, "nauc_recall_at_10_diff1": 0.2638311881347135, "nauc_recall_at_10_max": 0.31671324018241753, "nauc_recall_at_10_std": 0.047722887210953885, "nauc_recall_at_1_diff1": 0.49365461114311804, "nauc_recall_at_1_max": 0.2918514591021837, "nauc_recall_at_1_std": -0.11208835818015112, "nauc_recall_at_20_diff1": 0.20618528582383414, "nauc_recall_at_20_max": 0.3113722053667383, "nauc_recall_at_20_std": 0.09157097795159481, "nauc_recall_at_3_diff1": 0.3481381867556256, "nauc_recall_at_3_max": 0.32068460146844313, "nauc_recall_at_3_std": -0.03481664824465478, "nauc_recall_at_5_diff1": 0.3112152698315664, "nauc_recall_at_5_max": 0.3086393422242716, "nauc_recall_at_5_std": -0.01626746932818709, "ndcg_at_1": 0.51174, "ndcg_at_10": 0.56248, "ndcg_at_100": 0.62302, "ndcg_at_1000": 0.63886, "ndcg_at_20": 0.58752, "ndcg_at_3": 0.51214, "ndcg_at_5": 0.53072, "precision_at_1": 0.51174, "precision_at_10": 0.14319, "precision_at_100": 0.02169, "precision_at_1000": 0.00245, "precision_at_20": 0.08404, "precision_at_3": 0.30516, "precision_at_5": 0.23005, "recall_at_1": 0.30948, "recall_at_10": 0.66458, "recall_at_100": 0.86656, "recall_at_1000": 0.96072, "recall_at_20": 0.73488, "recall_at_3": 0.48693, "recall_at_5": 0.56965 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 33973.33324599266, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.44728, "map_at_1": 0.28583, "map_at_10": 0.38924, "map_at_100": 0.39971, "map_at_1000": 0.40018, "map_at_20": 0.3961, "map_at_3": 0.35854, "map_at_5": 0.37754, "mrr_at_1": 0.3087885985748218, "mrr_at_10": 0.4115861704935341, "mrr_at_100": 0.41973492049542444, "mrr_at_1000": 0.4200780686652876, "mrr_at_20": 0.4168646874081562, "mrr_at_3": 0.3828186856690421, "mrr_at_5": 0.4013460015835315, "nauc_map_at_1000_diff1": 0.48372971824247185, "nauc_map_at_1000_max": 0.40168966875755646, "nauc_map_at_1000_std": 0.1285061926289655, "nauc_map_at_100_diff1": 0.4833848030996116, "nauc_map_at_100_max": 0.4018035826214681, "nauc_map_at_100_std": 0.12885925526978198, "nauc_map_at_10_diff1": 0.4830161176312886, "nauc_map_at_10_max": 0.3985904062346475, "nauc_map_at_10_std": 0.11965352228819547, "nauc_map_at_1_diff1": 0.5746242473139114, "nauc_map_at_1_max": 0.396500930373006, "nauc_map_at_1_std": 0.09460472034396682, "nauc_map_at_20_diff1": 0.4836382254283399, "nauc_map_at_20_max": 0.40127233440151805, "nauc_map_at_20_std": 0.12589792975498248, "nauc_map_at_3_diff1": 0.4936982288981077, "nauc_map_at_3_max": 0.4028372641792409, "nauc_map_at_3_std": 0.10674461390494162, "nauc_map_at_5_diff1": 0.48598164884819683, "nauc_map_at_5_max": 0.401841357514279, "nauc_map_at_5_std": 0.11702141380744592, "nauc_mrr_at_1000_diff1": 0.4943420682279905, "nauc_mrr_at_1000_max": 0.41808539497314195, "nauc_mrr_at_1000_std": 0.1598519050725324, "nauc_mrr_at_100_diff1": 0.4940577062017333, "nauc_mrr_at_100_max": 0.41816429953760254, "nauc_mrr_at_100_std": 0.16004526001331285, "nauc_mrr_at_10_diff1": 0.4934310284365195, "nauc_mrr_at_10_max": 0.41665182716899923, "nauc_mrr_at_10_std": 0.15560215149520606, "nauc_mrr_at_1_diff1": 0.5724327254732907, "nauc_mrr_at_1_max": 0.4058774880102169, "nauc_mrr_at_1_std": 0.1250378922282418, "nauc_mrr_at_20_diff1": 0.49445123004973524, "nauc_mrr_at_20_max": 0.41857409422818076, "nauc_mrr_at_20_std": 0.15927696365870916, "nauc_mrr_at_3_diff1": 0.5036890367233479, "nauc_mrr_at_3_max": 0.4203619306278547, "nauc_mrr_at_3_std": 0.1395379625838852, "nauc_mrr_at_5_diff1": 0.49654517183667035, "nauc_mrr_at_5_max": 0.4185014234088432, "nauc_mrr_at_5_std": 0.15114613437898916, "nauc_ndcg_at_1000_diff1": 0.4625779088303276, "nauc_ndcg_at_1000_max": 0.41406576604336215, "nauc_ndcg_at_1000_std": 0.17748518231716406, "nauc_ndcg_at_100_diff1": 0.45159432918365955, "nauc_ndcg_at_100_max": 0.41426577077646537, "nauc_ndcg_at_100_std": 0.18599051372730077, "nauc_ndcg_at_10_diff1": 0.4522864180155836, "nauc_ndcg_at_10_max": 0.4011682556085032, "nauc_ndcg_at_10_std": 0.14626934147248594, "nauc_ndcg_at_1_diff1": 0.5724327254732907, "nauc_ndcg_at_1_max": 0.4058774880102169, "nauc_ndcg_at_1_std": 0.1250378922282418, "nauc_ndcg_at_20_diff1": 0.45435864027081746, "nauc_ndcg_at_20_max": 0.40988667988806715, "nauc_ndcg_at_20_std": 0.16624740189899134, "nauc_ndcg_at_3_diff1": 0.47389621715921837, "nauc_ndcg_at_3_max": 0.40901774289509407, "nauc_ndcg_at_3_std": 0.11705012854210183, "nauc_ndcg_at_5_diff1": 0.45980908807654736, "nauc_ndcg_at_5_max": 0.40752173290426674, "nauc_ndcg_at_5_std": 0.13774330119040987, "nauc_precision_at_1000_diff1": 0.04140219661244718, "nauc_precision_at_1000_max": 0.17654948776428595, "nauc_precision_at_1000_std": 0.3039186497711353, "nauc_precision_at_100_diff1": 0.05980477244767715, "nauc_precision_at_100_max": 0.24382763958828502, "nauc_precision_at_100_std": 0.3711864834409094, "nauc_precision_at_10_diff1": 0.2614488984819199, "nauc_precision_at_10_max": 0.3408588588696674, "nauc_precision_at_10_std": 0.21319520575289586, "nauc_precision_at_1_diff1": 0.5724327254732907, "nauc_precision_at_1_max": 0.4058774880102169, "nauc_precision_at_1_std": 0.1250378922282418, "nauc_precision_at_20_diff1": 0.20986441359275726, "nauc_precision_at_20_max": 0.3232709607257924, "nauc_precision_at_20_std": 0.28231811638624704, "nauc_precision_at_3_diff1": 0.38717992771711085, "nauc_precision_at_3_max": 0.4146312510950824, "nauc_precision_at_3_std": 0.1439453573034876, "nauc_precision_at_5_diff1": 0.3288179741781324, "nauc_precision_at_5_max": 0.3957686802944973, "nauc_precision_at_5_std": 0.19930637202133739, "nauc_recall_at_1000_diff1": 0.35946258297874323, "nauc_recall_at_1000_max": 0.5201793036771298, "nauc_recall_at_1000_std": 0.590054630887816, "nauc_recall_at_100_diff1": 0.24716884378565085, "nauc_recall_at_100_max": 0.46118861559094754, "nauc_recall_at_100_std": 0.5169952188036749, "nauc_recall_at_10_diff1": 0.32824913211442686, "nauc_recall_at_10_max": 0.37297663482851995, "nauc_recall_at_10_std": 0.19311853884927607, "nauc_recall_at_1_diff1": 0.5746242473139114, "nauc_recall_at_1_max": 0.396500930373006, "nauc_recall_at_1_std": 0.09460472034396682, "nauc_recall_at_20_diff1": 0.3223491189352304, "nauc_recall_at_20_max": 0.40927823196987545, "nauc_recall_at_20_std": 0.28640455005390064, "nauc_recall_at_3_diff1": 0.39794868348142554, "nauc_recall_at_3_max": 0.397940641626162, "nauc_recall_at_3_std": 0.11302745138912197, "nauc_recall_at_5_diff1": 0.35830984073737415, "nauc_recall_at_5_max": 0.393494120447945, "nauc_recall_at_5_std": 0.16341818262680563, "ndcg_at_1": 0.30879, "ndcg_at_10": 0.44728, "ndcg_at_100": 0.49391, "ndcg_at_1000": 0.50578, "ndcg_at_20": 0.46943, "ndcg_at_3": 0.38692, "ndcg_at_5": 0.42028, "precision_at_1": 0.30879, "precision_at_10": 0.0677, "precision_at_100": 0.00938, "precision_at_1000": 0.00105, "precision_at_20": 0.03884, "precision_at_3": 0.1631, "precision_at_5": 0.11639, "recall_at_1": 0.28583, "recall_at_10": 0.60451, "recall_at_100": 0.81235, "recall_at_1000": 0.9038, "recall_at_20": 0.68725, "recall_at_3": 0.44576, "recall_at_5": 0.52336 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 791.2342638969421, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.31108, "map_at_1": 0.225, "map_at_10": 0.27634, "map_at_100": 0.28547, "map_at_1000": 0.2864, "map_at_20": 0.28083, "map_at_3": 0.25333, "map_at_5": 0.26533, "mrr_at_1": 0.225, "mrr_at_10": 0.2763373015873016, "mrr_at_100": 0.28547413152370693, "mrr_at_1000": 0.2864043033289739, "mrr_at_20": 0.28082997615989114, "mrr_at_3": 0.2533333333333333, "mrr_at_5": 0.26533333333333337, "nauc_map_at_1000_diff1": 0.5827440283312539, "nauc_map_at_1000_max": 0.47121346840093437, "nauc_map_at_1000_std": 0.01761850507064754, "nauc_map_at_100_diff1": 0.5818862178246292, "nauc_map_at_100_max": 0.4702677970629844, "nauc_map_at_100_std": 0.01732113924124195, "nauc_map_at_10_diff1": 0.5854386442392447, "nauc_map_at_10_max": 0.47577814442978594, "nauc_map_at_10_std": 0.013683084601007884, "nauc_map_at_1_diff1": 0.6303121271600662, "nauc_map_at_1_max": 0.45306546588153795, "nauc_map_at_1_std": -0.03716276026632798, "nauc_map_at_20_diff1": 0.5828572421752594, "nauc_map_at_20_max": 0.46939532577476534, "nauc_map_at_20_std": 0.012200111756481068, "nauc_map_at_3_diff1": 0.5953459236004133, "nauc_map_at_3_max": 0.4673015424214653, "nauc_map_at_3_std": -0.007863229530253665, "nauc_map_at_5_diff1": 0.596001269055143, "nauc_map_at_5_max": 0.47341165290086296, "nauc_map_at_5_std": 0.005731512744635958, "nauc_mrr_at_1000_diff1": 0.5827440283312539, "nauc_mrr_at_1000_max": 0.47121346840093437, "nauc_mrr_at_1000_std": 0.01761850507064754, "nauc_mrr_at_100_diff1": 0.5818862178246292, "nauc_mrr_at_100_max": 0.4702677970629844, "nauc_mrr_at_100_std": 0.01732113924124195, "nauc_mrr_at_10_diff1": 0.5854386442392447, "nauc_mrr_at_10_max": 0.47577814442978594, "nauc_mrr_at_10_std": 0.013683084601007884, "nauc_mrr_at_1_diff1": 0.6303121271600662, "nauc_mrr_at_1_max": 0.45306546588153795, "nauc_mrr_at_1_std": -0.03716276026632798, "nauc_mrr_at_20_diff1": 0.5828572421752594, "nauc_mrr_at_20_max": 0.46939532577476534, "nauc_mrr_at_20_std": 0.012200111756481068, "nauc_mrr_at_3_diff1": 0.5953459236004133, "nauc_mrr_at_3_max": 0.4673015424214653, "nauc_mrr_at_3_std": -0.007863229530253665, "nauc_mrr_at_5_diff1": 0.596001269055143, "nauc_mrr_at_5_max": 0.47341165290086296, "nauc_mrr_at_5_std": 0.005731512744635958, "nauc_ndcg_at_1000_diff1": 0.5519537330473182, "nauc_ndcg_at_1000_max": 0.4776051956831099, "nauc_ndcg_at_1000_std": 0.07098420071862646, "nauc_ndcg_at_100_diff1": 0.5358392865731979, "nauc_ndcg_at_100_max": 0.46350611930764696, "nauc_ndcg_at_100_std": 0.07691855816662034, "nauc_ndcg_at_10_diff1": 0.558761866391538, "nauc_ndcg_at_10_max": 0.4815954148923924, "nauc_ndcg_at_10_std": 0.04145292681314251, "nauc_ndcg_at_1_diff1": 0.6303121271600662, "nauc_ndcg_at_1_max": 0.45306546588153795, "nauc_ndcg_at_1_std": -0.03716276026632798, "nauc_ndcg_at_20_diff1": 0.5458553168628171, "nauc_ndcg_at_20_max": 0.45640758615377236, "nauc_ndcg_at_20_std": 0.037411713002657526, "nauc_ndcg_at_3_diff1": 0.581931256292059, "nauc_ndcg_at_3_max": 0.4672095603839993, "nauc_ndcg_at_3_std": 0.0010732582708155496, "nauc_ndcg_at_5_diff1": 0.5828683592678144, "nauc_ndcg_at_5_max": 0.4768678411192577, "nauc_ndcg_at_5_std": 0.023373647848388267, "nauc_precision_at_1000_diff1": 0.22665732959850426, "nauc_precision_at_1000_max": 0.6512313258636786, "nauc_precision_at_1000_std": 0.7869105975723624, "nauc_precision_at_100_diff1": 0.31753884685256706, "nauc_precision_at_100_max": 0.4285852866994729, "nauc_precision_at_100_std": 0.3884332504365306, "nauc_precision_at_10_diff1": 0.4804829757538381, "nauc_precision_at_10_max": 0.49708851309682217, "nauc_precision_at_10_std": 0.12169171346035226, "nauc_precision_at_1_diff1": 0.6303121271600662, "nauc_precision_at_1_max": 0.45306546588153795, "nauc_precision_at_1_std": -0.03716276026632798, "nauc_precision_at_20_diff1": 0.42342438374417013, "nauc_precision_at_20_max": 0.39866755496335765, "nauc_precision_at_20_std": 0.11081279147235165, "nauc_precision_at_3_diff1": 0.544668488935896, "nauc_precision_at_3_max": 0.46566762358694047, "nauc_precision_at_3_std": 0.025362965811848406, "nauc_precision_at_5_diff1": 0.5472598349226488, "nauc_precision_at_5_max": 0.48527983188279383, "nauc_precision_at_5_std": 0.0718595322588108, "nauc_recall_at_1000_diff1": 0.2266573295985049, "nauc_recall_at_1000_max": 0.6512313258636785, "nauc_recall_at_1000_std": 0.7869105975723616, "nauc_recall_at_100_diff1": 0.3175388468525674, "nauc_recall_at_100_max": 0.42858528669947293, "nauc_recall_at_100_std": 0.38843325043653104, "nauc_recall_at_10_diff1": 0.4804829757538381, "nauc_recall_at_10_max": 0.4970885130968222, "nauc_recall_at_10_std": 0.12169171346035257, "nauc_recall_at_1_diff1": 0.6303121271600662, "nauc_recall_at_1_max": 0.45306546588153795, "nauc_recall_at_1_std": -0.03716276026632798, "nauc_recall_at_20_diff1": 0.4234243837441704, "nauc_recall_at_20_max": 0.39866755496335765, "nauc_recall_at_20_std": 0.11081279147235173, "nauc_recall_at_3_diff1": 0.5446684889358954, "nauc_recall_at_3_max": 0.46566762358694014, "nauc_recall_at_3_std": 0.0253629658118484, "nauc_recall_at_5_diff1": 0.5472598349226487, "nauc_recall_at_5_max": 0.4852798318827938, "nauc_recall_at_5_std": 0.07185953225881045, "ndcg_at_1": 0.225, "ndcg_at_10": 0.31108, "ndcg_at_100": 0.36408, "ndcg_at_1000": 0.3922, "ndcg_at_20": 0.32913, "ndcg_at_3": 0.26274, "ndcg_at_5": 0.28489, "precision_at_1": 0.225, "precision_at_10": 0.0425, "precision_at_100": 0.0069, "precision_at_1000": 0.00092, "precision_at_20": 0.025, "precision_at_3": 0.09667, "precision_at_5": 0.069, "recall_at_1": 0.225, "recall_at_10": 0.425, "recall_at_100": 0.69, "recall_at_1000": 0.92, "recall_at_20": 0.5, "recall_at_3": 0.29, "recall_at_5": 0.345 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.28479, "map_at_1": 0.2, "map_at_10": 0.25486, "map_at_100": 0.26535, "map_at_1000": 0.26645, "map_at_20": 0.26155, "map_at_3": 0.23917, "map_at_5": 0.24692, "mrr_at_1": 0.2, "mrr_at_10": 0.2548571428571429, "mrr_at_100": 0.2653479973134919, "mrr_at_1000": 0.2664475273861746, "mrr_at_20": 0.26155286766896685, "mrr_at_3": 0.23916666666666672, "mrr_at_5": 0.2469166666666667, "nauc_map_at_1000_diff1": 0.4582594378042708, "nauc_map_at_1000_max": 0.3948624842187872, "nauc_map_at_1000_std": -0.03468906962815208, "nauc_map_at_100_diff1": 0.457657589855601, "nauc_map_at_100_max": 0.3943825734840979, "nauc_map_at_100_std": -0.03445673443990469, "nauc_map_at_10_diff1": 0.4609797691297197, "nauc_map_at_10_max": 0.39313889251279743, "nauc_map_at_10_std": -0.04730129167400886, "nauc_map_at_1_diff1": 0.4804246044559252, "nauc_map_at_1_max": 0.38819825637713934, "nauc_map_at_1_std": -0.06094607684856327, "nauc_map_at_20_diff1": 0.45850715411026194, "nauc_map_at_20_max": 0.39479073700656353, "nauc_map_at_20_std": -0.03687865755487684, "nauc_map_at_3_diff1": 0.47511215498825665, "nauc_map_at_3_max": 0.39136309716515716, "nauc_map_at_3_std": -0.073459173584297, "nauc_map_at_5_diff1": 0.46072255066461987, "nauc_map_at_5_max": 0.38753280167396637, "nauc_map_at_5_std": -0.06931887538819474, "nauc_mrr_at_1000_diff1": 0.4582594378042708, "nauc_mrr_at_1000_max": 0.3948624842187872, "nauc_mrr_at_1000_std": -0.03468906962815208, "nauc_mrr_at_100_diff1": 0.457657589855601, "nauc_mrr_at_100_max": 0.3943825734840979, "nauc_mrr_at_100_std": -0.03445673443990469, "nauc_mrr_at_10_diff1": 0.4609797691297197, "nauc_mrr_at_10_max": 0.39313889251279743, "nauc_mrr_at_10_std": -0.04730129167400886, "nauc_mrr_at_1_diff1": 0.4804246044559252, "nauc_mrr_at_1_max": 0.38819825637713934, "nauc_mrr_at_1_std": -0.06094607684856327, "nauc_mrr_at_20_diff1": 0.45850715411026194, "nauc_mrr_at_20_max": 0.39479073700656353, "nauc_mrr_at_20_std": -0.03687865755487684, "nauc_mrr_at_3_diff1": 0.47511215498825665, "nauc_mrr_at_3_max": 0.39136309716515716, "nauc_mrr_at_3_std": -0.073459173584297, "nauc_mrr_at_5_diff1": 0.46072255066461987, "nauc_mrr_at_5_max": 0.38753280167396637, "nauc_mrr_at_5_std": -0.06931887538819474, "nauc_ndcg_at_1000_diff1": 0.4488569762316068, "nauc_ndcg_at_1000_max": 0.40849512245632413, "nauc_ndcg_at_1000_std": 0.027126231441878132, "nauc_ndcg_at_100_diff1": 0.4363550168495924, "nauc_ndcg_at_100_max": 0.40152755948454205, "nauc_ndcg_at_100_std": 0.036757853379914364, "nauc_ndcg_at_10_diff1": 0.44503079744042595, "nauc_ndcg_at_10_max": 0.39587888961386797, "nauc_ndcg_at_10_std": -0.017562145039247794, "nauc_ndcg_at_1_diff1": 0.4804246044559252, "nauc_ndcg_at_1_max": 0.38819825637713934, "nauc_ndcg_at_1_std": -0.06094607684856327, "nauc_ndcg_at_20_diff1": 0.43562189371533044, "nauc_ndcg_at_20_max": 0.4023564273040959, "nauc_ndcg_at_20_std": 0.020538959255559326, "nauc_ndcg_at_3_diff1": 0.4686025661010145, "nauc_ndcg_at_3_max": 0.3898646738127454, "nauc_ndcg_at_3_std": -0.0745048617099372, "nauc_ndcg_at_5_diff1": 0.44299593443243157, "nauc_ndcg_at_5_max": 0.38332430221771063, "nauc_ndcg_at_5_std": -0.066726525440986, "nauc_precision_at_1000_diff1": 0.5314439501290708, "nauc_precision_at_1000_max": 0.6654858021640021, "nauc_precision_at_1000_std": 0.7862086010874936, "nauc_precision_at_100_diff1": 0.37186915127349696, "nauc_precision_at_100_max": 0.4383106843518594, "nauc_precision_at_100_std": 0.3533810725038655, "nauc_precision_at_10_diff1": 0.40163231357055595, "nauc_precision_at_10_max": 0.4053037194915917, "nauc_precision_at_10_std": 0.0749551633613182, "nauc_precision_at_1_diff1": 0.4804246044559252, "nauc_precision_at_1_max": 0.38819825637713934, "nauc_precision_at_1_std": -0.06094607684856327, "nauc_precision_at_20_diff1": 0.3656232281543797, "nauc_precision_at_20_max": 0.4300689618797718, "nauc_precision_at_20_std": 0.21663655902635845, "nauc_precision_at_3_diff1": 0.44994024971498453, "nauc_precision_at_3_max": 0.38519703858357507, "nauc_precision_at_3_std": -0.07651470406439288, "nauc_precision_at_5_diff1": 0.39206566347469224, "nauc_precision_at_5_max": 0.37099195889359715, "nauc_precision_at_5_std": -0.05794267792199136, "nauc_recall_at_1000_diff1": 0.5314439501290714, "nauc_recall_at_1000_max": 0.6654858021640025, "nauc_recall_at_1000_std": 0.7862086010874946, "nauc_recall_at_100_diff1": 0.3718691512734966, "nauc_recall_at_100_max": 0.4383106843518591, "nauc_recall_at_100_std": 0.35338107250386497, "nauc_recall_at_10_diff1": 0.4016323135705559, "nauc_recall_at_10_max": 0.4053037194915914, "nauc_recall_at_10_std": 0.07495516336131847, "nauc_recall_at_1_diff1": 0.4804246044559252, "nauc_recall_at_1_max": 0.38819825637713934, "nauc_recall_at_1_std": -0.06094607684856327, "nauc_recall_at_20_diff1": 0.3656232281543799, "nauc_recall_at_20_max": 0.43006896187977184, "nauc_recall_at_20_std": 0.21663655902635856, "nauc_recall_at_3_diff1": 0.44994024971498436, "nauc_recall_at_3_max": 0.3851970385835748, "nauc_recall_at_3_std": -0.07651470406439295, "nauc_recall_at_5_diff1": 0.39206566347469235, "nauc_recall_at_5_max": 0.3709919588935971, "nauc_recall_at_5_std": -0.057942677921991474, "ndcg_at_1": 0.2, "ndcg_at_10": 0.28479, "ndcg_at_100": 0.33976, "ndcg_at_1000": 0.37259, "ndcg_at_20": 0.3081, "ndcg_at_3": 0.2522, "ndcg_at_5": 0.2664, "precision_at_1": 0.2, "precision_at_10": 0.038, "precision_at_100": 0.0065, "precision_at_1000": 0.00092, "precision_at_20": 0.0235, "precision_at_3": 0.09667, "precision_at_5": 0.065, "recall_at_1": 0.2, "recall_at_10": 0.38, "recall_at_100": 0.65, "recall_at_1000": 0.915, "recall_at_20": 0.47, "recall_at_3": 0.29, "recall_at_5": 0.325 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 5.753293514251709, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.85617, "map_at_1": 0.72727, "map_at_10": 0.818, "map_at_100": 0.81972, "map_at_1000": 0.81972, "map_at_20": 0.81972, "map_at_3": 0.80519, "map_at_5": 0.80844, "mrr_at_1": 0.7272727272727273, "mrr_at_10": 0.8180014430014431, "mrr_at_100": 0.8197219447219448, "mrr_at_1000": 0.8197219447219448, "mrr_at_20": 0.8197219447219448, "mrr_at_3": 0.8051948051948052, "mrr_at_5": 0.8084415584415584, "nauc_map_at_1000_diff1": 0.646244043446091, "nauc_map_at_1000_max": 0.11771466620451258, "nauc_map_at_1000_std": -0.22293982162876316, "nauc_map_at_100_diff1": 0.646244043446091, "nauc_map_at_100_max": 0.11771466620451258, "nauc_map_at_100_std": -0.22293982162876316, "nauc_map_at_10_diff1": 0.642856708479778, "nauc_map_at_10_max": 0.11035422874096561, "nauc_map_at_10_std": -0.22329349236179952, "nauc_map_at_1_diff1": 0.6496291120568579, "nauc_map_at_1_max": 0.08105503859326908, "nauc_map_at_1_std": -0.15869430013902078, "nauc_map_at_20_diff1": 0.646244043446091, "nauc_map_at_20_max": 0.11771466620451258, "nauc_map_at_20_std": -0.22293982162876316, "nauc_map_at_3_diff1": 0.6487588838089147, "nauc_map_at_3_max": 0.13649166333805868, "nauc_map_at_3_std": -0.24378692695065055, "nauc_map_at_5_diff1": 0.6422022858465769, "nauc_map_at_5_max": 0.13669426466557213, "nauc_map_at_5_std": -0.2378759106262703, "nauc_mrr_at_1000_diff1": 0.646244043446091, "nauc_mrr_at_1000_max": 0.11771466620451258, "nauc_mrr_at_1000_std": -0.22293982162876316, "nauc_mrr_at_100_diff1": 0.646244043446091, "nauc_mrr_at_100_max": 0.11771466620451258, "nauc_mrr_at_100_std": -0.22293982162876316, "nauc_mrr_at_10_diff1": 0.642856708479778, "nauc_mrr_at_10_max": 0.11035422874096561, "nauc_mrr_at_10_std": -0.22329349236179952, "nauc_mrr_at_1_diff1": 0.6496291120568579, "nauc_mrr_at_1_max": 0.08105503859326908, "nauc_mrr_at_1_std": -0.15869430013902078, "nauc_mrr_at_20_diff1": 0.646244043446091, "nauc_mrr_at_20_max": 0.11771466620451258, "nauc_mrr_at_20_std": -0.22293982162876316, "nauc_mrr_at_3_diff1": 0.6487588838089147, "nauc_mrr_at_3_max": 0.13649166333805868, "nauc_mrr_at_3_std": -0.24378692695065055, "nauc_mrr_at_5_diff1": 0.6422022858465769, "nauc_mrr_at_5_max": 0.13669426466557213, "nauc_mrr_at_5_std": -0.2378759106262703, "nauc_ndcg_at_1000_diff1": 0.6454336670366856, "nauc_ndcg_at_1000_max": 0.1204399199301231, "nauc_ndcg_at_1000_std": -0.22851662405485756, "nauc_ndcg_at_100_diff1": 0.6454336670366856, "nauc_ndcg_at_100_max": 0.1204399199301231, "nauc_ndcg_at_100_std": -0.22851662405485756, "nauc_ndcg_at_10_diff1": 0.6325430114209823, "nauc_ndcg_at_10_max": 0.09272330063627189, "nauc_ndcg_at_10_std": -0.22420639498410985, "nauc_ndcg_at_1_diff1": 0.6496291120568579, "nauc_ndcg_at_1_max": 0.08105503859326908, "nauc_ndcg_at_1_std": -0.15869430013902078, "nauc_ndcg_at_20_diff1": 0.6454336670366856, "nauc_ndcg_at_20_max": 0.1204399199301231, "nauc_ndcg_at_20_std": -0.22851662405485756, "nauc_ndcg_at_3_diff1": 0.6474924583756264, "nauc_ndcg_at_3_max": 0.16124916234667075, "nauc_ndcg_at_3_std": -0.2811007748212722, "nauc_ndcg_at_5_diff1": 0.6345773846760506, "nauc_ndcg_at_5_max": 0.1625538675994191, "nauc_ndcg_at_5_std": -0.2708662815048982, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.4167090338610833, "nauc_precision_at_10_max": -0.36527093203830124, "nauc_precision_at_10_std": -0.08218269674418595, "nauc_precision_at_1_diff1": 0.6496291120568579, "nauc_precision_at_1_max": 0.08105503859326908, "nauc_precision_at_1_std": -0.15869430013902078, "nauc_precision_at_20_diff1": 1.0, "nauc_precision_at_20_max": 1.0, "nauc_precision_at_20_std": 1.0, "nauc_precision_at_3_diff1": 0.6408580232419018, "nauc_precision_at_3_max": 0.28213621696446, "nauc_precision_at_3_std": -0.46294543948589373, "nauc_precision_at_5_diff1": 0.5895520265621751, "nauc_precision_at_5_max": 0.30449307916307494, "nauc_precision_at_5_std": -0.4490170987180294, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.41670903386108427, "nauc_recall_at_10_max": -0.3652709320383255, "nauc_recall_at_10_std": -0.08218269674420323, "nauc_recall_at_1_diff1": 0.6496291120568579, "nauc_recall_at_1_max": 0.08105503859326908, "nauc_recall_at_1_std": -0.15869430013902078, "nauc_recall_at_20_diff1": NaN, "nauc_recall_at_20_max": NaN, "nauc_recall_at_20_std": NaN, "nauc_recall_at_3_diff1": 0.6408580232419027, "nauc_recall_at_3_max": 0.28213621696445973, "nauc_recall_at_3_std": -0.4629454394858956, "nauc_recall_at_5_diff1": 0.5895520265621734, "nauc_recall_at_5_max": 0.3044930791630734, "nauc_recall_at_5_std": -0.4490170987180282, "ndcg_at_1": 0.72727, "ndcg_at_10": 0.85617, "ndcg_at_100": 0.86264, "ndcg_at_1000": 0.86264, "ndcg_at_20": 0.86264, "ndcg_at_3": 0.82869, "ndcg_at_5": 0.83429, "precision_at_1": 0.72727, "precision_at_10": 0.0974, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.05, "precision_at_3": 0.2987, "precision_at_5": 0.18182, "recall_at_1": 0.72727, "recall_at_10": 0.97403, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 1.0, "recall_at_3": 0.8961, "recall_at_5": 0.90909 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 127.63971376419067, "kg_co2_emissions": null, "mteb_version": "1.20.1", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.37423, "map_at_1": 0.20652, "map_at_10": 0.32424, "map_at_100": 0.34083, "map_at_1000": 0.34195, "map_at_20": 0.33307, "map_at_3": 0.29049, "map_at_5": 0.30924, "mrr_at_1": 0.2889908256880734, "mrr_at_10": 0.36114569681083425, "mrr_at_100": 0.3735580971418379, "mrr_at_1000": 0.37422727602805517, "mrr_at_20": 0.36821475588783903, "mrr_at_3": 0.3363914373088686, "mrr_at_5": 0.3509174311926605, "nauc_map_at_1000_diff1": 0.47524766111826294, "nauc_map_at_1000_max": 0.09883153525011047, "nauc_map_at_1000_std": -0.32764285382136416, "nauc_map_at_100_diff1": 0.4744188521819488, "nauc_map_at_100_max": 0.09861768287384905, "nauc_map_at_100_std": -0.3273774197260536, "nauc_map_at_10_diff1": 0.47796628713312844, "nauc_map_at_10_max": 0.09671833361089476, "nauc_map_at_10_std": -0.33068239680612016, "nauc_map_at_1_diff1": 0.5312068574156018, "nauc_map_at_1_max": 0.08316273395496018, "nauc_map_at_1_std": -0.2770183950295717, "nauc_map_at_20_diff1": 0.4754634006576872, "nauc_map_at_20_max": 0.09667059424098755, "nauc_map_at_20_std": -0.33196629807587275, "nauc_map_at_3_diff1": 0.48320602714004945, "nauc_map_at_3_max": 0.10272353700709959, "nauc_map_at_3_std": -0.3222699822858497, "nauc_map_at_5_diff1": 0.47935771756155027, "nauc_map_at_5_max": 0.0936269166017143, "nauc_map_at_5_std": -0.3279742827497809, "nauc_mrr_at_1000_diff1": 0.49733197428074555, "nauc_mrr_at_1000_max": 0.096006678397459, "nauc_mrr_at_1000_std": -0.3391850920791737, "nauc_mrr_at_100_diff1": 0.4970710841874183, "nauc_mrr_at_100_max": 0.09609958107752674, "nauc_mrr_at_100_std": -0.3388351833013782, "nauc_mrr_at_10_diff1": 0.49583599307300874, "nauc_mrr_at_10_max": 0.09263389688314552, "nauc_mrr_at_10_std": -0.34279253646801777, "nauc_mrr_at_1_diff1": 0.530507552915156, "nauc_mrr_at_1_max": 0.11081685259047605, "nauc_mrr_at_1_std": -0.336591986082507, "nauc_mrr_at_20_diff1": 0.4974467054890617, "nauc_mrr_at_20_max": 0.09442546029359035, "nauc_mrr_at_20_std": -0.3415754228379661, "nauc_mrr_at_3_diff1": 0.5005066657676188, "nauc_mrr_at_3_max": 0.09523918796369456, "nauc_mrr_at_3_std": -0.34511618876714556, "nauc_mrr_at_5_diff1": 0.4951612263800776, "nauc_mrr_at_5_max": 0.09065455861736223, "nauc_mrr_at_5_std": -0.3404558034485618, "nauc_ndcg_at_1000_diff1": 0.4712828373716081, "nauc_ndcg_at_1000_max": 0.10383926602469376, "nauc_ndcg_at_1000_std": -0.3214816132955133, "nauc_ndcg_at_100_diff1": 0.4553816037095231, "nauc_ndcg_at_100_max": 0.10841608065924647, "nauc_ndcg_at_100_std": -0.30666351560829147, "nauc_ndcg_at_10_diff1": 0.4690219510049616, "nauc_ndcg_at_10_max": 0.08740305491826425, "nauc_ndcg_at_10_std": -0.3417997128940579, "nauc_ndcg_at_1_diff1": 0.530507552915156, "nauc_ndcg_at_1_max": 0.11081685259047605, "nauc_ndcg_at_1_std": -0.336591986082507, "nauc_ndcg_at_20_diff1": 0.46514937364526887, "nauc_ndcg_at_20_max": 0.09016332315940649, "nauc_ndcg_at_20_std": -0.34221828530596565, "nauc_ndcg_at_3_diff1": 0.4751896909410173, "nauc_ndcg_at_3_max": 0.09018621762145725, "nauc_ndcg_at_3_std": -0.33853718672197586, "nauc_ndcg_at_5_diff1": 0.47079315412116823, "nauc_ndcg_at_5_max": 0.0819286070811535, "nauc_ndcg_at_5_std": -0.3355510330595144, "nauc_precision_at_1000_diff1": -0.005767266391941639, "nauc_precision_at_1000_max": 0.10655339932047396, "nauc_precision_at_1000_std": 0.0034373965971917554, "nauc_precision_at_100_diff1": 0.04565703614267389, "nauc_precision_at_100_max": 0.12321925001636382, "nauc_precision_at_100_std": -0.028986377035503448, "nauc_precision_at_10_diff1": 0.2606865910429876, "nauc_precision_at_10_max": 0.0690376536277112, "nauc_precision_at_10_std": -0.3017927914358375, "nauc_precision_at_1_diff1": 0.530507552915156, "nauc_precision_at_1_max": 0.11081685259047605, "nauc_precision_at_1_std": -0.336591986082507, "nauc_precision_at_20_diff1": 0.2032930893089947, "nauc_precision_at_20_max": 0.07073110216590488, "nauc_precision_at_20_std": -0.26140795871276373, "nauc_precision_at_3_diff1": 0.3338599510767471, "nauc_precision_at_3_max": 0.10242310759589818, "nauc_precision_at_3_std": -0.32842806479168957, "nauc_precision_at_5_diff1": 0.29836479908807134, "nauc_precision_at_5_max": 0.0777515735924914, "nauc_precision_at_5_std": -0.31047088963223834, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.2317669452615089, "nauc_recall_at_100_max": 0.20594504302420488, "nauc_recall_at_100_std": -0.05681131555526767, "nauc_recall_at_10_diff1": 0.4102379566937134, "nauc_recall_at_10_max": 0.06361937146897327, "nauc_recall_at_10_std": -0.34785219614907414, "nauc_recall_at_1_diff1": 0.5312068574156018, "nauc_recall_at_1_max": 0.08316273395496018, "nauc_recall_at_1_std": -0.2770183950295717, "nauc_recall_at_20_diff1": 0.389303726811902, "nauc_recall_at_20_max": 0.07067454954471056, "nauc_recall_at_20_std": -0.3490048604348846, "nauc_recall_at_3_diff1": 0.43805948201691913, "nauc_recall_at_3_max": 0.07672959491844454, "nauc_recall_at_3_std": -0.3312609114334921, "nauc_recall_at_5_diff1": 0.41987819758641604, "nauc_recall_at_5_max": 0.04949803095594345, "nauc_recall_at_5_std": -0.3334354339134594, "ndcg_at_1": 0.28899, "ndcg_at_10": 0.37423, "ndcg_at_100": 0.45146, "ndcg_at_1000": 0.47572, "ndcg_at_20": 0.40221, "ndcg_at_3": 0.319, "ndcg_at_5": 0.34348, "precision_at_1": 0.28899, "precision_at_10": 0.07768, "precision_at_100": 0.01317, "precision_at_1000": 0.00156, "precision_at_20": 0.04656, "precision_at_3": 0.17788, "precision_at_5": 0.12783, "recall_at_1": 0.20652, "recall_at_10": 0.48629, "recall_at_100": 0.82454, "recall_at_1000": 1.0, "recall_at_20": 0.58496, "recall_at_3": 0.34088, "recall_at_5": 0.40183 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.3659, "map_at_1": 0.18038, "map_at_10": 0.31357, "map_at_100": 0.3329, "map_at_1000": 0.33458, "map_at_20": 0.3239, "map_at_3": 0.26744, "map_at_5": 0.29445, "mrr_at_1": 0.2996941896024465, "mrr_at_10": 0.36893172661521256, "mrr_at_100": 0.37999973579962687, "mrr_at_1000": 0.38081355802900246, "mrr_at_20": 0.3751262464551014, "mrr_at_3": 0.34480122324159035, "mrr_at_5": 0.35856269113149836, "nauc_map_at_1000_diff1": 0.5475845862896238, "nauc_map_at_1000_max": 0.17129985115486587, "nauc_map_at_1000_std": -0.31192134531931526, "nauc_map_at_100_diff1": 0.5472658081934074, "nauc_map_at_100_max": 0.1702211431621068, "nauc_map_at_100_std": -0.31310792303685064, "nauc_map_at_10_diff1": 0.551458713298644, "nauc_map_at_10_max": 0.17502023070573752, "nauc_map_at_10_std": -0.3108407143933329, "nauc_map_at_1_diff1": 0.5812609502108893, "nauc_map_at_1_max": 0.14606228480022937, "nauc_map_at_1_std": -0.2755845356500218, "nauc_map_at_20_diff1": 0.5491856326088752, "nauc_map_at_20_max": 0.17151836955292996, "nauc_map_at_20_std": -0.3146971451556958, "nauc_map_at_3_diff1": 0.5632033415363726, "nauc_map_at_3_max": 0.16810213892994397, "nauc_map_at_3_std": -0.3018261794541308, "nauc_map_at_5_diff1": 0.5541230617030946, "nauc_map_at_5_max": 0.1752806060272934, "nauc_map_at_5_std": -0.30863452026905913, "nauc_mrr_at_1000_diff1": 0.5319526790628266, "nauc_mrr_at_1000_max": 0.15798458903661572, "nauc_mrr_at_1000_std": -0.3161777593887362, "nauc_mrr_at_100_diff1": 0.5314497575661327, "nauc_mrr_at_100_max": 0.1579687609307439, "nauc_mrr_at_100_std": -0.3158669687764293, "nauc_mrr_at_10_diff1": 0.5330646219432362, "nauc_mrr_at_10_max": 0.15607701799166418, "nauc_mrr_at_10_std": -0.3216853173474818, "nauc_mrr_at_1_diff1": 0.5777369192363048, "nauc_mrr_at_1_max": 0.17996075609578352, "nauc_mrr_at_1_std": -0.3055325778055966, "nauc_mrr_at_20_diff1": 0.5312353932468608, "nauc_mrr_at_20_max": 0.15651230811629652, "nauc_mrr_at_20_std": -0.3182475877521702, "nauc_mrr_at_3_diff1": 0.5443148099829979, "nauc_mrr_at_3_max": 0.158954825500078, "nauc_mrr_at_3_std": -0.3192038118068102, "nauc_mrr_at_5_diff1": 0.5343630391590962, "nauc_mrr_at_5_max": 0.15514715460628098, "nauc_mrr_at_5_std": -0.3220709171492349, "nauc_ndcg_at_1000_diff1": 0.5202673417802097, "nauc_ndcg_at_1000_max": 0.16310877019932138, "nauc_ndcg_at_1000_std": -0.3045619273813806, "nauc_ndcg_at_100_diff1": 0.5102621009967564, "nauc_ndcg_at_100_max": 0.1554204416240501, "nauc_ndcg_at_100_std": -0.30558837713364984, "nauc_ndcg_at_10_diff1": 0.5301561213420717, "nauc_ndcg_at_10_max": 0.16256242494569756, "nauc_ndcg_at_10_std": -0.3197634187745068, "nauc_ndcg_at_1_diff1": 0.5777369192363048, "nauc_ndcg_at_1_max": 0.17996075609578352, "nauc_ndcg_at_1_std": -0.3055325778055966, "nauc_ndcg_at_20_diff1": 0.5195791786524678, "nauc_ndcg_at_20_max": 0.15426717535597573, "nauc_ndcg_at_20_std": -0.3234141170942771, "nauc_ndcg_at_3_diff1": 0.5439887228884779, "nauc_ndcg_at_3_max": 0.15863062376493972, "nauc_ndcg_at_3_std": -0.31517249866480807, "nauc_ndcg_at_5_diff1": 0.5320369390007491, "nauc_ndcg_at_5_max": 0.16414407022065952, "nauc_ndcg_at_5_std": -0.31603967683181655, "nauc_precision_at_1000_diff1": -0.01972151472070089, "nauc_precision_at_1000_max": 0.04885050874249441, "nauc_precision_at_1000_std": -0.011980239281004945, "nauc_precision_at_100_diff1": 0.07956393359823302, "nauc_precision_at_100_max": 0.05148451548152324, "nauc_precision_at_100_std": -0.10063971949175372, "nauc_precision_at_10_diff1": 0.3016511844169242, "nauc_precision_at_10_max": 0.11760651902070267, "nauc_precision_at_10_std": -0.25074452316449297, "nauc_precision_at_1_diff1": 0.5777369192363048, "nauc_precision_at_1_max": 0.17996075609578352, "nauc_precision_at_1_std": -0.3055325778055966, "nauc_precision_at_20_diff1": 0.23822808536130927, "nauc_precision_at_20_max": 0.08461029451434389, "nauc_precision_at_20_std": -0.23469224953329992, "nauc_precision_at_3_diff1": 0.4023597377637611, "nauc_precision_at_3_max": 0.14598522494350358, "nauc_precision_at_3_std": -0.2759980327072713, "nauc_precision_at_5_diff1": 0.34710591102430166, "nauc_precision_at_5_max": 0.13793631724712827, "nauc_precision_at_5_std": -0.2653965283294146, "nauc_recall_at_1000_diff1": -0.21622330025083975, "nauc_recall_at_1000_max": -0.17411302595557848, "nauc_recall_at_1000_std": -0.46808130028818207, "nauc_recall_at_100_diff1": 0.3248732360733499, "nauc_recall_at_100_max": 0.09632461146835207, "nauc_recall_at_100_std": -0.21395135506452787, "nauc_recall_at_10_diff1": 0.4661512723671577, "nauc_recall_at_10_max": 0.14417075776566576, "nauc_recall_at_10_std": -0.31186651082212025, "nauc_recall_at_1_diff1": 0.5812609502108893, "nauc_recall_at_1_max": 0.14606228480022937, "nauc_recall_at_1_std": -0.2755845356500218, "nauc_recall_at_20_diff1": 0.4243012119763299, "nauc_recall_at_20_max": 0.11425505582759143, "nauc_recall_at_20_std": -0.32209030412851514, "nauc_recall_at_3_diff1": 0.5128831174573368, "nauc_recall_at_3_max": 0.14039851355316052, "nauc_recall_at_3_std": -0.3051519092441379, "nauc_recall_at_5_diff1": 0.47049871862950743, "nauc_recall_at_5_max": 0.14809208942815275, "nauc_recall_at_5_std": -0.3090292790912479, "ndcg_at_1": 0.29969, "ndcg_at_10": 0.3659, "ndcg_at_100": 0.44749, "ndcg_at_1000": 0.47656, "ndcg_at_20": 0.39505, "ndcg_at_3": 0.31337, "ndcg_at_5": 0.33242, "precision_at_1": 0.29969, "precision_at_10": 0.09572, "precision_at_100": 0.01619, "precision_at_1000": 0.00194, "precision_at_20": 0.05772, "precision_at_3": 0.20489, "precision_at_5": 0.15474, "recall_at_1": 0.18038, "recall_at_10": 0.45609, "recall_at_100": 0.79796, "recall_at_1000": 0.9949, "recall_at_20": 0.55099, "recall_at_3": 0.30647, "recall_at_5": 0.3726 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.37028, "map_at_1": 0.19803, "map_at_10": 0.31863, "map_at_100": 0.33429, "map_at_1000": 0.33557, "map_at_20": 0.3257, "map_at_3": 0.28057, "map_at_5": 0.29986, "mrr_at_1": 0.28664495114006516, "mrr_at_10": 0.35963238715681695, "mrr_at_100": 0.37055198162289404, "mrr_at_1000": 0.3713710790315084, "mrr_at_20": 0.36522946934537664, "mrr_at_3": 0.3330618892508144, "mrr_at_5": 0.3460097719869707, "nauc_map_at_1000_diff1": 0.45384537054326274, "nauc_map_at_1000_max": 0.06941323095795682, "nauc_map_at_1000_std": -0.3190300006388068, "nauc_map_at_100_diff1": 0.45331188809340084, "nauc_map_at_100_max": 0.06974115300695743, "nauc_map_at_100_std": -0.31834923704324725, "nauc_map_at_10_diff1": 0.4548177046458187, "nauc_map_at_10_max": 0.06609084456502891, "nauc_map_at_10_std": -0.3263912570727043, "nauc_map_at_1_diff1": 0.5411729166156709, "nauc_map_at_1_max": 0.04965791121034343, "nauc_map_at_1_std": -0.2770971704111151, "nauc_map_at_20_diff1": 0.4550026878212889, "nauc_map_at_20_max": 0.06544234925863715, "nauc_map_at_20_std": -0.32794064880249657, "nauc_map_at_3_diff1": 0.46628565155101115, "nauc_map_at_3_max": 0.06837861613381846, "nauc_map_at_3_std": -0.30792101181267695, "nauc_map_at_5_diff1": 0.45639933652657977, "nauc_map_at_5_max": 0.06699970410481666, "nauc_map_at_5_std": -0.3176249865537211, "nauc_mrr_at_1000_diff1": 0.46678779520345426, "nauc_mrr_at_1000_max": 0.06683354615752005, "nauc_mrr_at_1000_std": -0.3098309567636614, "nauc_mrr_at_100_diff1": 0.4663617454366421, "nauc_mrr_at_100_max": 0.06742196992757753, "nauc_mrr_at_100_std": -0.30897623574808364, "nauc_mrr_at_10_diff1": 0.46575145011988317, "nauc_mrr_at_10_max": 0.06308966595195759, "nauc_mrr_at_10_std": -0.316220345622948, "nauc_mrr_at_1_diff1": 0.5217613398284591, "nauc_mrr_at_1_max": 0.06723111294341066, "nauc_mrr_at_1_std": -0.30864470486943385, "nauc_mrr_at_20_diff1": 0.4670769592041788, "nauc_mrr_at_20_max": 0.06530971752756526, "nauc_mrr_at_20_std": -0.31389838093131917, "nauc_mrr_at_3_diff1": 0.47668011938645516, "nauc_mrr_at_3_max": 0.06650270899588769, "nauc_mrr_at_3_std": -0.30852685782572564, "nauc_mrr_at_5_diff1": 0.4690693607040199, "nauc_mrr_at_5_max": 0.06325125895453224, "nauc_mrr_at_5_std": -0.3123463685542158, "nauc_ndcg_at_1000_diff1": 0.4425089332119335, "nauc_ndcg_at_1000_max": 0.0813733799435709, "nauc_ndcg_at_1000_std": -0.30069417441181623, "nauc_ndcg_at_100_diff1": 0.4277002642353046, "nauc_ndcg_at_100_max": 0.09225279866642008, "nauc_ndcg_at_100_std": -0.2763152217911081, "nauc_ndcg_at_10_diff1": 0.4357781494862851, "nauc_ndcg_at_10_max": 0.06497312147959357, "nauc_ndcg_at_10_std": -0.3347075429482461, "nauc_ndcg_at_1_diff1": 0.5217613398284591, "nauc_ndcg_at_1_max": 0.06723111294341066, "nauc_ndcg_at_1_std": -0.30864470486943385, "nauc_ndcg_at_20_diff1": 0.437615883595116, "nauc_ndcg_at_20_max": 0.06585660117698056, "nauc_ndcg_at_20_std": -0.3360552708508598, "nauc_ndcg_at_3_diff1": 0.46016302280312, "nauc_ndcg_at_3_max": 0.06306325920843273, "nauc_ndcg_at_3_std": -0.31453094531104747, "nauc_ndcg_at_5_diff1": 0.4429885882606201, "nauc_ndcg_at_5_max": 0.06364424725181266, "nauc_ndcg_at_5_std": -0.32161883938094465, "nauc_precision_at_1000_diff1": -0.01977690378890249, "nauc_precision_at_1000_max": 0.15031719296078636, "nauc_precision_at_1000_std": 0.09028119274489933, "nauc_precision_at_100_diff1": 0.035411720860641184, "nauc_precision_at_100_max": 0.17217751658003042, "nauc_precision_at_100_std": 0.07385988957101293, "nauc_precision_at_10_diff1": 0.22252752974305495, "nauc_precision_at_10_max": 0.05524962251923645, "nauc_precision_at_10_std": -0.2854246386801379, "nauc_precision_at_1_diff1": 0.5217613398284591, "nauc_precision_at_1_max": 0.06723111294341066, "nauc_precision_at_1_std": -0.30864470486943385, "nauc_precision_at_20_diff1": 0.19163443790426712, "nauc_precision_at_20_max": 0.055944718940985134, "nauc_precision_at_20_std": -0.25304253423825107, "nauc_precision_at_3_diff1": 0.31039815192793946, "nauc_precision_at_3_max": 0.07700281650111754, "nauc_precision_at_3_std": -0.2914386559768999, "nauc_precision_at_5_diff1": 0.2566827685957296, "nauc_precision_at_5_max": 0.05795553331673374, "nauc_precision_at_5_std": -0.29418793418071815, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.22548139370284953, "nauc_recall_at_100_max": 0.21967501152347532, "nauc_recall_at_100_std": 0.043216409696874146, "nauc_recall_at_10_diff1": 0.35174012919899866, "nauc_recall_at_10_max": 0.06359013710810518, "nauc_recall_at_10_std": -0.34903356214691456, "nauc_recall_at_1_diff1": 0.5411729166156709, "nauc_recall_at_1_max": 0.04965791121034343, "nauc_recall_at_1_std": -0.2770971704111151, "nauc_recall_at_20_diff1": 0.3456468055090329, "nauc_recall_at_20_max": 0.06377354394911941, "nauc_recall_at_20_std": -0.3507476267229391, "nauc_recall_at_3_diff1": 0.412487641826751, "nauc_recall_at_3_max": 0.06124492567012575, "nauc_recall_at_3_std": -0.2995915030276966, "nauc_recall_at_5_diff1": 0.3829534227232191, "nauc_recall_at_5_max": 0.05735351902047361, "nauc_recall_at_5_std": -0.3216100683761451, "ndcg_at_1": 0.28664, "ndcg_at_10": 0.37028, "ndcg_at_100": 0.44438, "ndcg_at_1000": 0.47193, "ndcg_at_20": 0.39278, "ndcg_at_3": 0.31, "ndcg_at_5": 0.33396, "precision_at_1": 0.28664, "precision_at_10": 0.08078, "precision_at_100": 0.01379, "precision_at_1000": 0.00167, "precision_at_20": 0.04723, "precision_at_3": 0.17427, "precision_at_5": 0.12736, "recall_at_1": 0.19803, "recall_at_10": 0.48125, "recall_at_100": 0.8032, "recall_at_1000": 1.0, "recall_at_20": 0.559, "recall_at_3": 0.32611, "recall_at_5": 0.38459 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/model_meta.json ================================================ {"name": "no_model_name_available", "revision": "no_revision_available", "release_date": null, "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": [], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "loader": null} ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/AutoRAGRetrieval.json ================================================ { "dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c", "evaluation_time": 19.277517318725586, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.72946, "map_at_1": 0.55263, "map_at_10": 0.67118, "map_at_100": 0.67467, "map_at_1000": 0.67467, "map_at_20": 0.67281, "map_at_3": 0.64474, "map_at_5": 0.66053, "mrr_at_1": 0.5526315789473685, "mrr_at_10": 0.6711849067112226, "mrr_at_100": 0.6746715904241167, "mrr_at_1000": 0.6746715904241167, "mrr_at_20": 0.6728147984339936, "mrr_at_3": 0.6447368421052632, "mrr_at_5": 0.6605263157894737, "nauc_map_at_1000_diff1": 0.6707428725853308, "nauc_map_at_1000_max": 0.36408406762747764, "nauc_map_at_1000_std": -0.20570630552244473, "nauc_map_at_100_diff1": 0.6707428725853308, "nauc_map_at_100_max": 0.36408406762747764, "nauc_map_at_100_std": -0.20570630552244473, "nauc_map_at_10_diff1": 0.6694228696798035, "nauc_map_at_10_max": 0.36958077127650485, "nauc_map_at_10_std": -0.2031257096763581, "nauc_map_at_1_diff1": 0.6675668434290756, "nauc_map_at_1_max": 0.2661749445307839, "nauc_map_at_1_std": -0.21180878980375847, "nauc_map_at_20_diff1": 0.6699889586317481, "nauc_map_at_20_max": 0.3658466072887328, "nauc_map_at_20_std": -0.20962057627984584, "nauc_map_at_3_diff1": 0.6862468818525479, "nauc_map_at_3_max": 0.3838859599279802, "nauc_map_at_3_std": -0.18434990538136026, "nauc_map_at_5_diff1": 0.6734220459216618, "nauc_map_at_5_max": 0.3715153642651667, "nauc_map_at_5_std": -0.19638862477177044, "nauc_mrr_at_1000_diff1": 0.6707428725853308, "nauc_mrr_at_1000_max": 0.36408406762747764, "nauc_mrr_at_1000_std": -0.20570630552244473, "nauc_mrr_at_100_diff1": 0.6707428725853308, "nauc_mrr_at_100_max": 0.36408406762747764, "nauc_mrr_at_100_std": -0.20570630552244473, "nauc_mrr_at_10_diff1": 0.6694228696798035, "nauc_mrr_at_10_max": 0.36958077127650485, "nauc_mrr_at_10_std": -0.2031257096763581, "nauc_mrr_at_1_diff1": 0.6675668434290756, "nauc_mrr_at_1_max": 0.2661749445307839, "nauc_mrr_at_1_std": -0.21180878980375847, "nauc_mrr_at_20_diff1": 0.6699889586317481, "nauc_mrr_at_20_max": 0.3658466072887328, "nauc_mrr_at_20_std": -0.20962057627984584, "nauc_mrr_at_3_diff1": 0.6862468818525479, "nauc_mrr_at_3_max": 0.3838859599279802, "nauc_mrr_at_3_std": -0.18434990538136026, "nauc_mrr_at_5_diff1": 0.6734220459216618, "nauc_mrr_at_5_max": 0.3715153642651667, "nauc_mrr_at_5_std": -0.19638862477177044, "nauc_ndcg_at_1000_diff1": 0.6671740713837205, "nauc_ndcg_at_1000_max": 0.3756626204320296, "nauc_ndcg_at_1000_std": -0.21166796127675785, "nauc_ndcg_at_100_diff1": 0.6671740713837205, "nauc_ndcg_at_100_max": 0.3756626204320296, "nauc_ndcg_at_100_std": -0.21166796127675785, "nauc_ndcg_at_10_diff1": 0.6568831312543059, "nauc_ndcg_at_10_max": 0.4052814278786497, "nauc_ndcg_at_10_std": -0.2119241383539628, "nauc_ndcg_at_1_diff1": 0.6675668434290756, "nauc_ndcg_at_1_max": 0.2661749445307839, "nauc_ndcg_at_1_std": -0.21180878980375847, "nauc_ndcg_at_20_diff1": 0.6594793623983968, "nauc_ndcg_at_20_max": 0.3885895716791715, "nauc_ndcg_at_20_std": -0.24290058030624878, "nauc_ndcg_at_3_diff1": 0.6928031605606169, "nauc_ndcg_at_3_max": 0.4232920523897769, "nauc_ndcg_at_3_std": -0.17400981818457717, "nauc_ndcg_at_5_diff1": 0.6676229670124798, "nauc_ndcg_at_5_max": 0.403635040677057, "nauc_ndcg_at_5_std": -0.19753091492323405, "nauc_precision_at_1000_diff1": 1.0, "nauc_precision_at_1000_max": 1.0, "nauc_precision_at_1000_std": 1.0, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.5288063479518919, "nauc_precision_at_10_max": 0.708870042679463, "nauc_precision_at_10_std": -0.30338470765024184, "nauc_precision_at_1_diff1": 0.6675668434290756, "nauc_precision_at_1_max": 0.2661749445307839, "nauc_precision_at_1_std": -0.21180878980375847, "nauc_precision_at_20_diff1": 0.5125144625120732, "nauc_precision_at_20_max": 0.6027016763741028, "nauc_precision_at_20_std": -0.7797707674708182, "nauc_precision_at_3_diff1": 0.7174378963535222, "nauc_precision_at_3_max": 0.570515649257075, "nauc_precision_at_3_std": -0.13484764350266676, "nauc_precision_at_5_diff1": 0.6323812858648742, "nauc_precision_at_5_max": 0.5549884337148923, "nauc_precision_at_5_std": -0.20953464522227783, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.5288063479518927, "nauc_recall_at_10_max": 0.7088700426794637, "nauc_recall_at_10_std": -0.3033847076502421, "nauc_recall_at_1_diff1": 0.6675668434290756, "nauc_recall_at_1_max": 0.2661749445307839, "nauc_recall_at_1_std": -0.21180878980375847, "nauc_recall_at_20_diff1": 0.5125144625120719, "nauc_recall_at_20_max": 0.602701676374104, "nauc_recall_at_20_std": -0.7797707674708185, "nauc_recall_at_3_diff1": 0.717437896353523, "nauc_recall_at_3_max": 0.5705156492570749, "nauc_recall_at_3_std": -0.13484764350266565, "nauc_recall_at_5_diff1": 0.6323812858648742, "nauc_recall_at_5_max": 0.5549884337148914, "nauc_recall_at_5_std": -0.20953464522227613, "ndcg_at_1": 0.55263, "ndcg_at_10": 0.72946, "ndcg_at_100": 0.74777, "ndcg_at_1000": 0.74777, "ndcg_at_20": 0.73587, "ndcg_at_3": 0.67512, "ndcg_at_5": 0.70381, "precision_at_1": 0.55263, "precision_at_10": 0.09123, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.04693, "precision_at_3": 0.25439, "precision_at_5": 0.16667, "recall_at_1": 0.55263, "recall_at_10": 0.91228, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.9386, "recall_at_3": 0.76316, "recall_at_5": 0.83333 } ] }, "task_name": "AutoRAGRetrieval" } ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/BelebeleRetrieval.json ================================================ { "dataset_revision": "75b399394a9803252cfec289d103de462763db7c", "evaluation_time": 20.426774740219116, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "kor_Hang-kor_Hang", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.8731, "map_at_1": 0.80222, "map_at_10": 0.85083, "map_at_100": 0.853, "map_at_1000": 0.85306, "map_at_20": 0.85226, "map_at_3": 0.83944, "map_at_5": 0.84744, "mrr_at_1": 0.8022222222222222, "mrr_at_10": 0.8508262786596121, "mrr_at_100": 0.8529968861639116, "mrr_at_1000": 0.8530557336878666, "mrr_at_20": 0.8522600180988984, "mrr_at_3": 0.8394444444444445, "mrr_at_5": 0.8474444444444444, "nauc_map_at_1000_diff1": 0.8560640977493851, "nauc_map_at_1000_max": 0.8063463833899528, "nauc_map_at_1000_std": -0.08489516754902722, "nauc_map_at_100_diff1": 0.8560402544947975, "nauc_map_at_100_max": 0.8063864802759896, "nauc_map_at_100_std": -0.08482547418762357, "nauc_map_at_10_diff1": 0.8549918293194821, "nauc_map_at_10_max": 0.8069540239085331, "nauc_map_at_10_std": -0.08483361982813872, "nauc_map_at_1_diff1": 0.8780100718940327, "nauc_map_at_1_max": 0.7914175798292935, "nauc_map_at_1_std": -0.09726163371056187, "nauc_map_at_20_diff1": 0.8553785393171915, "nauc_map_at_20_max": 0.8065336965287219, "nauc_map_at_20_std": -0.08371384444228923, "nauc_map_at_3_diff1": 0.8524735012353063, "nauc_map_at_3_max": 0.8072587980883054, "nauc_map_at_3_std": -0.09268025786806994, "nauc_map_at_5_diff1": 0.854626838155166, "nauc_map_at_5_max": 0.8078977938446023, "nauc_map_at_5_std": -0.0873739578815138, "nauc_mrr_at_1000_diff1": 0.8560640977493851, "nauc_mrr_at_1000_max": 0.8063463833899528, "nauc_mrr_at_1000_std": -0.08489516754902722, "nauc_mrr_at_100_diff1": 0.8560402544947975, "nauc_mrr_at_100_max": 0.8063864802759896, "nauc_mrr_at_100_std": -0.08482547418762357, "nauc_mrr_at_10_diff1": 0.8549918293194821, "nauc_mrr_at_10_max": 0.8069540239085331, "nauc_mrr_at_10_std": -0.08483361982813872, "nauc_mrr_at_1_diff1": 0.8780100718940327, "nauc_mrr_at_1_max": 0.7914175798292935, "nauc_mrr_at_1_std": -0.09726163371056187, "nauc_mrr_at_20_diff1": 0.8553785393171915, "nauc_mrr_at_20_max": 0.8065336965287219, "nauc_mrr_at_20_std": -0.08371384444228923, "nauc_mrr_at_3_diff1": 0.8524735012353063, "nauc_mrr_at_3_max": 0.8072587980883054, "nauc_mrr_at_3_std": -0.09268025786806994, "nauc_mrr_at_5_diff1": 0.854626838155166, "nauc_mrr_at_5_max": 0.8078977938446023, "nauc_mrr_at_5_std": -0.0873739578815138, "nauc_ndcg_at_1000_diff1": 0.8529039894642638, "nauc_ndcg_at_1000_max": 0.8087187703322236, "nauc_ndcg_at_1000_std": -0.07918583237480917, "nauc_ndcg_at_100_diff1": 0.8524104038120888, "nauc_ndcg_at_100_max": 0.8101262523349947, "nauc_ndcg_at_100_std": -0.0757592935305003, "nauc_ndcg_at_10_diff1": 0.8466716122240695, "nauc_ndcg_at_10_max": 0.812224133560615, "nauc_ndcg_at_10_std": -0.07233879254576772, "nauc_ndcg_at_1_diff1": 0.8780100718940327, "nauc_ndcg_at_1_max": 0.7914175798292935, "nauc_ndcg_at_1_std": -0.09726163371056187, "nauc_ndcg_at_20_diff1": 0.8478973857717044, "nauc_ndcg_at_20_max": 0.8102840983169886, "nauc_ndcg_at_20_std": -0.06787643804581135, "nauc_ndcg_at_3_diff1": 0.8428769936195931, "nauc_ndcg_at_3_max": 0.8126819826757915, "nauc_ndcg_at_3_std": -0.09209093244222816, "nauc_ndcg_at_5_diff1": 0.8458023636227519, "nauc_ndcg_at_5_max": 0.8143924646960852, "nauc_ndcg_at_5_std": -0.0809920849336958, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.8223834988540969, "nauc_precision_at_100_max": 0.9104490280961036, "nauc_precision_at_100_std": 0.18385536032594635, "nauc_precision_at_10_diff1": 0.7905534008475149, "nauc_precision_at_10_max": 0.84715937657114, "nauc_precision_at_10_std": 0.03006715506715187, "nauc_precision_at_1_diff1": 0.8780100718940327, "nauc_precision_at_1_max": 0.7914175798292935, "nauc_precision_at_1_std": -0.09726163371056187, "nauc_precision_at_20_diff1": 0.7728828905299473, "nauc_precision_at_20_max": 0.8407888407888411, "nauc_precision_at_20_std": 0.13823104999575037, "nauc_precision_at_3_diff1": 0.8063250419101998, "nauc_precision_at_3_max": 0.8331113225499538, "nauc_precision_at_3_std": -0.09033120384214648, "nauc_precision_at_5_diff1": 0.8015999192469749, "nauc_precision_at_5_max": 0.8471433618492425, "nauc_precision_at_5_std": -0.04609861962803618, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.8223834988540899, "nauc_recall_at_100_max": 0.9104490280960834, "nauc_recall_at_100_std": 0.18385536032594385, "nauc_recall_at_10_diff1": 0.7905534008475215, "nauc_recall_at_10_max": 0.8471593765711437, "nauc_recall_at_10_std": 0.03006715506715634, "nauc_recall_at_1_diff1": 0.8780100718940327, "nauc_recall_at_1_max": 0.7914175798292935, "nauc_recall_at_1_std": -0.09726163371056187, "nauc_recall_at_20_diff1": 0.7728828905299517, "nauc_recall_at_20_max": 0.840788840788842, "nauc_recall_at_20_std": 0.13823104999575303, "nauc_recall_at_3_diff1": 0.8063250419101982, "nauc_recall_at_3_max": 0.8331113225499521, "nauc_recall_at_3_std": -0.0903312038421456, "nauc_recall_at_5_diff1": 0.8015999192469797, "nauc_recall_at_5_max": 0.8471433618492444, "nauc_recall_at_5_std": -0.046098619628033, "ndcg_at_1": 0.80222, "ndcg_at_10": 0.8731, "ndcg_at_100": 0.88315, "ndcg_at_1000": 0.88472, "ndcg_at_20": 0.8784, "ndcg_at_3": 0.85078, "ndcg_at_5": 0.86508, "precision_at_1": 0.80222, "precision_at_10": 0.09422, "precision_at_100": 0.00988, "precision_at_1000": 0.001, "precision_at_20": 0.04817, "precision_at_3": 0.29444, "precision_at_5": 0.18356, "recall_at_1": 0.80222, "recall_at_10": 0.94222, "recall_at_100": 0.98778, "recall_at_1000": 1.0, "recall_at_20": 0.96333, "recall_at_3": 0.88333, "recall_at_5": 0.91778 }, { "hf_subset": "kor_Hang-eng_Latn", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.83888, "map_at_1": 0.75556, "map_at_10": 0.81336, "map_at_100": 0.81681, "map_at_1000": 0.81686, "map_at_20": 0.81569, "map_at_3": 0.80352, "map_at_5": 0.80957, "mrr_at_1": 0.7555555555555555, "mrr_at_10": 0.8133562610229279, "mrr_at_100": 0.8168110342899199, "mrr_at_1000": 0.8168636074795783, "mrr_at_20": 0.8156923300202454, "mrr_at_3": 0.8035185185185189, "mrr_at_5": 0.8095740740740744, "nauc_map_at_1000_diff1": 0.8552079792935121, "nauc_map_at_1000_max": 0.7655597821372521, "nauc_map_at_1000_std": -0.052744148523482345, "nauc_map_at_100_diff1": 0.8551799309543388, "nauc_map_at_100_max": 0.7655729563607201, "nauc_map_at_100_std": -0.05267908144900576, "nauc_map_at_10_diff1": 0.8539404180083635, "nauc_map_at_10_max": 0.7669291493096944, "nauc_map_at_10_std": -0.050172170353697, "nauc_map_at_1_diff1": 0.8750686813186811, "nauc_map_at_1_max": 0.7513523114816217, "nauc_map_at_1_std": -0.06911472148541131, "nauc_map_at_20_diff1": 0.8543470168817537, "nauc_map_at_20_max": 0.7656251112308703, "nauc_map_at_20_std": -0.052179337171268055, "nauc_map_at_3_diff1": 0.8554683573147109, "nauc_map_at_3_max": 0.7678759319132484, "nauc_map_at_3_std": -0.057746787669783646, "nauc_map_at_5_diff1": 0.8542533469592593, "nauc_map_at_5_max": 0.767135369326461, "nauc_map_at_5_std": -0.056006509755512475, "nauc_mrr_at_1000_diff1": 0.8552079792935121, "nauc_mrr_at_1000_max": 0.7655597821372521, "nauc_mrr_at_1000_std": -0.052744148523482345, "nauc_mrr_at_100_diff1": 0.8551799309543388, "nauc_mrr_at_100_max": 0.7655729563607201, "nauc_mrr_at_100_std": -0.05267908144900576, "nauc_mrr_at_10_diff1": 0.8539404180083635, "nauc_mrr_at_10_max": 0.7669291493096944, "nauc_mrr_at_10_std": -0.050172170353697, "nauc_mrr_at_1_diff1": 0.8750686813186811, "nauc_mrr_at_1_max": 0.7513523114816217, "nauc_mrr_at_1_std": -0.06911472148541131, "nauc_mrr_at_20_diff1": 0.8543470168817537, "nauc_mrr_at_20_max": 0.7656251112308703, "nauc_mrr_at_20_std": -0.052179337171268055, "nauc_mrr_at_3_diff1": 0.8554683573147109, "nauc_mrr_at_3_max": 0.7678759319132484, "nauc_mrr_at_3_std": -0.057746787669783646, "nauc_mrr_at_5_diff1": 0.8542533469592593, "nauc_mrr_at_5_max": 0.767135369326461, "nauc_mrr_at_5_std": -0.056006509755512475, "nauc_ndcg_at_1000_diff1": 0.8516502375385397, "nauc_ndcg_at_1000_max": 0.7681734581374354, "nauc_ndcg_at_1000_std": -0.046068889340921834, "nauc_ndcg_at_100_diff1": 0.8507752097738177, "nauc_ndcg_at_100_max": 0.7685687755564266, "nauc_ndcg_at_100_std": -0.043780455888798045, "nauc_ndcg_at_10_diff1": 0.8444678096177537, "nauc_ndcg_at_10_max": 0.7745939843597542, "nauc_ndcg_at_10_std": -0.029952471574620623, "nauc_ndcg_at_1_diff1": 0.8750686813186811, "nauc_ndcg_at_1_max": 0.7513523114816217, "nauc_ndcg_at_1_std": -0.06911472148541131, "nauc_ndcg_at_20_diff1": 0.8455288219624316, "nauc_ndcg_at_20_max": 0.7694035427816696, "nauc_ndcg_at_20_std": -0.03916335368484008, "nauc_ndcg_at_3_diff1": 0.8483046588257898, "nauc_ndcg_at_3_max": 0.7759423187598142, "nauc_ndcg_at_3_std": -0.049595196825266315, "nauc_ndcg_at_5_diff1": 0.8453704821374776, "nauc_ndcg_at_5_max": 0.7745320077009648, "nauc_ndcg_at_5_std": -0.04643041354083876, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.770308123249306, "nauc_precision_at_100_max": 0.805088702147535, "nauc_precision_at_100_std": 0.17478991596638663, "nauc_precision_at_10_diff1": 0.7871148459383727, "nauc_precision_at_10_max": 0.8209617180205405, "nauc_precision_at_10_std": 0.10627223862517551, "nauc_precision_at_1_diff1": 0.8750686813186811, "nauc_precision_at_1_max": 0.7513523114816217, "nauc_precision_at_1_std": -0.06911472148541131, "nauc_precision_at_20_diff1": 0.7612407926133399, "nauc_precision_at_20_max": 0.7927793339558018, "nauc_precision_at_20_std": 0.08127399107790856, "nauc_precision_at_3_diff1": 0.8203684910276686, "nauc_precision_at_3_max": 0.8083162225736156, "nauc_precision_at_3_std": -0.015997576959084468, "nauc_precision_at_5_diff1": 0.8041012298959324, "nauc_precision_at_5_max": 0.8085241248817403, "nauc_precision_at_5_std": -0.0006811731315055293, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.7703081232493021, "nauc_recall_at_100_max": 0.8050887021475257, "nauc_recall_at_100_std": 0.17478991596638907, "nauc_recall_at_10_diff1": 0.7871148459383758, "nauc_recall_at_10_max": 0.8209617180205421, "nauc_recall_at_10_std": 0.10627223862517948, "nauc_recall_at_1_diff1": 0.8750686813186811, "nauc_recall_at_1_max": 0.7513523114816217, "nauc_recall_at_1_std": -0.06911472148541131, "nauc_recall_at_20_diff1": 0.7612407926133409, "nauc_recall_at_20_max": 0.7927793339558047, "nauc_recall_at_20_std": 0.08127399107791237, "nauc_recall_at_3_diff1": 0.8203684910276674, "nauc_recall_at_3_max": 0.8083162225736152, "nauc_recall_at_3_std": -0.01599757695908346, "nauc_recall_at_5_diff1": 0.8041012298959324, "nauc_recall_at_5_max": 0.8085241248817412, "nauc_recall_at_5_std": -0.0006811731315034448, "ndcg_at_1": 0.75556, "ndcg_at_10": 0.83888, "ndcg_at_100": 0.85459, "ndcg_at_1000": 0.85601, "ndcg_at_20": 0.84716, "ndcg_at_3": 0.8189, "ndcg_at_5": 0.82961, "precision_at_1": 0.75556, "precision_at_10": 0.09178, "precision_at_100": 0.00989, "precision_at_1000": 0.001, "precision_at_20": 0.0475, "precision_at_3": 0.28778, "precision_at_5": 0.17778, "recall_at_1": 0.75556, "recall_at_10": 0.91778, "recall_at_100": 0.98889, "recall_at_1000": 1.0, "recall_at_20": 0.95, "recall_at_3": 0.86333, "recall_at_5": 0.88889 }, { "hf_subset": "eng_Latn-kor_Hang", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.82498, "map_at_1": 0.72667, "map_at_10": 0.79367, "map_at_100": 0.79628, "map_at_1000": 0.79635, "map_at_20": 0.7953, "map_at_3": 0.78056, "map_at_5": 0.78767, "mrr_at_1": 0.7266666666666667, "mrr_at_10": 0.7936684303350976, "mrr_at_100": 0.7962803748187756, "mrr_at_1000": 0.7963519416455496, "mrr_at_20": 0.795298418424064, "mrr_at_3": 0.7805555555555558, "mrr_at_5": 0.7876666666666673, "nauc_map_at_1000_diff1": 0.8198877016857965, "nauc_map_at_1000_max": 0.7039621373903541, "nauc_map_at_1000_std": -0.010282032620789295, "nauc_map_at_100_diff1": 0.8198307635640973, "nauc_map_at_100_max": 0.7039987287897214, "nauc_map_at_100_std": -0.010170817226117472, "nauc_map_at_10_diff1": 0.8190978533154528, "nauc_map_at_10_max": 0.7043790265300705, "nauc_map_at_10_std": -0.007775808697076773, "nauc_map_at_1_diff1": 0.8389424428385471, "nauc_map_at_1_max": 0.6697198904991112, "nauc_map_at_1_std": -0.06391227819799271, "nauc_map_at_20_diff1": 0.8194881547456134, "nauc_map_at_20_max": 0.704000317261751, "nauc_map_at_20_std": -0.010215529201033439, "nauc_map_at_3_diff1": 0.8216468435498626, "nauc_map_at_3_max": 0.7111519191391099, "nauc_map_at_3_std": -0.005581841153662009, "nauc_map_at_5_diff1": 0.8189828919805343, "nauc_map_at_5_max": 0.7095371290717255, "nauc_map_at_5_std": -0.0040165946637628165, "nauc_mrr_at_1000_diff1": 0.8198877016857965, "nauc_mrr_at_1000_max": 0.7039621373903541, "nauc_mrr_at_1000_std": -0.010282032620789295, "nauc_mrr_at_100_diff1": 0.8198307635640973, "nauc_mrr_at_100_max": 0.7039987287897214, "nauc_mrr_at_100_std": -0.010170817226117472, "nauc_mrr_at_10_diff1": 0.8190978533154528, "nauc_mrr_at_10_max": 0.7043790265300705, "nauc_mrr_at_10_std": -0.007775808697076773, "nauc_mrr_at_1_diff1": 0.8389424428385471, "nauc_mrr_at_1_max": 0.6697198904991112, "nauc_mrr_at_1_std": -0.06391227819799271, "nauc_mrr_at_20_diff1": 0.8194881547456134, "nauc_mrr_at_20_max": 0.704000317261751, "nauc_mrr_at_20_std": -0.010215529201033439, "nauc_mrr_at_3_diff1": 0.8216468435498626, "nauc_mrr_at_3_max": 0.7111519191391099, "nauc_mrr_at_3_std": -0.005581841153662009, "nauc_mrr_at_5_diff1": 0.8189828919805343, "nauc_mrr_at_5_max": 0.7095371290717255, "nauc_mrr_at_5_std": -0.0040165946637628165, "nauc_ndcg_at_1000_diff1": 0.8166294722148444, "nauc_ndcg_at_1000_max": 0.7082342519759786, "nauc_ndcg_at_1000_std": -0.0006063837025108052, "nauc_ndcg_at_100_diff1": 0.8150190642956522, "nauc_ndcg_at_100_max": 0.7095296953964633, "nauc_ndcg_at_100_std": 0.004120037570128214, "nauc_ndcg_at_10_diff1": 0.811486575865088, "nauc_ndcg_at_10_max": 0.711251275982116, "nauc_ndcg_at_10_std": 0.01407065567263889, "nauc_ndcg_at_1_diff1": 0.8389424428385471, "nauc_ndcg_at_1_max": 0.6697198904991112, "nauc_ndcg_at_1_std": -0.06391227819799271, "nauc_ndcg_at_20_diff1": 0.8131008675319239, "nauc_ndcg_at_20_max": 0.7099436542200415, "nauc_ndcg_at_20_std": 0.004561534822046999, "nauc_ndcg_at_3_diff1": 0.8165304433896998, "nauc_ndcg_at_3_max": 0.7257597506185963, "nauc_ndcg_at_3_std": 0.015108113368058916, "nauc_ndcg_at_5_diff1": 0.8110766298697383, "nauc_ndcg_at_5_max": 0.7241590386588409, "nauc_ndcg_at_5_std": 0.020597282917691275, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 0.7012138188608817, "nauc_precision_at_100_max": 0.8060891023075896, "nauc_precision_at_100_std": 0.3697145524876703, "nauc_precision_at_10_diff1": 0.7599358584013315, "nauc_precision_at_10_max": 0.7470872406933767, "nauc_precision_at_10_std": 0.1667546245551344, "nauc_precision_at_1_diff1": 0.8389424428385471, "nauc_precision_at_1_max": 0.6697198904991112, "nauc_precision_at_1_std": -0.06391227819799271, "nauc_precision_at_20_diff1": 0.7569736227824434, "nauc_precision_at_20_max": 0.748229847494551, "nauc_precision_at_20_std": 0.12646864301275607, "nauc_precision_at_3_diff1": 0.7974318529107323, "nauc_precision_at_3_max": 0.7815772239787632, "nauc_precision_at_3_std": 0.09422290136540673, "nauc_precision_at_5_diff1": 0.7745448115852023, "nauc_precision_at_5_max": 0.790755642956331, "nauc_precision_at_5_std": 0.13480563389999892, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.7012138188608772, "nauc_recall_at_100_max": 0.80608910230759, "nauc_recall_at_100_std": 0.3697145524876553, "nauc_recall_at_10_diff1": 0.7599358584013316, "nauc_recall_at_10_max": 0.7470872406933793, "nauc_recall_at_10_std": 0.1667546245551364, "nauc_recall_at_1_diff1": 0.8389424428385471, "nauc_recall_at_1_max": 0.6697198904991112, "nauc_recall_at_1_std": -0.06391227819799271, "nauc_recall_at_20_diff1": 0.7569736227824453, "nauc_recall_at_20_max": 0.7482298474945528, "nauc_recall_at_20_std": 0.1264686430127625, "nauc_recall_at_3_diff1": 0.7974318529107312, "nauc_recall_at_3_max": 0.7815772239787608, "nauc_recall_at_3_std": 0.09422290136540572, "nauc_recall_at_5_diff1": 0.7745448115852033, "nauc_recall_at_5_max": 0.7907556429563315, "nauc_recall_at_5_std": 0.1348056338999996, "ndcg_at_1": 0.72667, "ndcg_at_10": 0.82498, "ndcg_at_100": 0.83794, "ndcg_at_1000": 0.83992, "ndcg_at_20": 0.8309, "ndcg_at_3": 0.79731, "ndcg_at_5": 0.81013, "precision_at_1": 0.72667, "precision_at_10": 0.09233, "precision_at_100": 0.00984, "precision_at_1000": 0.001, "precision_at_20": 0.04733, "precision_at_3": 0.28185, "precision_at_5": 0.17533, "recall_at_1": 0.72667, "recall_at_10": 0.92333, "recall_at_100": 0.98444, "recall_at_1000": 1.0, "recall_at_20": 0.94667, "recall_at_3": 0.84556, "recall_at_5": 0.87667 } ] }, "task_name": "BelebeleRetrieval" } ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/Ko-StrategyQA.json ================================================ { "dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c", "evaluation_time": 112.89276218414307, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "default", "languages": [ "kor-Hang" ], "main_score": 0.75277, "map_at_1": 0.46061, "map_at_10": 0.70381, "map_at_100": 0.70988, "map_at_1000": 0.71025, "map_at_20": 0.70718, "map_at_3": 0.66324, "map_at_5": 0.69005, "mrr_at_1": 0.7094594594594594, "mrr_at_10": 0.7702065905190908, "mrr_at_100": 0.7724194172647475, "mrr_at_1000": 0.77260268439593, "mrr_at_20": 0.7713943086282484, "mrr_at_3": 0.762668918918919, "mrr_at_5": 0.7662162162162162, "nauc_map_at_1000_diff1": 0.5283723980423071, "nauc_map_at_1000_max": 0.5639612303477157, "nauc_map_at_1000_std": 0.03188120725857407, "nauc_map_at_100_diff1": 0.5284713837594557, "nauc_map_at_100_max": 0.5643147055598294, "nauc_map_at_100_std": 0.032082227552883445, "nauc_map_at_10_diff1": 0.5256042644404846, "nauc_map_at_10_max": 0.5623005303848427, "nauc_map_at_10_std": 0.0297382912059091, "nauc_map_at_1_diff1": 0.5701858649712257, "nauc_map_at_1_max": 0.27644800311234535, "nauc_map_at_1_std": -0.0689066272003743, "nauc_map_at_20_diff1": 0.5280904970352152, "nauc_map_at_20_max": 0.5633255574048996, "nauc_map_at_20_std": 0.030878945418817243, "nauc_map_at_3_diff1": 0.5294488430554147, "nauc_map_at_3_max": 0.5079972019662355, "nauc_map_at_3_std": -0.002753515625402979, "nauc_map_at_5_diff1": 0.5230331286467887, "nauc_map_at_5_max": 0.5521673525537383, "nauc_map_at_5_std": 0.018118497110511143, "nauc_mrr_at_1000_diff1": 0.6110877608775925, "nauc_mrr_at_1000_max": 0.655476433933367, "nauc_mrr_at_1000_std": 0.09332923134454195, "nauc_mrr_at_100_diff1": 0.611094567812382, "nauc_mrr_at_100_max": 0.6556526188303674, "nauc_mrr_at_100_std": 0.09345938564755246, "nauc_mrr_at_10_diff1": 0.6110172404956511, "nauc_mrr_at_10_max": 0.6580183379655195, "nauc_mrr_at_10_std": 0.09396654331046725, "nauc_mrr_at_1_diff1": 0.6196685261632404, "nauc_mrr_at_1_max": 0.5628198471368047, "nauc_mrr_at_1_std": 0.06250151061095299, "nauc_mrr_at_20_diff1": 0.6114055724751905, "nauc_mrr_at_20_max": 0.6565513476004787, "nauc_mrr_at_20_std": 0.09336678821083362, "nauc_mrr_at_3_diff1": 0.610102556468651, "nauc_mrr_at_3_max": 0.6546153414720447, "nauc_mrr_at_3_std": 0.0980530815344508, "nauc_mrr_at_5_diff1": 0.6102606171597507, "nauc_mrr_at_5_max": 0.6588326469437563, "nauc_mrr_at_5_std": 0.0956488852031488, "nauc_ndcg_at_1000_diff1": 0.548364721562242, "nauc_ndcg_at_1000_max": 0.6210691913997829, "nauc_ndcg_at_1000_std": 0.06170363548170726, "nauc_ndcg_at_100_diff1": 0.5478737500892652, "nauc_ndcg_at_100_max": 0.6302602500235459, "nauc_ndcg_at_100_std": 0.06464152290901783, "nauc_ndcg_at_10_diff1": 0.5404294253877192, "nauc_ndcg_at_10_max": 0.6254988627051006, "nauc_ndcg_at_10_std": 0.05611631319806918, "nauc_ndcg_at_1_diff1": 0.6196685261632404, "nauc_ndcg_at_1_max": 0.5628198471368047, "nauc_ndcg_at_1_std": 0.06250151061095299, "nauc_ndcg_at_20_diff1": 0.5458839263846998, "nauc_ndcg_at_20_max": 0.6273474088149236, "nauc_ndcg_at_20_std": 0.058500300554178264, "nauc_ndcg_at_3_diff1": 0.5380531600639286, "nauc_ndcg_at_3_max": 0.55783541791414, "nauc_ndcg_at_3_std": 0.017346169525437556, "nauc_ndcg_at_5_diff1": 0.5335841119966354, "nauc_ndcg_at_5_max": 0.6084086071554122, "nauc_ndcg_at_5_std": 0.04155408370757714, "nauc_precision_at_1000_diff1": -0.25699322093687643, "nauc_precision_at_1000_max": 0.07601960825033816, "nauc_precision_at_1000_std": 0.14744955792025402, "nauc_precision_at_100_diff1": -0.15632197997122793, "nauc_precision_at_100_max": 0.21329386391075608, "nauc_precision_at_100_std": 0.1615057936864428, "nauc_precision_at_10_diff1": -0.057457221919012806, "nauc_precision_at_10_max": 0.3279489619845634, "nauc_precision_at_10_std": 0.15133353580564418, "nauc_precision_at_1_diff1": 0.6196685261632404, "nauc_precision_at_1_max": 0.5628198471368047, "nauc_precision_at_1_std": 0.06250151061095299, "nauc_precision_at_20_diff1": -0.08043791260906195, "nauc_precision_at_20_max": 0.29662091164606197, "nauc_precision_at_20_std": 0.1538681230302332, "nauc_precision_at_3_diff1": 0.08073152560763562, "nauc_precision_at_3_max": 0.4082838602872238, "nauc_precision_at_3_std": 0.11105723443150181, "nauc_precision_at_5_diff1": -0.0008838666573368835, "nauc_precision_at_5_max": 0.37939011955604734, "nauc_precision_at_5_std": 0.1333475642609226, "nauc_recall_at_1000_diff1": 0.5643819020368003, "nauc_recall_at_1000_max": 0.8204347370517291, "nauc_recall_at_1000_std": 0.19033850555495097, "nauc_recall_at_100_diff1": 0.5304304823659386, "nauc_recall_at_100_max": 0.8262527382803844, "nauc_recall_at_100_std": 0.14912134664777515, "nauc_recall_at_10_diff1": 0.49163547770924193, "nauc_recall_at_10_max": 0.7174822582587085, "nauc_recall_at_10_std": 0.07108521174262873, "nauc_recall_at_1_diff1": 0.5701858649712257, "nauc_recall_at_1_max": 0.27644800311234535, "nauc_recall_at_1_std": -0.0689066272003743, "nauc_recall_at_20_diff1": 0.5092786343003812, "nauc_recall_at_20_max": 0.7359239824179382, "nauc_recall_at_20_std": 0.08327847033603757, "nauc_recall_at_3_diff1": 0.487954999737232, "nauc_recall_at_3_max": 0.5440221661839623, "nauc_recall_at_3_std": -0.004510327744451479, "nauc_recall_at_5_diff1": 0.4750711658142984, "nauc_recall_at_5_max": 0.6544747050418612, "nauc_recall_at_5_std": 0.03719332438438219, "ndcg_at_1": 0.70946, "ndcg_at_10": 0.75277, "ndcg_at_100": 0.77389, "ndcg_at_1000": 0.78241, "ndcg_at_20": 0.76106, "ndcg_at_3": 0.70598, "ndcg_at_5": 0.73118, "precision_at_1": 0.70946, "precision_at_10": 0.15135, "precision_at_100": 0.01693, "precision_at_1000": 0.00182, "precision_at_20": 0.07838, "precision_at_3": 0.41892, "precision_at_5": 0.2777, "recall_at_1": 0.46061, "recall_at_10": 0.81975, "recall_at_100": 0.90035, "recall_at_1000": 0.95687, "recall_at_20": 0.84632, "recall_at_3": 0.71699, "recall_at_5": 0.76854 } ] }, "task_name": "Ko-StrategyQA" } ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/MIRACLRetrieval.json ================================================ { "dataset_revision": "main", "evaluation_time": 11371.988452672958, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Kore" ], "main_score": 0.59892, "map_at_1": 0.33166, "map_at_10": 0.50683, "map_at_100": 0.53331, "map_at_1000": 0.53409, "map_at_20": 0.5235, "map_at_3": 0.43921, "map_at_5": 0.47596, "mrr_at_1": 0.539906103286385, "mrr_at_10": 0.654562560548476, "mrr_at_100": 0.6584680305976771, "mrr_at_1000": 0.6586721474663049, "mrr_at_20": 0.6570736531540831, "mrr_at_3": 0.6330203442879497, "mrr_at_5": 0.647809076682316, "nauc_map_at_1000_diff1": 0.3356785312787551, "nauc_map_at_1000_max": 0.3235511706611351, "nauc_map_at_1000_std": -0.07993364772635123, "nauc_map_at_100_diff1": 0.3355767982975825, "nauc_map_at_100_max": 0.32366533560928623, "nauc_map_at_100_std": -0.08006408616329992, "nauc_map_at_10_diff1": 0.33847390960408513, "nauc_map_at_10_max": 0.3027052556364594, "nauc_map_at_10_std": -0.130456084374079, "nauc_map_at_1_diff1": 0.4436529715971849, "nauc_map_at_1_max": 0.2197593675886129, "nauc_map_at_1_std": -0.2338662252219611, "nauc_map_at_20_diff1": 0.33870857019496, "nauc_map_at_20_max": 0.3128885452188113, "nauc_map_at_20_std": -0.09798507583805648, "nauc_map_at_3_diff1": 0.37776447643229705, "nauc_map_at_3_max": 0.2767416668664703, "nauc_map_at_3_std": -0.19863192356358927, "nauc_map_at_5_diff1": 0.3561167429750661, "nauc_map_at_5_max": 0.2849009688135222, "nauc_map_at_5_std": -0.18058479036602415, "nauc_mrr_at_1000_diff1": 0.26418759775085326, "nauc_mrr_at_1000_max": 0.42615412318814566, "nauc_mrr_at_1000_std": 0.16920356600320913, "nauc_mrr_at_100_diff1": 0.26416301871133957, "nauc_mrr_at_100_max": 0.4264119448078765, "nauc_mrr_at_100_std": 0.16948480024623644, "nauc_mrr_at_10_diff1": 0.2635977730704618, "nauc_mrr_at_10_max": 0.42979782822917384, "nauc_mrr_at_10_std": 0.1722637550445164, "nauc_mrr_at_1_diff1": 0.31806203990846843, "nauc_mrr_at_1_max": 0.3772431818974325, "nauc_mrr_at_1_std": 0.10571097874830919, "nauc_mrr_at_20_diff1": 0.2650788317901155, "nauc_mrr_at_20_max": 0.4272246710009317, "nauc_mrr_at_20_std": 0.16955994974903277, "nauc_mrr_at_3_diff1": 0.26143767164359477, "nauc_mrr_at_3_max": 0.4207971187432844, "nauc_mrr_at_3_std": 0.17140995146958404, "nauc_mrr_at_5_diff1": 0.25139687618821654, "nauc_mrr_at_5_max": 0.42323570646317255, "nauc_mrr_at_5_std": 0.16558258004255935, "nauc_ndcg_at_1000_diff1": 0.29958892618620975, "nauc_ndcg_at_1000_max": 0.38501029247145113, "nauc_ndcg_at_1000_std": 0.059259573627608066, "nauc_ndcg_at_100_diff1": 0.3009377107043353, "nauc_ndcg_at_100_max": 0.39527292405597536, "nauc_ndcg_at_100_std": 0.06903667287487025, "nauc_ndcg_at_10_diff1": 0.30065589964999173, "nauc_ndcg_at_10_max": 0.34411859353149415, "nauc_ndcg_at_10_std": -0.03588746180506714, "nauc_ndcg_at_1_diff1": 0.31806203990846843, "nauc_ndcg_at_1_max": 0.3772431818974325, "nauc_ndcg_at_1_std": 0.10571097874830919, "nauc_ndcg_at_20_diff1": 0.30998840438403896, "nauc_ndcg_at_20_max": 0.3663471267563369, "nauc_ndcg_at_20_std": 0.01414153978661343, "nauc_ndcg_at_3_diff1": 0.30193750136919056, "nauc_ndcg_at_3_max": 0.3458377385105322, "nauc_ndcg_at_3_std": 0.004422511116013512, "nauc_ndcg_at_5_diff1": 0.3042528293597453, "nauc_ndcg_at_5_max": 0.329355225386227, "nauc_ndcg_at_5_std": -0.04888581154378307, "nauc_precision_at_1000_diff1": -0.22484858090221838, "nauc_precision_at_1000_max": 0.14007405630073602, "nauc_precision_at_1000_std": 0.5620988868690893, "nauc_precision_at_100_diff1": -0.20199411739609105, "nauc_precision_at_100_max": 0.18794875350999649, "nauc_precision_at_100_std": 0.5848460512676487, "nauc_precision_at_10_diff1": -0.1780495088533368, "nauc_precision_at_10_max": 0.23748412451595022, "nauc_precision_at_10_std": 0.49889886269187383, "nauc_precision_at_1_diff1": 0.31806203990846843, "nauc_precision_at_1_max": 0.3772431818974325, "nauc_precision_at_1_std": 0.10571097874830919, "nauc_precision_at_20_diff1": -0.17607522736467468, "nauc_precision_at_20_max": 0.21173930270938313, "nauc_precision_at_20_std": 0.5620839514918033, "nauc_precision_at_3_diff1": -0.04710361351541399, "nauc_precision_at_3_max": 0.3553157438328994, "nauc_precision_at_3_std": 0.34595623758315647, "nauc_precision_at_5_diff1": -0.14216537428749043, "nauc_precision_at_5_max": 0.27558625362486805, "nauc_precision_at_5_std": 0.4195412647387249, "nauc_recall_at_1000_diff1": 0.3746757572669107, "nauc_recall_at_1000_max": 0.5928548386757957, "nauc_recall_at_1000_std": 0.5142911199199388, "nauc_recall_at_100_diff1": 0.28745646235411654, "nauc_recall_at_100_max": 0.5775877005126822, "nauc_recall_at_100_std": 0.3759204362505022, "nauc_recall_at_10_diff1": 0.23963010961221182, "nauc_recall_at_10_max": 0.2713974290618537, "nauc_recall_at_10_std": -0.10668557001230727, "nauc_recall_at_1_diff1": 0.4436529715971849, "nauc_recall_at_1_max": 0.2197593675886129, "nauc_recall_at_1_std": -0.2338662252219611, "nauc_recall_at_20_diff1": 0.28452710089292077, "nauc_recall_at_20_max": 0.32921281235198213, "nauc_recall_at_20_std": -0.0009927656338690539, "nauc_recall_at_3_diff1": 0.31662499652398335, "nauc_recall_at_3_max": 0.2300903670686967, "nauc_recall_at_3_std": -0.20219157373562308, "nauc_recall_at_5_diff1": 0.24517982270251407, "nauc_recall_at_5_max": 0.21475414644023236, "nauc_recall_at_5_std": -0.2104959961896699, "ndcg_at_1": 0.53991, "ndcg_at_10": 0.59892, "ndcg_at_100": 0.66341, "ndcg_at_1000": 0.67497, "ndcg_at_20": 0.63091, "ndcg_at_3": 0.55108, "ndcg_at_5": 0.56797, "precision_at_1": 0.53991, "precision_at_10": 0.15822, "precision_at_100": 0.02357, "precision_at_1000": 0.00254, "precision_at_20": 0.09507, "precision_at_3": 0.33959, "precision_at_5": 0.25352, "recall_at_1": 0.33166, "recall_at_10": 0.70097, "recall_at_100": 0.91204, "recall_at_1000": 0.98419, "recall_at_20": 0.79295, "recall_at_3": 0.50931, "recall_at_5": 0.60515 } ] }, "task_name": "MIRACLRetrieval" } ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/MrTidyRetrieval.json ================================================ { "dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7", "evaluation_time": 11457.999158143997, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Kore" ], "main_score": 0.55011, "map_at_1": 0.37173, "map_at_10": 0.49022, "map_at_100": 0.49864, "map_at_1000": 0.49898, "map_at_20": 0.49577, "map_at_3": 0.45322, "map_at_5": 0.47929, "mrr_at_1": 0.40617577197149646, "mrr_at_10": 0.5142206763940729, "mrr_at_100": 0.5211613964658944, "mrr_at_1000": 0.5213639580879118, "mrr_at_20": 0.5188662330481526, "mrr_at_3": 0.48218527315914494, "mrr_at_5": 0.5058194774346793, "nauc_map_at_1000_diff1": 0.3534765508644741, "nauc_map_at_1000_max": 0.362924613909079, "nauc_map_at_1000_std": 0.022637065199890526, "nauc_map_at_100_diff1": 0.353334465744014, "nauc_map_at_100_max": 0.3627436546134224, "nauc_map_at_100_std": 0.022577027925581254, "nauc_map_at_10_diff1": 0.35081454613547175, "nauc_map_at_10_max": 0.35824316801536765, "nauc_map_at_10_std": 0.018621993483028804, "nauc_map_at_1_diff1": 0.4394346374015472, "nauc_map_at_1_max": 0.35914751186291155, "nauc_map_at_1_std": -0.03112772622952899, "nauc_map_at_20_diff1": 0.3515758956767678, "nauc_map_at_20_max": 0.36139434521556835, "nauc_map_at_20_std": 0.022340965234366888, "nauc_map_at_3_diff1": 0.3643412305788555, "nauc_map_at_3_max": 0.35824502290711857, "nauc_map_at_3_std": 0.007773972628907927, "nauc_map_at_5_diff1": 0.3542438089845706, "nauc_map_at_5_max": 0.35294327035520345, "nauc_map_at_5_std": 0.012824952270608616, "nauc_mrr_at_1000_diff1": 0.37030847743191486, "nauc_mrr_at_1000_max": 0.4014409059065291, "nauc_mrr_at_1000_std": 0.04284497455893501, "nauc_mrr_at_100_diff1": 0.37021214811364633, "nauc_mrr_at_100_max": 0.4014604238515054, "nauc_mrr_at_100_std": 0.042864488196709745, "nauc_mrr_at_10_diff1": 0.36840780492276015, "nauc_mrr_at_10_max": 0.3985126565719115, "nauc_mrr_at_10_std": 0.040760639604665264, "nauc_mrr_at_1_diff1": 0.4476333755929182, "nauc_mrr_at_1_max": 0.40326469057519526, "nauc_mrr_at_1_std": 0.0072997114123084985, "nauc_mrr_at_20_diff1": 0.3696458244513162, "nauc_mrr_at_20_max": 0.4017034248300554, "nauc_mrr_at_20_std": 0.0426350028542941, "nauc_mrr_at_3_diff1": 0.37427363850577317, "nauc_mrr_at_3_max": 0.39667457388663585, "nauc_mrr_at_3_std": 0.0360488155766075, "nauc_mrr_at_5_diff1": 0.37018973625063745, "nauc_mrr_at_5_max": 0.39118360580739486, "nauc_mrr_at_5_std": 0.035550641671329056, "nauc_ndcg_at_1000_diff1": 0.34179738872130816, "nauc_ndcg_at_1000_max": 0.38723451860916625, "nauc_ndcg_at_1000_std": 0.05766002535067514, "nauc_ndcg_at_100_diff1": 0.33198200853952514, "nauc_ndcg_at_100_max": 0.3849533600748478, "nauc_ndcg_at_100_std": 0.06016231442477039, "nauc_ndcg_at_10_diff1": 0.3213002162445567, "nauc_ndcg_at_10_max": 0.3669678880126573, "nauc_ndcg_at_10_std": 0.04435716135388231, "nauc_ndcg_at_1_diff1": 0.4476333755929182, "nauc_ndcg_at_1_max": 0.40326469057519526, "nauc_ndcg_at_1_std": 0.0072997114123084985, "nauc_ndcg_at_20_diff1": 0.3233671091658442, "nauc_ndcg_at_20_max": 0.3784652743856314, "nauc_ndcg_at_20_std": 0.05610222103228094, "nauc_ndcg_at_3_diff1": 0.34540185310356236, "nauc_ndcg_at_3_max": 0.36440518603972377, "nauc_ndcg_at_3_std": 0.023929022932015993, "nauc_ndcg_at_5_diff1": 0.3292907310376978, "nauc_ndcg_at_5_max": 0.3535803636201188, "nauc_ndcg_at_5_std": 0.031205350457471002, "nauc_precision_at_1000_diff1": 0.038071768151506126, "nauc_precision_at_1000_max": 0.2023526720585822, "nauc_precision_at_1000_std": 0.2245887465920116, "nauc_precision_at_100_diff1": 0.034165106875631755, "nauc_precision_at_100_max": 0.2546944915960753, "nauc_precision_at_100_std": 0.24449808850665333, "nauc_precision_at_10_diff1": 0.10354267350812409, "nauc_precision_at_10_max": 0.2800684643390209, "nauc_precision_at_10_std": 0.15076718576951242, "nauc_precision_at_1_diff1": 0.4476333755929182, "nauc_precision_at_1_max": 0.40326469057519526, "nauc_precision_at_1_std": 0.0072997114123084985, "nauc_precision_at_20_diff1": 0.061548928520456604, "nauc_precision_at_20_max": 0.27662177632394025, "nauc_precision_at_20_std": 0.2128774243275918, "nauc_precision_at_3_diff1": 0.2501330958282208, "nauc_precision_at_3_max": 0.34690185014533564, "nauc_precision_at_3_std": 0.09171446549360808, "nauc_precision_at_5_diff1": 0.1750713626614356, "nauc_precision_at_5_max": 0.2914769521390908, "nauc_precision_at_5_std": 0.11260780828219168, "nauc_recall_at_1000_diff1": 0.370279255219639, "nauc_recall_at_1000_max": 0.6750328138256553, "nauc_recall_at_1000_std": 0.5043423318119773, "nauc_recall_at_100_diff1": 0.14232466984906053, "nauc_recall_at_100_max": 0.4792748970125092, "nauc_recall_at_100_std": 0.3228399242838028, "nauc_recall_at_10_diff1": 0.17444062059210683, "nauc_recall_at_10_max": 0.3396516241541569, "nauc_recall_at_10_std": 0.11756635139731124, "nauc_recall_at_1_diff1": 0.4394346374015472, "nauc_recall_at_1_max": 0.35914751186291155, "nauc_recall_at_1_std": -0.03112772622952899, "nauc_recall_at_20_diff1": 0.15478654557339946, "nauc_recall_at_20_max": 0.395072895370965, "nauc_recall_at_20_std": 0.18701562249696008, "nauc_recall_at_3_diff1": 0.2718140986712021, "nauc_recall_at_3_max": 0.3380662842993732, "nauc_recall_at_3_std": 0.04541815103444281, "nauc_recall_at_5_diff1": 0.2174882358215161, "nauc_recall_at_5_max": 0.3016629817024547, "nauc_recall_at_5_std": 0.06951134218978933, "ndcg_at_1": 0.40618, "ndcg_at_10": 0.55011, "ndcg_at_100": 0.58706, "ndcg_at_1000": 0.59621, "ndcg_at_20": 0.56841, "ndcg_at_3": 0.48167, "ndcg_at_5": 0.52645, "precision_at_1": 0.40618, "precision_at_10": 0.081, "precision_at_100": 0.01014, "precision_at_1000": 0.0011, "precision_at_20": 0.04489, "precision_at_3": 0.19873, "precision_at_5": 0.14489, "recall_at_1": 0.37173, "recall_at_10": 0.70942, "recall_at_100": 0.87253, "recall_at_1000": 0.94418, "recall_at_20": 0.77672, "recall_at_3": 0.53603, "recall_at_5": 0.64054 } ] }, "task_name": "MrTidyRetrieval" } ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/MultiLongDocRetrieval.json ================================================ { "dataset_revision": "d67138e705d963e346253a80e59676ddb418810a", "evaluation_time": 411.78907680511475, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "dev": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.21968, "map_at_1": 0.15, "map_at_10": 0.19486, "map_at_100": 0.20348, "map_at_1000": 0.20422, "map_at_20": 0.19862, "map_at_3": 0.1825, "map_at_5": 0.185, "mrr_at_1": 0.15, "mrr_at_10": 0.1948611111111111, "mrr_at_100": 0.20348075961449408, "mrr_at_1000": 0.2042227028085186, "mrr_at_20": 0.19861868094066237, "mrr_at_3": 0.1825, "mrr_at_5": 0.185, "nauc_map_at_1000_diff1": 0.5228641163292851, "nauc_map_at_1000_max": 0.3412128831254867, "nauc_map_at_1000_std": 0.0012824128179374956, "nauc_map_at_100_diff1": 0.5229167197883231, "nauc_map_at_100_max": 0.3413522374374235, "nauc_map_at_100_std": 0.0010966877334498038, "nauc_map_at_10_diff1": 0.521436223842329, "nauc_map_at_10_max": 0.33387114549522184, "nauc_map_at_10_std": -0.006027959979052935, "nauc_map_at_1_diff1": 0.6252091523031954, "nauc_map_at_1_max": 0.36016530591674234, "nauc_map_at_1_std": -0.026327991129926537, "nauc_map_at_20_diff1": 0.5205561717274496, "nauc_map_at_20_max": 0.335644036778537, "nauc_map_at_20_std": -0.004322490258848994, "nauc_map_at_3_diff1": 0.5377286501316654, "nauc_map_at_3_max": 0.3340684836642245, "nauc_map_at_3_std": -0.02057244685965842, "nauc_map_at_5_diff1": 0.5309102396475923, "nauc_map_at_5_max": 0.34128063601821707, "nauc_map_at_5_std": -0.017655472582244612, "nauc_mrr_at_1000_diff1": 0.5228641163292851, "nauc_mrr_at_1000_max": 0.3412128831254867, "nauc_mrr_at_1000_std": 0.0012824128179374956, "nauc_mrr_at_100_diff1": 0.5229167197883231, "nauc_mrr_at_100_max": 0.3413522374374235, "nauc_mrr_at_100_std": 0.0010966877334498038, "nauc_mrr_at_10_diff1": 0.521436223842329, "nauc_mrr_at_10_max": 0.33387114549522184, "nauc_mrr_at_10_std": -0.006027959979052935, "nauc_mrr_at_1_diff1": 0.6252091523031954, "nauc_mrr_at_1_max": 0.36016530591674234, "nauc_mrr_at_1_std": -0.026327991129926537, "nauc_mrr_at_20_diff1": 0.5205561717274496, "nauc_mrr_at_20_max": 0.335644036778537, "nauc_mrr_at_20_std": -0.004322490258848994, "nauc_mrr_at_3_diff1": 0.5377286501316654, "nauc_mrr_at_3_max": 0.3340684836642245, "nauc_mrr_at_3_std": -0.02057244685965842, "nauc_mrr_at_5_diff1": 0.5309102396475923, "nauc_mrr_at_5_max": 0.34128063601821707, "nauc_mrr_at_5_std": -0.017655472582244612, "nauc_ndcg_at_1000_diff1": 0.49450756826140746, "nauc_ndcg_at_1000_max": 0.3625588888678582, "nauc_ndcg_at_1000_std": 0.06012473989322547, "nauc_ndcg_at_100_diff1": 0.4933383125619591, "nauc_ndcg_at_100_max": 0.36004732048036603, "nauc_ndcg_at_100_std": 0.05438097701534137, "nauc_ndcg_at_10_diff1": 0.4866665045019168, "nauc_ndcg_at_10_max": 0.32095272853954526, "nauc_ndcg_at_10_std": 0.011149336322515323, "nauc_ndcg_at_1_diff1": 0.6252091523031954, "nauc_ndcg_at_1_max": 0.36016530591674234, "nauc_ndcg_at_1_std": -0.026327991129926537, "nauc_ndcg_at_20_diff1": 0.4826000708364039, "nauc_ndcg_at_20_max": 0.3273890002320808, "nauc_ndcg_at_20_std": 0.019339722067390635, "nauc_ndcg_at_3_diff1": 0.5172758390255361, "nauc_ndcg_at_3_max": 0.3259425664001633, "nauc_ndcg_at_3_std": -0.01780903214899896, "nauc_ndcg_at_5_diff1": 0.5061486447758137, "nauc_ndcg_at_5_max": 0.33797366721555006, "nauc_ndcg_at_5_std": -0.012996151109373837, "nauc_precision_at_1000_diff1": 0.40466972852093236, "nauc_precision_at_1000_max": 0.5651645947050766, "nauc_precision_at_1000_std": 0.4529059561882323, "nauc_precision_at_100_diff1": 0.4242039205720622, "nauc_precision_at_100_max": 0.44951751692396735, "nauc_precision_at_100_std": 0.2506511933250036, "nauc_precision_at_10_diff1": 0.4020470206728822, "nauc_precision_at_10_max": 0.2867450344548034, "nauc_precision_at_10_std": 0.05682340224294023, "nauc_precision_at_1_diff1": 0.6252091523031954, "nauc_precision_at_1_max": 0.36016530591674234, "nauc_precision_at_1_std": -0.026327991129926537, "nauc_precision_at_20_diff1": 0.3886963441341078, "nauc_precision_at_20_max": 0.3091011923160395, "nauc_precision_at_20_std": 0.0864544216971267, "nauc_precision_at_3_diff1": 0.46614041085247243, "nauc_precision_at_3_max": 0.30496161854740966, "nauc_precision_at_3_std": -0.01043447182770295, "nauc_precision_at_5_diff1": 0.4437546712222209, "nauc_precision_at_5_max": 0.33057000350818333, "nauc_precision_at_5_std": -0.00048046857125419895, "nauc_recall_at_1000_diff1": 0.40466972852093197, "nauc_recall_at_1000_max": 0.565164594705076, "nauc_recall_at_1000_std": 0.4529059561882311, "nauc_recall_at_100_diff1": 0.4242039205720618, "nauc_recall_at_100_max": 0.44951751692396746, "nauc_recall_at_100_std": 0.2506511933250037, "nauc_recall_at_10_diff1": 0.4020470206728819, "nauc_recall_at_10_max": 0.28674503445480354, "nauc_recall_at_10_std": 0.05682340224293989, "nauc_recall_at_1_diff1": 0.6252091523031954, "nauc_recall_at_1_max": 0.36016530591674234, "nauc_recall_at_1_std": -0.026327991129926537, "nauc_recall_at_20_diff1": 0.3886963441341077, "nauc_recall_at_20_max": 0.3091011923160393, "nauc_recall_at_20_std": 0.08645442169712678, "nauc_recall_at_3_diff1": 0.46614041085247204, "nauc_recall_at_3_max": 0.3049616185474097, "nauc_recall_at_3_std": -0.010434471827703058, "nauc_recall_at_5_diff1": 0.4437546712222204, "nauc_recall_at_5_max": 0.3305700035081832, "nauc_recall_at_5_std": -0.00048046857125430553, "ndcg_at_1": 0.15, "ndcg_at_10": 0.21968, "ndcg_at_100": 0.27014, "ndcg_at_1000": 0.29778, "ndcg_at_20": 0.23352, "ndcg_at_3": 0.1922, "ndcg_at_5": 0.19651, "precision_at_1": 0.15, "precision_at_10": 0.03, "precision_at_100": 0.00555, "precision_at_1000": 0.00079, "precision_at_20": 0.01775, "precision_at_3": 0.07333, "precision_at_5": 0.046, "recall_at_1": 0.15, "recall_at_10": 0.3, "recall_at_100": 0.555, "recall_at_1000": 0.79, "recall_at_20": 0.355, "recall_at_3": 0.22, "recall_at_5": 0.23 } ], "test": [ { "hf_subset": "ko", "languages": [ "kor-Hang" ], "main_score": 0.22787, "map_at_1": 0.14, "map_at_10": 0.1987, "map_at_100": 0.20557, "map_at_1000": 0.20665, "map_at_20": 0.2017, "map_at_3": 0.185, "map_at_5": 0.1945, "mrr_at_1": 0.14, "mrr_at_10": 0.19870039682539684, "mrr_at_100": 0.2055687357826064, "mrr_at_1000": 0.20664794980331383, "mrr_at_20": 0.2016985743015155, "mrr_at_3": 0.185, "mrr_at_5": 0.19450000000000003, "nauc_map_at_1000_diff1": 0.388027241959978, "nauc_map_at_1000_max": 0.40123520494204395, "nauc_map_at_1000_std": 0.04780702712023158, "nauc_map_at_100_diff1": 0.3877132605487392, "nauc_map_at_100_max": 0.40102565182458855, "nauc_map_at_100_std": 0.04748594852112135, "nauc_map_at_10_diff1": 0.3925371205072556, "nauc_map_at_10_max": 0.39907470125816463, "nauc_map_at_10_std": 0.04097899189000299, "nauc_map_at_1_diff1": 0.5254571683143111, "nauc_map_at_1_max": 0.47881751453180027, "nauc_map_at_1_std": 0.03938436081293219, "nauc_map_at_20_diff1": 0.39352216894549075, "nauc_map_at_20_max": 0.4015975417502318, "nauc_map_at_20_std": 0.041859869003760916, "nauc_map_at_3_diff1": 0.4075827351322252, "nauc_map_at_3_max": 0.4147374865792952, "nauc_map_at_3_std": 0.04135351689012322, "nauc_map_at_5_diff1": 0.4039580621195217, "nauc_map_at_5_max": 0.40326439248540985, "nauc_map_at_5_std": 0.03782948945277216, "nauc_mrr_at_1000_diff1": 0.388027241959978, "nauc_mrr_at_1000_max": 0.40123520494204395, "nauc_mrr_at_1000_std": 0.04780702712023158, "nauc_mrr_at_100_diff1": 0.3877132605487392, "nauc_mrr_at_100_max": 0.40102565182458855, "nauc_mrr_at_100_std": 0.04748594852112135, "nauc_mrr_at_10_diff1": 0.3925371205072556, "nauc_mrr_at_10_max": 0.39907470125816463, "nauc_mrr_at_10_std": 0.04097899189000299, "nauc_mrr_at_1_diff1": 0.5254571683143111, "nauc_mrr_at_1_max": 0.47881751453180027, "nauc_mrr_at_1_std": 0.03938436081293219, "nauc_mrr_at_20_diff1": 0.39352216894549075, "nauc_mrr_at_20_max": 0.4015975417502318, "nauc_mrr_at_20_std": 0.041859869003760916, "nauc_mrr_at_3_diff1": 0.4075827351322252, "nauc_mrr_at_3_max": 0.4147374865792952, "nauc_mrr_at_3_std": 0.04135351689012322, "nauc_mrr_at_5_diff1": 0.4039580621195217, "nauc_mrr_at_5_max": 0.40326439248540985, "nauc_mrr_at_5_std": 0.03782948945277216, "nauc_ndcg_at_1000_diff1": 0.3299411072050852, "nauc_ndcg_at_1000_max": 0.38494496638411996, "nauc_ndcg_at_1000_std": 0.08822983305464273, "nauc_ndcg_at_100_diff1": 0.3179348100164186, "nauc_ndcg_at_100_max": 0.3783068852575221, "nauc_ndcg_at_100_std": 0.0879924637618615, "nauc_ndcg_at_10_diff1": 0.3480091281920641, "nauc_ndcg_at_10_max": 0.3669584525955309, "nauc_ndcg_at_10_std": 0.04323131580056823, "nauc_ndcg_at_1_diff1": 0.5254571683143111, "nauc_ndcg_at_1_max": 0.47881751453180027, "nauc_ndcg_at_1_std": 0.03938436081293219, "nauc_ndcg_at_20_diff1": 0.3523221210131712, "nauc_ndcg_at_20_max": 0.3775461181355033, "nauc_ndcg_at_20_std": 0.048983482187103106, "nauc_ndcg_at_3_diff1": 0.3789291832529862, "nauc_ndcg_at_3_max": 0.3968734473338933, "nauc_ndcg_at_3_std": 0.04163039160544848, "nauc_ndcg_at_5_diff1": 0.3749576778048497, "nauc_ndcg_at_5_max": 0.37668940459113825, "nauc_ndcg_at_5_std": 0.03543307330469742, "nauc_precision_at_1000_diff1": 0.10452785726308714, "nauc_precision_at_1000_max": 0.3883545338687571, "nauc_precision_at_1000_std": 0.35327145502200163, "nauc_precision_at_100_diff1": 0.10860492979551226, "nauc_precision_at_100_max": 0.3360728565082189, "nauc_precision_at_100_std": 0.24945815327867799, "nauc_precision_at_10_diff1": 0.23936369474472868, "nauc_precision_at_10_max": 0.2881903564165687, "nauc_precision_at_10_std": 0.05033848870812367, "nauc_precision_at_1_diff1": 0.5254571683143111, "nauc_precision_at_1_max": 0.47881751453180027, "nauc_precision_at_1_std": 0.03938436081293219, "nauc_precision_at_20_diff1": 0.25693921762407507, "nauc_precision_at_20_max": 0.3270611488896598, "nauc_precision_at_20_std": 0.07357763219037795, "nauc_precision_at_3_diff1": 0.30945962137550925, "nauc_precision_at_3_max": 0.35290109034267914, "nauc_precision_at_3_std": 0.042243290198897755, "nauc_precision_at_5_diff1": 0.3088456291248556, "nauc_precision_at_5_max": 0.3120104709866414, "nauc_precision_at_5_std": 0.029009896190416957, "nauc_recall_at_1000_diff1": 0.10452785726308628, "nauc_recall_at_1000_max": 0.38835453386875723, "nauc_recall_at_1000_std": 0.3532714550220019, "nauc_recall_at_100_diff1": 0.10860492979551183, "nauc_recall_at_100_max": 0.3360728565082186, "nauc_recall_at_100_std": 0.24945815327867757, "nauc_recall_at_10_diff1": 0.23936369474472857, "nauc_recall_at_10_max": 0.28819035641656854, "nauc_recall_at_10_std": 0.05033848870812356, "nauc_recall_at_1_diff1": 0.5254571683143111, "nauc_recall_at_1_max": 0.47881751453180027, "nauc_recall_at_1_std": 0.03938436081293219, "nauc_recall_at_20_diff1": 0.25693921762407457, "nauc_recall_at_20_max": 0.3270611488896596, "nauc_recall_at_20_std": 0.07357763219037759, "nauc_recall_at_3_diff1": 0.3094596213755092, "nauc_recall_at_3_max": 0.352901090342679, "nauc_recall_at_3_std": 0.0422432901988977, "nauc_recall_at_5_diff1": 0.30884562912485536, "nauc_recall_at_5_max": 0.31201047098664125, "nauc_recall_at_5_std": 0.02900989619041678, "ndcg_at_1": 0.14, "ndcg_at_10": 0.22787, "ndcg_at_100": 0.26642, "ndcg_at_1000": 0.3004, "ndcg_at_20": 0.23909, "ndcg_at_3": 0.19917, "ndcg_at_5": 0.21701, "precision_at_1": 0.14, "precision_at_10": 0.032, "precision_at_100": 0.0051, "precision_at_1000": 0.00079, "precision_at_20": 0.01825, "precision_at_3": 0.08, "precision_at_5": 0.057, "recall_at_1": 0.14, "recall_at_10": 0.32, "recall_at_100": 0.51, "recall_at_1000": 0.79, "recall_at_20": 0.365, "recall_at_3": 0.24, "recall_at_5": 0.285 } ] }, "task_name": "MultiLongDocRetrieval" } ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/PublicHealthQA.json ================================================ { "dataset_revision": "main", "evaluation_time": 1.244837760925293, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "korean", "languages": [ "kor-Hang" ], "main_score": 0.77559, "map_at_1": 0.61039, "map_at_10": 0.72063, "map_at_100": 0.72355, "map_at_1000": 0.72355, "map_at_20": 0.72263, "map_at_3": 0.69481, "map_at_5": 0.71234, "mrr_at_1": 0.6103896103896104, "mrr_at_10": 0.7206349206349209, "mrr_at_100": 0.7235476992385783, "mrr_at_1000": 0.7235476992385783, "mrr_at_20": 0.722627246490883, "mrr_at_3": 0.6948051948051949, "mrr_at_5": 0.7123376623376625, "nauc_map_at_1000_diff1": 0.581514277430857, "nauc_map_at_1000_max": 0.2600121945421672, "nauc_map_at_1000_std": 0.0508272097576097, "nauc_map_at_100_diff1": 0.581514277430857, "nauc_map_at_100_max": 0.2600121945421672, "nauc_map_at_100_std": 0.0508272097576097, "nauc_map_at_10_diff1": 0.5798020406094601, "nauc_map_at_10_max": 0.25947368479173144, "nauc_map_at_10_std": 0.048451816839532806, "nauc_map_at_1_diff1": 0.6100966757438426, "nauc_map_at_1_max": 0.2804109120002006, "nauc_map_at_1_std": 0.05816858573239319, "nauc_map_at_20_diff1": 0.5784057148274875, "nauc_map_at_20_max": 0.2544781162654774, "nauc_map_at_20_std": 0.048219088535449, "nauc_map_at_3_diff1": 0.569994023647647, "nauc_map_at_3_max": 0.272733130545418, "nauc_map_at_3_std": 0.06125117458543027, "nauc_map_at_5_diff1": 0.576767098858931, "nauc_map_at_5_max": 0.270790922453173, "nauc_map_at_5_std": 0.06666333477243593, "nauc_mrr_at_1000_diff1": 0.581514277430857, "nauc_mrr_at_1000_max": 0.2600121945421672, "nauc_mrr_at_1000_std": 0.0508272097576097, "nauc_mrr_at_100_diff1": 0.581514277430857, "nauc_mrr_at_100_max": 0.2600121945421672, "nauc_mrr_at_100_std": 0.0508272097576097, "nauc_mrr_at_10_diff1": 0.5798020406094601, "nauc_mrr_at_10_max": 0.25947368479173144, "nauc_mrr_at_10_std": 0.048451816839532806, "nauc_mrr_at_1_diff1": 0.6100966757438426, "nauc_mrr_at_1_max": 0.2804109120002006, "nauc_mrr_at_1_std": 0.05816858573239319, "nauc_mrr_at_20_diff1": 0.5784057148274875, "nauc_mrr_at_20_max": 0.2544781162654774, "nauc_mrr_at_20_std": 0.048219088535449, "nauc_mrr_at_3_diff1": 0.569994023647647, "nauc_mrr_at_3_max": 0.272733130545418, "nauc_mrr_at_3_std": 0.06125117458543027, "nauc_mrr_at_5_diff1": 0.576767098858931, "nauc_mrr_at_5_max": 0.270790922453173, "nauc_mrr_at_5_std": 0.06666333477243593, "nauc_ndcg_at_1000_diff1": 0.5758895970977408, "nauc_ndcg_at_1000_max": 0.2563332621286461, "nauc_ndcg_at_1000_std": 0.052889533846057366, "nauc_ndcg_at_100_diff1": 0.5758895970977408, "nauc_ndcg_at_100_max": 0.2563332621286461, "nauc_ndcg_at_100_std": 0.052889533846057366, "nauc_ndcg_at_10_diff1": 0.5633651886103245, "nauc_ndcg_at_10_max": 0.24550419270187773, "nauc_ndcg_at_10_std": 0.044610907154761195, "nauc_ndcg_at_1_diff1": 0.6100966757438426, "nauc_ndcg_at_1_max": 0.2804109120002006, "nauc_ndcg_at_1_std": 0.05816858573239319, "nauc_ndcg_at_20_diff1": 0.5574381993765986, "nauc_ndcg_at_20_max": 0.22284415830217352, "nauc_ndcg_at_20_std": 0.03997714559180442, "nauc_ndcg_at_3_diff1": 0.5484583236884947, "nauc_ndcg_at_3_max": 0.2799253810560104, "nauc_ndcg_at_3_std": 0.07669422284650178, "nauc_ndcg_at_5_diff1": 0.5557446471304487, "nauc_ndcg_at_5_max": 0.2774984592893927, "nauc_ndcg_at_5_std": 0.0958730828908401, "nauc_precision_at_1000_diff1": NaN, "nauc_precision_at_1000_max": NaN, "nauc_precision_at_1000_std": NaN, "nauc_precision_at_100_diff1": 1.0, "nauc_precision_at_100_max": 1.0, "nauc_precision_at_100_std": 1.0, "nauc_precision_at_10_diff1": 0.3544905465332846, "nauc_precision_at_10_max": 0.029424740760962274, "nauc_precision_at_10_std": -0.04249064102210631, "nauc_precision_at_1_diff1": 0.6100966757438426, "nauc_precision_at_1_max": 0.2804109120002006, "nauc_precision_at_1_std": 0.05816858573239319, "nauc_precision_at_20_diff1": -0.00279858530002641, "nauc_precision_at_20_max": -0.8024518412512177, "nauc_precision_at_20_std": -0.304759369613741, "nauc_precision_at_3_diff1": 0.45974310015268893, "nauc_precision_at_3_max": 0.3130926276273598, "nauc_precision_at_3_std": 0.14462348982364673, "nauc_precision_at_5_diff1": 0.42345017705413346, "nauc_precision_at_5_max": 0.32459075504746615, "nauc_precision_at_5_std": 0.29382656447244254, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": NaN, "nauc_recall_at_100_max": NaN, "nauc_recall_at_100_std": NaN, "nauc_recall_at_10_diff1": 0.35449054653328155, "nauc_recall_at_10_max": 0.029424740760955526, "nauc_recall_at_10_std": -0.0424906410221141, "nauc_recall_at_1_diff1": 0.6100966757438426, "nauc_recall_at_1_max": 0.2804109120002006, "nauc_recall_at_1_std": 0.05816858573239319, "nauc_recall_at_20_diff1": -0.002798585300030213, "nauc_recall_at_20_max": -0.8024518412512368, "nauc_recall_at_20_std": -0.3047593696137533, "nauc_recall_at_3_diff1": 0.4597431001526878, "nauc_recall_at_3_max": 0.31309262762735857, "nauc_recall_at_3_std": 0.1446234898236454, "nauc_recall_at_5_diff1": 0.42345017705413185, "nauc_recall_at_5_max": 0.3245907550474637, "nauc_recall_at_5_std": 0.2938265644724397, "ndcg_at_1": 0.61039, "ndcg_at_10": 0.77559, "ndcg_at_100": 0.78768, "ndcg_at_1000": 0.78768, "ndcg_at_20": 0.78239, "ndcg_at_3": 0.7231, "ndcg_at_5": 0.75495, "precision_at_1": 0.61039, "precision_at_10": 0.09481, "precision_at_100": 0.01, "precision_at_1000": 0.001, "precision_at_20": 0.0487, "precision_at_3": 0.2684, "precision_at_5": 0.17662, "recall_at_1": 0.61039, "recall_at_10": 0.94805, "recall_at_100": 1.0, "recall_at_1000": 1.0, "recall_at_20": 0.97403, "recall_at_3": 0.80519, "recall_at_5": 0.88312 } ] }, "task_name": "PublicHealthQA" } ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/XPQARetrieval.json ================================================ { "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", "evaluation_time": 13.89358639717102, "kg_co2_emissions": null, "mteb_version": "1.20.0", "scores": { "test": [ { "hf_subset": "kor-kor", "languages": [ "kor-Hang", "kor-Hang" ], "main_score": 0.31695, "map_at_1": 0.17576, "map_at_10": 0.27113, "map_at_100": 0.28608, "map_at_1000": 0.28776, "map_at_20": 0.27892, "map_at_3": 0.24589, "map_at_5": 0.25816, "mrr_at_1": 0.25229357798165136, "mrr_at_10": 0.31745789039367023, "mrr_at_100": 0.32816024545088235, "mrr_at_1000": 0.32917734216519506, "mrr_at_20": 0.32299553219114674, "mrr_at_3": 0.30020387359836914, "mrr_at_5": 0.3079255861365954, "nauc_map_at_1000_diff1": 0.47010970462821955, "nauc_map_at_1000_max": 0.3098774398025671, "nauc_map_at_1000_std": -0.1963456368690233, "nauc_map_at_100_diff1": 0.4696861789142314, "nauc_map_at_100_max": 0.3093119757413642, "nauc_map_at_100_std": -0.19676930499005943, "nauc_map_at_10_diff1": 0.46920825182729364, "nauc_map_at_10_max": 0.30888721389787793, "nauc_map_at_10_std": -0.19545447098058139, "nauc_map_at_1_diff1": 0.5356241797932373, "nauc_map_at_1_max": 0.2738833679981866, "nauc_map_at_1_std": -0.169789162578628, "nauc_map_at_20_diff1": 0.46930293232753695, "nauc_map_at_20_max": 0.3074578890497353, "nauc_map_at_20_std": -0.19850651372859576, "nauc_map_at_3_diff1": 0.48349817727753475, "nauc_map_at_3_max": 0.3127191887552773, "nauc_map_at_3_std": -0.19032516458436943, "nauc_map_at_5_diff1": 0.4767338284192987, "nauc_map_at_5_max": 0.30830539578693966, "nauc_map_at_5_std": -0.19780921970754842, "nauc_mrr_at_1000_diff1": 0.5219129093886344, "nauc_mrr_at_1000_max": 0.31579091461596176, "nauc_mrr_at_1000_std": -0.22169124154266842, "nauc_mrr_at_100_diff1": 0.521776297162607, "nauc_mrr_at_100_max": 0.31609933760546466, "nauc_mrr_at_100_std": -0.22132323823988, "nauc_mrr_at_10_diff1": 0.5229792298473174, "nauc_mrr_at_10_max": 0.31191935916074975, "nauc_mrr_at_10_std": -0.22515088134298458, "nauc_mrr_at_1_diff1": 0.5440625092689464, "nauc_mrr_at_1_max": 0.32017015103420715, "nauc_mrr_at_1_std": -0.22188149293588674, "nauc_mrr_at_20_diff1": 0.52146104714408, "nauc_mrr_at_20_max": 0.3138962710865594, "nauc_mrr_at_20_std": -0.22327101012867404, "nauc_mrr_at_3_diff1": 0.536037870020009, "nauc_mrr_at_3_max": 0.31853282396028315, "nauc_mrr_at_3_std": -0.22170941151791226, "nauc_mrr_at_5_diff1": 0.5280003894597486, "nauc_mrr_at_5_max": 0.31199781678150607, "nauc_mrr_at_5_std": -0.2259123946480528, "nauc_ndcg_at_1000_diff1": 0.47542042721883954, "nauc_ndcg_at_1000_max": 0.319743954325265, "nauc_ndcg_at_1000_std": -0.1969569163821828, "nauc_ndcg_at_100_diff1": 0.46520084113259214, "nauc_ndcg_at_100_max": 0.32017555416152155, "nauc_ndcg_at_100_std": -0.19587487741313742, "nauc_ndcg_at_10_diff1": 0.46854597146746774, "nauc_ndcg_at_10_max": 0.30666262911980746, "nauc_ndcg_at_10_std": -0.2042751495469241, "nauc_ndcg_at_1_diff1": 0.5378096090850387, "nauc_ndcg_at_1_max": 0.31665420346212914, "nauc_ndcg_at_1_std": -0.2185862246391458, "nauc_ndcg_at_20_diff1": 0.46587586384321245, "nauc_ndcg_at_20_max": 0.30661705681445245, "nauc_ndcg_at_20_std": -0.20860794516424908, "nauc_ndcg_at_3_diff1": 0.49375417874271904, "nauc_ndcg_at_3_max": 0.31228234398000687, "nauc_ndcg_at_3_std": -0.20291869025747036, "nauc_ndcg_at_5_diff1": 0.4843844240743984, "nauc_ndcg_at_5_max": 0.30668841355544346, "nauc_ndcg_at_5_std": -0.20916296093832581, "nauc_precision_at_1000_diff1": 0.04877723637029814, "nauc_precision_at_1000_max": 0.16688314936689905, "nauc_precision_at_1000_std": 0.019828467896436743, "nauc_precision_at_100_diff1": 0.179882119045427, "nauc_precision_at_100_max": 0.25324013308929927, "nauc_precision_at_100_std": -0.07202221013960236, "nauc_precision_at_10_diff1": 0.3007906115132944, "nauc_precision_at_10_max": 0.27121384375921054, "nauc_precision_at_10_std": -0.18014855087915327, "nauc_precision_at_1_diff1": 0.5378096090850387, "nauc_precision_at_1_max": 0.31665420346212914, "nauc_precision_at_1_std": -0.2185862246391458, "nauc_precision_at_20_diff1": 0.27108689157120675, "nauc_precision_at_20_max": 0.25181094342953053, "nauc_precision_at_20_std": -0.17973160589793455, "nauc_precision_at_3_diff1": 0.3849078664119124, "nauc_precision_at_3_max": 0.3165770590612215, "nauc_precision_at_3_std": -0.20444308572079384, "nauc_precision_at_5_diff1": 0.35334428871263895, "nauc_precision_at_5_max": 0.28990972220689615, "nauc_precision_at_5_std": -0.20547892044831662, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.3360985759229544, "nauc_recall_at_100_max": 0.3328302204527823, "nauc_recall_at_100_std": -0.14266636407335864, "nauc_recall_at_10_diff1": 0.39515327362604297, "nauc_recall_at_10_max": 0.2827566725071189, "nauc_recall_at_10_std": -0.19174467066103854, "nauc_recall_at_1_diff1": 0.5356241797932373, "nauc_recall_at_1_max": 0.2738833679981866, "nauc_recall_at_1_std": -0.169789162578628, "nauc_recall_at_20_diff1": 0.3796528036373591, "nauc_recall_at_20_max": 0.27975372875136006, "nauc_recall_at_20_std": -0.2026172679066614, "nauc_recall_at_3_diff1": 0.4720342315302788, "nauc_recall_at_3_max": 0.31016277396024305, "nauc_recall_at_3_std": -0.193173168073115, "nauc_recall_at_5_diff1": 0.4470773059265365, "nauc_recall_at_5_max": 0.2887834167422638, "nauc_recall_at_5_std": -0.20932335416019043, "ndcg_at_1": 0.25382, "ndcg_at_10": 0.31695, "ndcg_at_100": 0.38782, "ndcg_at_1000": 0.42593, "ndcg_at_20": 0.34113, "ndcg_at_3": 0.27438, "ndcg_at_5": 0.28968, "precision_at_1": 0.25382, "precision_at_10": 0.06483, "precision_at_100": 0.01159, "precision_at_1000": 0.00156, "precision_at_20": 0.0396, "precision_at_3": 0.15494, "precision_at_5": 0.10673, "recall_at_1": 0.17576, "recall_at_10": 0.40889, "recall_at_100": 0.72207, "recall_at_1000": 1.0, "recall_at_20": 0.49312, "recall_at_3": 0.29317, "recall_at_5": 0.33216 }, { "hf_subset": "eng-kor", "languages": [ "eng-Latn", "kor-Hang" ], "main_score": 0.32279, "map_at_1": 0.14964, "map_at_10": 0.26403, "map_at_100": 0.28199, "map_at_1000": 0.28389, "map_at_20": 0.2738, "map_at_3": 0.22309, "map_at_5": 0.24491, "mrr_at_1": 0.2798165137614679, "mrr_at_10": 0.3482482646473471, "mrr_at_100": 0.3576154284974784, "mrr_at_1000": 0.358538469457532, "mrr_at_20": 0.353191168637493, "mrr_at_3": 0.32390417940876665, "mrr_at_5": 0.3357543323139654, "nauc_map_at_1000_diff1": 0.3662708399002596, "nauc_map_at_1000_max": 0.11516861417385038, "nauc_map_at_1000_std": -0.2261251217271451, "nauc_map_at_100_diff1": 0.36630926098352395, "nauc_map_at_100_max": 0.11388820361248321, "nauc_map_at_100_std": -0.22757909547798452, "nauc_map_at_10_diff1": 0.36994023414439564, "nauc_map_at_10_max": 0.11481118314040571, "nauc_map_at_10_std": -0.22855637342757695, "nauc_map_at_1_diff1": 0.47423086837159883, "nauc_map_at_1_max": 0.10536961568854993, "nauc_map_at_1_std": -0.18893895789801746, "nauc_map_at_20_diff1": 0.366939272679902, "nauc_map_at_20_max": 0.10983679096927623, "nauc_map_at_20_std": -0.23374220242287974, "nauc_map_at_3_diff1": 0.39334244076238073, "nauc_map_at_3_max": 0.11436057658181406, "nauc_map_at_3_std": -0.2210910109711889, "nauc_map_at_5_diff1": 0.37737593049160195, "nauc_map_at_5_max": 0.11892726061718971, "nauc_map_at_5_std": -0.22262031548738906, "nauc_mrr_at_1000_diff1": 0.37123198474648256, "nauc_mrr_at_1000_max": 0.12727530442627744, "nauc_mrr_at_1000_std": -0.21112173800821682, "nauc_mrr_at_100_diff1": 0.37101870995622205, "nauc_mrr_at_100_max": 0.1271614417906171, "nauc_mrr_at_100_std": -0.2112405573972666, "nauc_mrr_at_10_diff1": 0.3709052997544111, "nauc_mrr_at_10_max": 0.12694165718497352, "nauc_mrr_at_10_std": -0.21254209422882392, "nauc_mrr_at_1_diff1": 0.41584437352589027, "nauc_mrr_at_1_max": 0.1328740002999804, "nauc_mrr_at_1_std": -0.2130308623550934, "nauc_mrr_at_20_diff1": 0.3705755370423673, "nauc_mrr_at_20_max": 0.12468510854757699, "nauc_mrr_at_20_std": -0.2141889212406071, "nauc_mrr_at_3_diff1": 0.38286783683707126, "nauc_mrr_at_3_max": 0.13574605621937452, "nauc_mrr_at_3_std": -0.21082410124444367, "nauc_mrr_at_5_diff1": 0.37683917849074244, "nauc_mrr_at_5_max": 0.12783644925485255, "nauc_mrr_at_5_std": -0.21141293436461875, "nauc_ndcg_at_1000_diff1": 0.3423736953036982, "nauc_ndcg_at_1000_max": 0.12924407470721727, "nauc_ndcg_at_1000_std": -0.20436857853331591, "nauc_ndcg_at_100_diff1": 0.33738996916468716, "nauc_ndcg_at_100_max": 0.12644350937190557, "nauc_ndcg_at_100_std": -0.20587008962489492, "nauc_ndcg_at_10_diff1": 0.3461413361400627, "nauc_ndcg_at_10_max": 0.10926953290970053, "nauc_ndcg_at_10_std": -0.22924642778609006, "nauc_ndcg_at_1_diff1": 0.41584437352589027, "nauc_ndcg_at_1_max": 0.1328740002999804, "nauc_ndcg_at_1_std": -0.2130308623550934, "nauc_ndcg_at_20_diff1": 0.3415544325633606, "nauc_ndcg_at_20_max": 0.09671625391922535, "nauc_ndcg_at_20_std": -0.2422354888009854, "nauc_ndcg_at_3_diff1": 0.367483739659615, "nauc_ndcg_at_3_max": 0.1265622124663101, "nauc_ndcg_at_3_std": -0.21788403159650543, "nauc_ndcg_at_5_diff1": 0.3608782556832026, "nauc_ndcg_at_5_max": 0.11871910231919881, "nauc_ndcg_at_5_std": -0.21799306222986042, "nauc_precision_at_1000_diff1": -0.022334056218972737, "nauc_precision_at_1000_max": 0.13940688277251082, "nauc_precision_at_1000_std": 0.028726723259119125, "nauc_precision_at_100_diff1": 0.04924307028385224, "nauc_precision_at_100_max": 0.1443955710683356, "nauc_precision_at_100_std": -0.039578863449027626, "nauc_precision_at_10_diff1": 0.1645766060046971, "nauc_precision_at_10_max": 0.10417358758584026, "nauc_precision_at_10_std": -0.19642475114107616, "nauc_precision_at_1_diff1": 0.41584437352589027, "nauc_precision_at_1_max": 0.1328740002999804, "nauc_precision_at_1_std": -0.2130308623550934, "nauc_precision_at_20_diff1": 0.1314953529866074, "nauc_precision_at_20_max": 0.07263940993521478, "nauc_precision_at_20_std": -0.2040657067850367, "nauc_precision_at_3_diff1": 0.24539317151793216, "nauc_precision_at_3_max": 0.13549592702027766, "nauc_precision_at_3_std": -0.2121180653666456, "nauc_precision_at_5_diff1": 0.20719525349224496, "nauc_precision_at_5_max": 0.12663867643220872, "nauc_precision_at_5_std": -0.2043996228265867, "nauc_recall_at_1000_diff1": -0.06774630366806214, "nauc_recall_at_1000_max": 0.3349907211075982, "nauc_recall_at_1000_std": 0.0408621395621626, "nauc_recall_at_100_diff1": 0.2052762602690791, "nauc_recall_at_100_max": 0.16620711929006302, "nauc_recall_at_100_std": -0.09607436340389881, "nauc_recall_at_10_diff1": 0.2614919975156877, "nauc_recall_at_10_max": 0.07035861293021471, "nauc_recall_at_10_std": -0.228483692408934, "nauc_recall_at_1_diff1": 0.47423086837159883, "nauc_recall_at_1_max": 0.10536961568854993, "nauc_recall_at_1_std": -0.18893895789801746, "nauc_recall_at_20_diff1": 0.2514547872416605, "nauc_recall_at_20_max": 0.030726476984294798, "nauc_recall_at_20_std": -0.26509466944101795, "nauc_recall_at_3_diff1": 0.329564352854595, "nauc_recall_at_3_max": 0.10484425240517557, "nauc_recall_at_3_std": -0.20715027169708047, "nauc_recall_at_5_diff1": 0.30092812079581244, "nauc_recall_at_5_max": 0.09858083422573315, "nauc_recall_at_5_std": -0.20002863615923247, "ndcg_at_1": 0.27982, "ndcg_at_10": 0.32279, "ndcg_at_100": 0.39834, "ndcg_at_1000": 0.43527, "ndcg_at_20": 0.34922, "ndcg_at_3": 0.27259, "ndcg_at_5": 0.28663, "precision_at_1": 0.27982, "precision_at_10": 0.0867, "precision_at_100": 0.01495, "precision_at_1000": 0.00193, "precision_at_20": 0.05283, "precision_at_3": 0.18196, "precision_at_5": 0.13517, "recall_at_1": 0.14964, "recall_at_10": 0.41042, "recall_at_100": 0.72564, "recall_at_1000": 0.98191, "recall_at_20": 0.49424, "recall_at_3": 0.26555, "recall_at_5": 0.31906 }, { "hf_subset": "kor-eng", "languages": [ "kor-Hang", "eng-Latn" ], "main_score": 0.2815, "map_at_1": 0.14746, "map_at_10": 0.2379, "map_at_100": 0.25255, "map_at_1000": 0.25448, "map_at_20": 0.24529, "map_at_3": 0.21036, "map_at_5": 0.22613, "mrr_at_1": 0.21498371335504887, "mrr_at_10": 0.27844152318908016, "mrr_at_100": 0.2898901926591287, "mrr_at_1000": 0.2909255381684909, "mrr_at_20": 0.28434342738920654, "mrr_at_3": 0.25678610206297503, "mrr_at_5": 0.26908251900108576, "nauc_map_at_1000_diff1": 0.3969527263143755, "nauc_map_at_1000_max": 0.24477892510950194, "nauc_map_at_1000_std": -0.17697408503150655, "nauc_map_at_100_diff1": 0.39621663673684815, "nauc_map_at_100_max": 0.24414822578522066, "nauc_map_at_100_std": -0.1774117796874293, "nauc_map_at_10_diff1": 0.40329655125674535, "nauc_map_at_10_max": 0.24510025952131498, "nauc_map_at_10_std": -0.17940346758074718, "nauc_map_at_1_diff1": 0.459756702022391, "nauc_map_at_1_max": 0.21062599804717708, "nauc_map_at_1_std": -0.15042748824219065, "nauc_map_at_20_diff1": 0.39763732662346135, "nauc_map_at_20_max": 0.24039193755105104, "nauc_map_at_20_std": -0.18070885755684768, "nauc_map_at_3_diff1": 0.42437784299115283, "nauc_map_at_3_max": 0.25598937841695124, "nauc_map_at_3_std": -0.1657926913213058, "nauc_map_at_5_diff1": 0.41252522347403225, "nauc_map_at_5_max": 0.24631329137451874, "nauc_map_at_5_std": -0.1804687351340366, "nauc_mrr_at_1000_diff1": 0.4159418939093896, "nauc_mrr_at_1000_max": 0.2518855932320016, "nauc_mrr_at_1000_std": -0.19804067491998278, "nauc_mrr_at_100_diff1": 0.4155660820331092, "nauc_mrr_at_100_max": 0.252084585579496, "nauc_mrr_at_100_std": -0.1977519441655383, "nauc_mrr_at_10_diff1": 0.41882598096572715, "nauc_mrr_at_10_max": 0.24975534932181165, "nauc_mrr_at_10_std": -0.20331611640254818, "nauc_mrr_at_1_diff1": 0.43034958740728674, "nauc_mrr_at_1_max": 0.24415080072183756, "nauc_mrr_at_1_std": -0.1960017531952652, "nauc_mrr_at_20_diff1": 0.4155559045006145, "nauc_mrr_at_20_max": 0.2487849936163397, "nauc_mrr_at_20_std": -0.20071623782706016, "nauc_mrr_at_3_diff1": 0.432824544485855, "nauc_mrr_at_3_max": 0.258881953077277, "nauc_mrr_at_3_std": -0.19994890791385014, "nauc_mrr_at_5_diff1": 0.4245282020571004, "nauc_mrr_at_5_max": 0.2526657216382904, "nauc_mrr_at_5_std": -0.20588903310501974, "nauc_ndcg_at_1000_diff1": 0.3881231065699723, "nauc_ndcg_at_1000_max": 0.25406410892726006, "nauc_ndcg_at_1000_std": -0.17343211138861236, "nauc_ndcg_at_100_diff1": 0.36918028335621594, "nauc_ndcg_at_100_max": 0.25332895045953363, "nauc_ndcg_at_100_std": -0.16633957934868546, "nauc_ndcg_at_10_diff1": 0.3932948883164321, "nauc_ndcg_at_10_max": 0.23936355850750848, "nauc_ndcg_at_10_std": -0.19078998661825733, "nauc_ndcg_at_1_diff1": 0.43034958740728674, "nauc_ndcg_at_1_max": 0.24415080072183756, "nauc_ndcg_at_1_std": -0.1960017531952652, "nauc_ndcg_at_20_diff1": 0.37776056898615223, "nauc_ndcg_at_20_max": 0.2291496653546807, "nauc_ndcg_at_20_std": -0.18960104920000687, "nauc_ndcg_at_3_diff1": 0.42242138926841405, "nauc_ndcg_at_3_max": 0.25415734830804504, "nauc_ndcg_at_3_std": -0.18193855363293832, "nauc_ndcg_at_5_diff1": 0.40917918459532354, "nauc_ndcg_at_5_max": 0.24332506218050745, "nauc_ndcg_at_5_std": -0.1958771002703526, "nauc_precision_at_1000_diff1": 0.0014243480474259468, "nauc_precision_at_1000_max": 0.16678180277908505, "nauc_precision_at_1000_std": 0.027359371138435964, "nauc_precision_at_100_diff1": 0.06508721331970349, "nauc_precision_at_100_max": 0.22982452992859312, "nauc_precision_at_100_std": -0.015544385060792282, "nauc_precision_at_10_diff1": 0.2299686644226259, "nauc_precision_at_10_max": 0.20508463425090528, "nauc_precision_at_10_std": -0.20238340654579362, "nauc_precision_at_1_diff1": 0.43034958740728674, "nauc_precision_at_1_max": 0.24415080072183756, "nauc_precision_at_1_std": -0.1960017531952652, "nauc_precision_at_20_diff1": 0.17946826792879048, "nauc_precision_at_20_max": 0.1744467450720577, "nauc_precision_at_20_std": -0.18591120569922195, "nauc_precision_at_3_diff1": 0.32050003491604123, "nauc_precision_at_3_max": 0.266713463768876, "nauc_precision_at_3_std": -0.19667977570584666, "nauc_precision_at_5_diff1": 0.27976985306893926, "nauc_precision_at_5_max": 0.2272630441916428, "nauc_precision_at_5_std": -0.21950656093547194, "nauc_recall_at_1000_diff1": NaN, "nauc_recall_at_1000_max": NaN, "nauc_recall_at_1000_std": NaN, "nauc_recall_at_100_diff1": 0.19485824080100417, "nauc_recall_at_100_max": 0.2484326517970991, "nauc_recall_at_100_std": -0.07243889229118167, "nauc_recall_at_10_diff1": 0.33520147413477436, "nauc_recall_at_10_max": 0.20709025445809287, "nauc_recall_at_10_std": -0.18525478614342566, "nauc_recall_at_1_diff1": 0.459756702022391, "nauc_recall_at_1_max": 0.21062599804717708, "nauc_recall_at_1_std": -0.15042748824219065, "nauc_recall_at_20_diff1": 0.28048438077204607, "nauc_recall_at_20_max": 0.1710650121519085, "nauc_recall_at_20_std": -0.17771177772845145, "nauc_recall_at_3_diff1": 0.4151834934058445, "nauc_recall_at_3_max": 0.25469360673270813, "nauc_recall_at_3_std": -0.16243993065702017, "nauc_recall_at_5_diff1": 0.3793251586166503, "nauc_recall_at_5_max": 0.22133888152640455, "nauc_recall_at_5_std": -0.20005187592158255, "ndcg_at_1": 0.21498, "ndcg_at_10": 0.2815, "ndcg_at_100": 0.35338, "ndcg_at_1000": 0.3971, "ndcg_at_20": 0.30508, "ndcg_at_3": 0.23709, "ndcg_at_5": 0.25698, "precision_at_1": 0.21498, "precision_at_10": 0.06091, "precision_at_100": 0.01176, "precision_at_1000": 0.00167, "precision_at_20": 0.03721, "precision_at_3": 0.13572, "precision_at_5": 0.09935, "recall_at_1": 0.14746, "recall_at_10": 0.36994, "recall_at_100": 0.68087, "recall_at_1000": 1.0, "recall_at_20": 0.45279, "recall_at_3": 0.25291, "recall_at_5": 0.30226 } ] }, "task_name": "XPQARetrieval" } ================================================ FILE: eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/model_meta.json ================================================ {"name": "upskyy/bge-m3-korean", "revision": "no_revision_available", "release_date": null, "languages": [], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_weights": null, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers"], "reference": null, "similarity_fn_name": null, "use_instructions": null, "zero_shot_benchmarks": null, "loader": null}