Showing preview only (1,708K chars total). Download the full file or copy to clipboard to get everything.
Repository: nlpai-lab/KURE
Branch: main
Commit: 992e31d8e0b1
Files: 169
Total size: 1.6 MB
Directory structure:
gitextract_pyyh9iti/
├── .gitignore
├── LICENSE
├── README.md
├── README_EN.md
└── eval/
├── evaluate.py
├── leaderboard.py
├── requirements.txt
└── results/
├── Alibaba-NLP/
│ ├── gte-Qwen2-7B-instruct/
│ │ └── Alibaba-NLP__gte-Qwen2-7B-instruct/
│ │ └── e26182b2122f4435e8b3ebecbf363990f409b45b/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ └── gte-multilingual-base/
│ └── Alibaba-NLP__gte-multilingual-base/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── BAAI/
│ ├── bge-m3/
│ │ └── BAAI__bge-m3/
│ │ └── no_revision_available/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ └── bge-multilingual-gemma2/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── Salesforce/
│ └── SFR-Embedding-2_R/
│ └── Salesforce__SFR-Embedding-2_R/
│ └── 91762139d94ed4371a9fa31db5551272e0b83818/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── Snowflake/
│ └── snowflake-arctic-embed-l-v2.0/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── dragonkue/
│ └── BGE-m3-ko/
│ └── dragonkue__BGE-m3-ko/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── intfloat/
│ ├── e5-mistral-7b-instruct/
│ │ └── intfloat__e5-mistral-7b-instruct/
│ │ └── 07163b72af1488142a360786df853f237b1a3ca1/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ ├── multilingual-e5-base/
│ │ └── intfloat__multilingual-e5-base/
│ │ └── d13f1b27baf31030b7fd040960d60d909913633f/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ ├── multilingual-e5-large/
│ │ └── intfloat__multilingual-e5-large/
│ │ └── ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ └── multilingual-e5-large-instruct/
│ └── intfloat__multilingual-e5-large-instruct/
│ └── baa7be480a7de1539afce709c8f13f833a510e0a/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── jhgan/
│ └── ko-sroberta-multitask/
│ └── jhgan__ko-sroberta-multitask/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── jinaai/
│ └── jina-embeddings-v3/
│ └── jinaai__jina-embeddings-v3/
│ └── 215a6e121fa0183376388ac6b1ae230326bfeaed/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── nlpai-lab/
│ ├── KURE-v1/
│ │ └── nlpai-lab__KURE-v1/
│ │ └── no_revision_available/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ └── KoE5/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── nomic-ai/
│ └── nomic-embed-text-v2-moe/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── openai/
│ └── text-embedding-3-large/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
└── upskyy/
└── bge-m3-korean/
└── upskyy__bge-m3-korean/
└── no_revision_available/
├── AutoRAGRetrieval.json
├── BelebeleRetrieval.json
├── Ko-StrategyQA.json
├── MIRACLRetrieval.json
├── MrTidyRetrieval.json
├── MultiLongDocRetrieval.json
├── PublicHealthQA.json
├── XPQARetrieval.json
└── model_meta.json
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
__pycache__
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) [year] [fullname]
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================
FILE: README.md
================================================
# 🔎 KURE: Korea University Retrieval Embedding model
## Update Logs
- 2024.12.21: [🤗 KURE-v1](https://huggingface.co/nlpai-lab/KURE-v1), [MTEB-ko-retrieval Leaderboard](https://github.com/nlpai-lab/KURE?tab=readme-ov-file#mteb-ko-retrieval-leaderboard) 공개
- 2024.10.02: [🤗 KoE5](https://huggingface.co/nlpai-lab/KoE5), [🤗 ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0) 공개
---
<br>
KURE는 고려대학교 [NLP & AI 연구실](http://nlp.korea.ac.kr/)과 [HIAI 연구소](http://hiai.korea.ac.kr)가 개발한 한국어 특화 임베딩 모델입니다.
KURE를 공개합니다.
<br/>
## KURE 모델 실행 코드
### sentence-transformers로 실행
```bash
pip install sentence-transformers
```
아래 예제 코드로 실행해볼 수 있습니다.
```python
from sentence_transformers import SentenceTransformer
# Download from the 🤗 Hub
model = SentenceTransformer("nlpai-lab/KURE-v1")
# model = SentenceTransformer("nlpai-lab/KoE5")
# Run inference
sentences = [
'헌법과 법원조직법은 어떤 방식을 통해 기본권 보장 등의 다양한 법적 모색을 가능하게 했어',
'4. 시사점과 개선방향 앞서 살펴본 바와 같이 우리 헌법과 ｢법원조직 법｣은 대법원 구성을 다양화하여 기본권 보장과 민주주의 확립에 있어 다각적인 법적 모색을 가능하게 하는 것을 근본 규범으로 하고 있다. 더욱이 합의체로서의 대법원 원리를 채택하고 있는 것 역시 그 구성의 다양성을 요청하는 것으로 해석된다. 이와 같은 관점에서 볼 때 현직 법원장급 고위법관을 중심으로 대법원을 구성하는 관행은 개선할 필요가 있는 것으로 보인다.',
'연방헌법재판소는 2001년 1월 24일 5:3의 다수견해로 「법원조직법」 제169조 제2문이 헌법에 합치된다는 판결을 내렸음 ○ 5인의 다수 재판관은 소송관계인의 인격권 보호, 공정한 절차의 보장과 방해받지 않는 법과 진실 발견 등을 근거로 하여 텔레비전 촬영에 대한 절대적인 금지를 헌법에 합치하는 것으로 보았음 ○ 그러나 나머지 3인의 재판관은 행정법원의 소송절차는 특별한 인격권 보호의 이익도 없으며, 텔레비전 공개주의로 인해 법과 진실 발견의 과정이 언제나 위태롭게 되는 것은 아니라면서 반대의견을 제시함 ○ 왜냐하면 행정법원의 소송절차에서는 소송당사자가 개인적으로 직접 심리에 참석하기보다는 변호사가 참석하는 경우가 많으며, 심리대상도 사실문제가 아닌 법률문제가 대부분이기 때문이라는 것임 □ 한편, 연방헌법재판소는 「연방헌법재판소법」(Bundesverfassungsgerichtsgesetz: BVerfGG) 제17a조에 따라 제한적이나마 재판에 대한 방송을 허용하고 있음 ○ 「연방헌법재판소법」 제17조에서 「법원조직법」 제14절 내지 제16절의 규정을 준용하도록 하고 있지만, 녹음이나 촬영을 통한 재판공개와 관련하여서는 「법원조직법」과 다른 내용을 규정하고 있음',
]
embeddings = model.encode(sentences)
print(embeddings.shape)
# [3, 1024]
# Get the similarity scores for the embeddings
similarities = model.similarity(embeddings, embeddings)
print(similarities)
# Results for KURE-v1
# tensor([[1.0000, 0.6967, 0.5306],
# [0.6967, 1.0000, 0.4427],
# [0.5306, 0.4427, 1.0000]])
# Results for KoE5
# tensor([[1.0000, 0.6721, 0.3897],
# [0.6721, 1.0000, 0.3740],
# [0.3897, 0.3740, 1.0000]])
```
<br/>
## MTEB-ko-retrieval Leaderboard
[MTEB](https://github.com/embeddings-benchmark/mteb)에 등록된 모든 Korean Retrieval Benchmark에 대한 평가를 진행하였습니다.
### Korean Retrieval Benchmark
- [Ko-StrategyQA](https://huggingface.co/datasets/taeminlee/Ko-StrategyQA): 한국어 ODQA multi-hop 검색 데이터셋 (StrategyQA 번역)
- [AutoRAGRetrieval](https://huggingface.co/datasets/yjoonjang/markers_bm): 금융, 공공, 의료, 법률, 커머스 5개 분야에 대해, pdf를 파싱하여 구성한 한국어 문서 검색 데이터셋
- [MIRACLRetrieval](https://huggingface.co/datasets/miracl/miracl): Wikipedia 기반의 한국어 문서 검색 데이터셋
- [PublicHealthQA](https://huggingface.co/datasets/xhluca/publichealth-qa): 의료 및 공중보건 도메인에 대한 한국어 문서 검색 데이터셋
- [BelebeleRetrieval](https://huggingface.co/datasets/facebook/belebele): FLORES-200 기반의 한국어 문서 검색 데이터셋
- [MrTidyRetrieval](https://huggingface.co/datasets/mteb/mrtidy): Wikipedia 기반의 한국어 문서 검색 데이터셋
- [MultiLongDocRetrieval](https://huggingface.co/datasets/Shitao/MLDR): 다양한 도메인의 한국어 장문 검색 데이터셋
- [XPQARetrieval](https://huggingface.co/datasets/jinaai/xpqa): 다양한 도메인의 한국어 문서 검색 데이터셋
### Evaluation code
`evaluate.py`에 모델을 추가하여 mteb를 활용한 평가를 진행할 수 있습니다.
```bash
cd eval
pip install -r requirements.txt
python evaluate.py
```
### Leaderboard
streamlit을 통해 모든 모델의 모든 태스크에 대한 평가 결과를 시각화합니다.
```bash
streamlit run leaderboard.py
```
아래는 모든 모델의, 모든 벤치마크 데이터셋에 대한 평균 결과입니다.
자세한 결과는 `eval/results`폴더에서 확인하실 수 있습니다.
### Top-k 1
| Model | Average Recall | Average Precision | Average NDCG | Average F1 |
|-----------------------------------------|----------------------|------------------------|-------------------|-----------------|
| **nlpai-lab/KURE-v1** | **0.52640** | **0.60551** | **0.60551** | **0.55784** |
| dragonkue/BGE-m3-ko | 0.52361 | 0.60394 | 0.60394 | 0.55535 |
| BAAI/bge-m3 | 0.51778 | 0.59846 | 0.59846 | 0.54998 |
| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.51246 | 0.59384 | 0.59384 | 0.54489 |
| nlpai-lab/KoE5 | 0.50157 | 0.57790 | 0.57790 | 0.53178 |
| intfloat/multilingual-e5-large | 0.50052 | 0.57727 | 0.57727 | 0.53122 |
| jinaai/jina-embeddings-v3 | 0.48287 | 0.56068 | 0.56068 | 0.51361 |
| BAAI/bge-multilingual-gemma2 | 0.47904 | 0.55472 | 0.55472 | 0.50916 |
| intfloat/multilingual-e5-large-instruct | 0.47842 | 0.55435 | 0.55435 | 0.50826 |
| intfloat/multilingual-e5-base | 0.46950 | 0.54490 | 0.54490 | 0.49947 |
| intfloat/e5-mistral-7b-instruct | 0.46772 | 0.54394 | 0.54394 | 0.49781 |
| Alibaba-NLP/gte-multilingual-base | 0.46469 | 0.53744 | 0.53744 | 0.49353 |
| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.46633 | 0.53625 | 0.53625 | 0.49429 |
| openai/text-embedding-3-large | 0.44884 | 0.51688 | 0.51688 | 0.47572 |
| Salesforce/SFR-Embedding-2_R | 0.43748 | 0.50815 | 0.50815 | 0.46504 |
| upskyy/bge-m3-korean | 0.43125 | 0.50245 | 0.50245 | 0.45945 |
| jhgan/ko-sroberta-multitask | 0.33788 | 0.38497 | 0.38497 | 0.35678 |
### Top-k 3
| Model | Average Recall | Average Precision | Average NDCG | Average F1 |
|-----------------------------------------|----------------------|------------------------|-------------------|-----------------|
| **nlpai-lab/KURE-v1** | **0.68678** | **0.28711** | **0.65538** | **0.39835** |
| dragonkue/BGE-m3-ko | 0.67834 | 0.28385 | 0.64950 | 0.39378 |
| BAAI/bge-m3 | 0.67526 | 0.28374 | 0.64556 | 0.39291 |
| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.67128 | 0.28193 | 0.64042 | 0.39072 |
| intfloat/multilingual-e5-large | 0.65807 | 0.27777 | 0.62822 | 0.38423 |
| nlpai-lab/KoE5 | 0.65174 | 0.27329 | 0.62369 | 0.37882 |
| BAAI/bge-multilingual-gemma2 | 0.64415 | 0.27416 | 0.61105 | 0.37782 |
| jinaai/jina-embeddings-v3 | 0.64116 | 0.27165 | 0.60954 | 0.37511 |
| intfloat/multilingual-e5-large-instruct | 0.64353 | 0.27040 | 0.60790 | 0.37453 |
| Alibaba-NLP/gte-multilingual-base | 0.63744 | 0.26404 | 0.59695 | 0.36764 |
| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.63163 | 0.25937 | 0.59237 | 0.36263 |
| intfloat/multilingual-e5-base | 0.62099 | 0.26144 | 0.59179 | 0.36203 |
| intfloat/e5-mistral-7b-instruct | 0.62087 | 0.26144 | 0.58917 | 0.36188 |
| openai/text-embedding-3-large | 0.61035 | 0.25356 | 0.57329 | 0.35270 |
| Salesforce/SFR-Embedding-2_R | 0.60001 | 0.25253 | 0.56346 | 0.34952 |
| upskyy/bge-m3-korean | 0.59215 | 0.25076 | 0.55722 | 0.34623 |
| jhgan/ko-sroberta-multitask | 0.46930 | 0.18994 | 0.43293 | 0.26696 |
### Top-k 5
| Model | Average Recall | Average Precision | Average NDCG | Average F1 |
|-----------------------------------------|----------------------|------------------------|-------------------|-----------------|
| **nlpai-lab/KURE-v1** | **0.73851** | **0.19130** | **0.67479** | **0.29903** |
| dragonkue/BGE-m3-ko | 0.72517 | 0.18799 | 0.66692 | 0.29401 |
| BAAI/bge-m3 | 0.72954 | 0.18975 | 0.66615 | 0.29632 |
| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.72962 | 0.18875 | 0.66236 | 0.29542 |
| nlpai-lab/KoE5 | 0.70820 | 0.18287 | 0.64499 | 0.28628 |
| intfloat/multilingual-e5-large | 0.70124 | 0.18316 | 0.64402 | 0.28588 |
| BAAI/bge-multilingual-gemma2 | 0.70258 | 0.18556 | 0.63338 | 0.28851 |
| jinaai/jina-embeddings-v3 | 0.69933 | 0.18256 | 0.63133 | 0.28505 |
| intfloat/multilingual-e5-large-instruct | 0.69018 | 0.17838 | 0.62486 | 0.27933 |
| Alibaba-NLP/gte-multilingual-base | 0.69365 | 0.17789 | 0.61896 | 0.27879 |
| intfloat/multilingual-e5-base | 0.67250 | 0.17406 | 0.61119 | 0.27247 |
| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.67447 | 0.17114 | 0.60952 | 0.26943 |
| intfloat/e5-mistral-7b-instruct | 0.67449 | 0.17484 | 0.60935 | 0.27349 |
| openai/text-embedding-3-large | 0.66365 | 0.17004 | 0.59389 | 0.26677 |
| Salesforce/SFR-Embedding-2_R | 0.65622 | 0.17018 | 0.58494 | 0.26612 |
| upskyy/bge-m3-korean | 0.65477 | 0.17015 | 0.58073 | 0.26589 |
| jhgan/ko-sroberta-multitask | 0.53136 | 0.13264 | 0.45879 | 0.20976 |
### Top-k 10
| Model | Average Recall | Average Precision | Average NDCG | Average F1 |
|-----------------------------------------|----------------------|------------------------|-------------------|-----------------|
| **nlpai-lab/KURE-v1** | **0.79682** | **0.10624** | **0.69473** | **0.18524** |
| dragonkue/BGE-m3-ko | 0.78450 | 0.10492 | 0.68748 | 0.18288 |
| BAAI/bge-m3 | 0.79195 | 0.10592 | 0.68723 | 0.18456 |
| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.78669 | 0.10462 | 0.68189 | 0.18260 |
| intfloat/multilingual-e5-large | 0.75902 | 0.10147 | 0.66370 | 0.17693 |
| nlpai-lab/KoE5 | 0.75296 | 0.09937 | 0.66012 | 0.17369 |
| BAAI/bge-multilingual-gemma2 | 0.76153 | 0.10364 | 0.65330 | 0.18003 |
| jinaai/jina-embeddings-v3 | 0.76277 | 0.10240 | 0.65290 | 0.17843 |
| intfloat/multilingual-e5-large-instruct | 0.74851 | 0.09888 | 0.64451 | 0.17283 |
| Alibaba-NLP/gte-multilingual-base | 0.75631 | 0.09938 | 0.64025 | 0.17363 |
| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.74092 | 0.09607 | 0.63258 | 0.16847 |
| intfloat/multilingual-e5-base | 0.73512 | 0.09717 | 0.63216 | 0.16977 |
| intfloat/e5-mistral-7b-instruct | 0.73795 | 0.09777 | 0.63076 | 0.17078 |
| openai/text-embedding-3-large | 0.72946 | 0.09571 | 0.61670 | 0.16739 |
| Salesforce/SFR-Embedding-2_R | 0.71662 | 0.09546 | 0.60589 | 0.16651 |
| upskyy/bge-m3-korean | 0.71895 | 0.09583 | 0.60258 | 0.16712 |
| jhgan/ko-sroberta-multitask | 0.61225 | 0.07826 | 0.48687 | 0.13757 |
<br/>
## Training Details
- KURE-v1은 [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3)를 기반으로 fine-tuning된 모델입니다.
- KoE5는 [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large)를 기반으로 fine-tuning된 모델입니다.
### Training Data
**KURE-v1**
- 한국어 query-document-hard_negative(5개) 데이터 쌍
- 약 2,000,000 examples
**KoE5**
- [ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0)
- 한국어 query-document-hard_negative(1개) 데이터 쌍 (open data)
- 약 700,000+ examples
### Training Procedure
**KURE-v1**
- loss: [CachedGISTEmbedLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedgistembedloss)
- batch size: 4096
- learning rate: 2e-05
- epochs: 1
**KoE5**
- loss: [CachedMultipleNegativesRankingLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedmultiplenegativesrankingloss)
- batch size: 512
- learning rate: 1e-05
- epochs: 1
<br/>
## 주의사항
- KoE5 사용 시, prefix를 붙여 주어야 합니다. (query: {query}, passage: {document})
## License
- ```MIT```
## Citation
If you find our paper or models helpful, please consider cite as follows:
```text
@inproceedings{jang2025kure,
title={KURE: Embedding Model for Korean-Specific Retrieval},
author={Jang, Youngjoon and Son, Junyoung and Lee, Taemin and Hong, Seongtae and Park, JeongBae and Lim, Heuiseok},
booktitle={Annual Conference on Human and Language Technology},
pages={129--134},
year={2025},
organization={Human and Language Technology}
},
@inproceedings{jang2024koe5,
title={KoE5: A New Dataset and Model for Improving Korean Embedding Performance},
author={Jang, Youngjoon and Son, Junyoung and Park, Chanjun and Choi, Soonwoo and Lee, Byeonggoo and Lee, Taemin and Lim, Heuiseok},
booktitle={Annual Conference on Human and Language Technology},
pages={239--244},
year={2024},
organization={Human and Language Technology}
}
```
================================================
FILE: README_EN.md
================================================
# 🔎 KURE: Korea University Retrieval Embedding model
## Update Logs
- 2024.12.21: [🤗 KURE-v1](https://huggingface.co/nlpai-lab/KURE-v1) and the [MTEB-ko-retrieval Leaderboard](https://github.com/nlpai-lab/KURE/blob/main/README_EN.md#mteb-ko-retrieval-leaderboard) have been released.
- 2024.10.02: [🤗 KoE5](https://huggingface.co/nlpai-lab/KoE5) and [🤗 ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0) have been released.
---
<br>
KURE is a Korean-specific embedding model developed by Korea University's [NLP & AI Lab](http://nlp.korea.ac.kr/) and [HIAI Research Institute](http://hiai.korea.ac.kr).
We are excited to release KURE.
<br/>
## How to Use KURE Models
### With sentence-transformers
```bash
pip install sentence-transformers
```
You can run the model with the following example code.
```python
from sentence_transformers import SentenceTransformer
# Download from the 🤗 Hub
model = SentenceTransformer("nlpai-lab/KURE-v1")
# model = SentenceTransformer("nlpai-lab/KoE5")
# Run inference
sentences = [
'헌법과 법원조직법은 어떤 방식을 통해 기본권 보장 등의 다양한 법적 모색을 가능하게 했어',
'4. 시사점과 개선방향 앞서 살펴본 바와 같이 우리 헌법과 ｢법원조직 법｣은 대법원 구성을 다양화하여 기본권 보장과 민주주의 확립에 있어 다각적인 법적 모색을 가능하게 하는 것을 근본 규범으로 하고 있다. 더욱이 합의체로서의 대법원 원리를 채택하고 있는 것 역시 그 구성의 다양성을 요청하는 것으로 해석된다. 이와 같은 관점에서 볼 때 현직 법원장급 고위법관을 중심으로 대법원을 구성하는 관행은 개선할 필요가 있는 것으로 보인다.',
'연방헌법재판소는 2001년 1월 24일 5:3의 다수견해로 「법원조직법」 제169조 제2문이 헌법에 합치된다는 판결을 내렸음 ○ 5인의 다수 재판관은 소송관계인의 인격권 보호, 공정한 절차의 보장과 방해받지 않는 법과 진실 발견 등을 근거로 하여 텔레비전 촬영에 대한 절대적인 금지를 헌법에 합치하는 것으로 보았음 ○ 그러나 나머지 3인의 재판관은 행정법원의 소송절차는 특별한 인격권 보호의 이익도 없으며, 텔레비전 공개주의로 인해 법과 진실 발견의 과정이 언제나 위태롭게 되는 것은 아니라면서 반대의견을 제시함 ○ 왜냐하면 행정법원의 소송절차에서는 소송당사자가 개인적으로 직접 심리에 참석하기보다는 변호사가 참석하는 경우가 많으며, 심리대상도 사실문제가 아닌 법률문제가 대부분이기 때문이라는 것임 □ 한편, 연방헌법재판소는 「연방헌법재판소법」(Bundesverfassungsgerichtsgesetz: BVerfGG) 제17a조에 따라 제한적이나마 재판에 대한 방송을 허용하고 있음 ○ 「연방헌법재판소법」 제17조에서 「법원조직법」 제14절 내지 제16절의 규정을 준용하도록 하고 있지만, 녹음이나 촬영을 통한 재판공개와 관련하여서는 「법원조직법」과 다른 내용을 규정하고 있음',
]
embeddings = model.encode(sentences)
print(embeddings.shape)
# [3, 1024]
# Get the similarity scores for the embeddings
similarities = model.similarity(embeddings, embeddings)
print(similarities)
# Results for KURE-v1
# tensor([[1.0000, 0.6967, 0.5306],
# [0.6967, 1.0000, 0.4427],
# [0.5306, 0.4427, 1.0000]])
# Results for KoE5
# tensor([[1.0000, 0.6721, 0.3897],
# [0.6721, 1.0000, 0.3740],
# [0.3897, 0.3740, 1.0000]])
```
<br/>
## MTEB-ko-retrieval Leaderboard
We evaluated our models on all Korean Retrieval Benchmarks registered in [MTEB](https://github.com/embeddings-benchmark/mteb).
### Korean Retrieval Benchmark
- [Ko-StrategyQA](https://huggingface.co/datasets/taeminlee/Ko-StrategyQA): Korean ODQA multi-hop retrieval dataset (Translated from StrategyQA).
- [AutoRAGRetrieval](https://huggingface.co/datasets/yjoonjang/markers_bm): A Korean document retrieval dataset constructed by parsing PDFs from 5 domains: finance, public, medical, legal, and commerce.
- [MIRACLRetrieval](https://huggingface.co/datasets/miracl/miracl): Wikipedia-based Korean document retrieval dataset.
- [PublicHealthQA](https://huggingface.co/datasets/xhluca/publichealth-qa): Korean document retrieval dataset for the medical and public health domains.
- [BelebeleRetrieval](https://huggingface.co/datasets/facebook/belebele): FLORES-200 based Korean document retrieval dataset.
- [MrTidyRetrieval](https://huggingface.co/datasets/mteb/mrtidy): Wikipedia-based Korean document retrieval dataset.
- [MultiLongDocRetrieval](https://huggingface.co/datasets/Shitao/MLDR): Korean long-document retrieval dataset from various domains.
- [XPQARetrieval](https://huggingface.co/datasets/jinaai/xpqa): Korean document retrieval dataset from various domains.
### Evaluation code
You can add a model to `evaluate.py` to evaluate it using MTEB.
```bash
cd eval
pip install -r requirements.txt
python evaluate.py
```
### Leaderboard
We visualize the evaluation results for all models on all tasks via streamlit.
```bash
streamlit run leaderboard.py
```
Below are the average results for all models across all benchmark datasets.
Detailed results can be found in the `eval/results` folder.
### Top-k 1
| Model | Average Recall | Average Precision | Average NDCG | Average F1 |
|-----------------------------------------|----------------------|------------------------|-------------------|-----------------|
| **nlpai-lab/KURE-v1** | **0.52640** | **0.60551** | **0.60551** | **0.55784** |
| dragonkue/BGE-m3-ko | 0.52361 | 0.60394 | 0.60394 | 0.55535 |
| BAAI/bge-m3 | 0.51778 | 0.59846 | 0.59846 | 0.54998 |
| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.51246 | 0.59384 | 0.59384 | 0.54489 |
| nlpai-lab/KoE5 | 0.50157 | 0.57790 | 0.57790 | 0.53178 |
| intfloat/multilingual-e5-large | 0.50052 | 0.57727 | 0.57727 | 0.53122 |
| jinaai/jina-embeddings-v3 | 0.48287 | 0.56068 | 0.56068 | 0.51361 |
| BAAI/bge-multilingual-gemma2 | 0.47904 | 0.55472 | 0.55472 | 0.50916 |
| intfloat/multilingual-e5-large-instruct | 0.47842 | 0.55435 | 0.55435 | 0.50826 |
| intfloat/multilingual-e5-base | 0.46950 | 0.54490 | 0.54490 | 0.49947 |
| intfloat/e5-mistral-7b-instruct | 0.46772 | 0.54394 | 0.54394 | 0.49781 |
| Alibaba-NLP/gte-multilingual-base | 0.46469 | 0.53744 | 0.53744 | 0.49353 |
| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.46633 | 0.53625 | 0.53625 | 0.49429 |
| openai/text-embedding-3-large | 0.44884 | 0.51688 | 0.51688 | 0.47572 |
| Salesforce/SFR-Embedding-2_R | 0.43748 | 0.50815 | 0.50815 | 0.46504 |
| upskyy/bge-m3-korean | 0.43125 | 0.50245 | 0.50245 | 0.45945 |
| jhgan/ko-sroberta-multitask | 0.33788 | 0.38497 | 0.38497 | 0.35678 |
### Top-k 3
| Model | Average Recall | Average Precision | Average NDCG | Average F1 |
|-----------------------------------------|----------------------|------------------------|-------------------|-----------------|
| **nlpai-lab/KURE-v1** | **0.68678** | **0.28711** | **0.65538** | **0.39835** |
| dragonkue/BGE-m3-ko | 0.67834 | 0.28385 | 0.64950 | 0.39378 |
| BAAI/bge-m3 | 0.67526 | 0.28374 | 0.64556 | 0.39291 |
| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.67128 | 0.28193 | 0.64042 | 0.39072 |
| intfloat/multilingual-e5-large | 0.65807 | 0.27777 | 0.62822 | 0.38423 |
| nlpai-lab/KoE5 | 0.65174 | 0.27329 | 0.62369 | 0.37882 |
| BAAI/bge-multilingual-gemma2 | 0.64415 | 0.27416 | 0.61105 | 0.37782 |
| jinaai/jina-embeddings-v3 | 0.64116 | 0.27165 | 0.60954 | 0.37511 |
| intfloat/multilingual-e5-large-instruct | 0.64353 | 0.27040 | 0.60790 | 0.37453 |
| Alibaba-NLP/gte-multilingual-base | 0.63744 | 0.26404 | 0.59695 | 0.36764 |
| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.63163 | 0.25937 | 0.59237 | 0.36263 |
| intfloat/multilingual-e5-base | 0.62099 | 0.26144 | 0.59179 | 0.36203 |
| intfloat/e5-mistral-7b-instruct | 0.62087 | 0.26144 | 0.58917 | 0.36188 |
| openai/text-embedding-3-large | 0.61035 | 0.25356 | 0.57329 | 0.35270 |
| Salesforce/SFR-Embedding-2_R | 0.60001 | 0.25253 | 0.56346 | 0.34952 |
| upskyy/bge-m3-korean | 0.59215 | 0.25076 | 0.55722 | 0.34623 |
| jhgan/ko-sroberta-multitask | 0.46930 | 0.18994 | 0.43293 | 0.26696 |
### Top-k 5
| Model | Average Recall | Average Precision | Average NDCG | Average F1 |
|-----------------------------------------|----------------------|------------------------|-------------------|-----------------|
| **nlpai-lab/KURE-v1** | **0.73851** | **0.19130** | **0.67479** | **0.29903** |
| dragonkue/BGE-m3-ko | 0.72517 | 0.18799 | 0.66692 | 0.29401 |
| BAAI/bge-m3 | 0.72954 | 0.18975 | 0.66615 | 0.29632 |
| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.72962 | 0.18875 | 0.66236 | 0.29542 |
| nlpai-lab/KoE5 | 0.70820 | 0.18287 | 0.64499 | 0.28628 |
| intfloat/multilingual-e5-large | 0.70124 | 0.18316 | 0.64402 | 0.28588 |
| BAAI/bge-multilingual-gemma2 | 0.70258 | 0.18556 | 0.63338 | 0.28851 |
| jinaai/jina-embeddings-v3 | 0.69933 | 0.18256 | 0.63133 | 0.28505 |
| intfloat/multilingual-e5-large-instruct | 0.69018 | 0.17838 | 0.62486 | 0.27933 |
| Alibaba-NLP/gte-multilingual-base | 0.69365 | 0.17789 | 0.61896 | 0.27879 |
| intfloat/multilingual-e5-base | 0.67250 | 0.17406 | 0.61119 | 0.27247 |
| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.67447 | 0.17114 | 0.60952 | 0.26943 |
| intfloat/e5-mistral-7b-instruct | 0.67449 | 0.17484 | 0.60935 | 0.27349 |
| openai/text-embedding-3-large | 0.66365 | 0.17004 | 0.59389 | 0.26677 |
| Salesforce/SFR-Embedding-2_R | 0.65622 | 0.17018 | 0.58494 | 0.26612 |
| upskyy/bge-m3-korean | 0.65477 | 0.17015 | 0.58073 | 0.26589 |
| jhgan/ko-sroberta-multitask | 0.53136 | 0.13264 | 0.45879 | 0.20976 |
### Top-k 10
| Model | Average Recall | Average Precision | Average NDCG | Average F1 |
|-----------------------------------------|----------------------|------------------------|-------------------|-----------------|
| **nlpai-lab/KURE-v1** | **0.79682** | **0.10624** | **0.69473** | **0.18524** |
| dragonkue/BGE-m3-ko | 0.78450 | 0.10492 | 0.68748 | 0.18288 |
| BAAI/bge-m3 | 0.79195 | 0.10592 | 0.68723 | 0.18456 |
| Snowflake/snowflake-arctic-embed-l-v2.0 | 0.78669 | 0.10462 | 0.68189 | 0.18260 |
| intfloat/multilingual-e5-large | 0.75902 | 0.10147 | 0.66370 | 0.17693 |
| nlpai-lab/KoE5 | 0.75296 | 0.09937 | 0.66012 | 0.17369 |
| BAAI/bge-multilingual-gemma2 | 0.76153 | 0.10364 | 0.65330 | 0.18003 |
| jinaai/jina-embeddings-v3 | 0.76277 | 0.10240 | 0.65290 | 0.17843 |
| intfloat/multilingual-e5-large-instruct | 0.74851 | 0.09888 | 0.64451 | 0.17283 |
| Alibaba-NLP/gte-multilingual-base | 0.75631 | 0.09938 | 0.64025 | 0.17363 |
| Alibaba-NLP/gte-Qwen2-7B-instruct | 0.74092 | 0.09607 | 0.63258 | 0.16847 |
| intfloat/multilingual-e5-base | 0.73512 | 0.09717 | 0.63216 | 0.16977 |
| intfloat/e5-mistral-7b-instruct | 0.73795 | 0.09777 | 0.63076 | 0.17078 |
| openai/text-embedding-3-large | 0.72946 | 0.09571 | 0.61670 | 0.16739 |
| Salesforce/SFR-Embedding-2_R | 0.71662 | 0.09546 | 0.60589 | 0.16651 |
| upskyy/bge-m3-korean | 0.71895 | 0.09583 | 0.60258 | 0.16712 |
| jhgan/ko-sroberta-multitask | 0.61225 | 0.07826 | 0.48687 | 0.13757 |
<br/>
## Training Details
- KURE-v1 is a model fine-tuned from [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3).
- KoE5 is a model fine-tuned from [intfloat/multilingual-e5-large](https://huggingface.co/intfloat/multilingual-e5-large).
### Training Data
**KURE-v1**
- Korean query-document-hard_negative(5) pairs
- Approx. 2,000,000 examples
**KoE5**
- [ko-triplet-v1.0](https://huggingface.co/datasets/nlpai-lab/ko-triplet-v1.0)
- Korean query-document-hard_negative(1) pairs (open data)
- Approx. 700,000+ examples
### Training Procedure
**KURE-v1**
- loss: [CachedGISTEmbedLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedgistembedloss)
- batch size: 4096
- learning rate: 2e-05
- epochs: 1
**KoE5**
- loss: [CachedMultipleNegativesRankingLoss](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#cachedmultiplenegativesrankingloss)
- batch size: 512
- learning rate: 1e-05
- epochs: 1
<br/>
## Important Notes
- When using KoE5, you must add a prefix for each input: (e.g., `query: {query}`, `passage: {document}`)
## License
- ```MIT```
## Citation
If you find our paper or models helpful, please consider citing them as follows:
```text
@misc{KURE,
publisher = {Youngjoon Jang, Junyoung Son, Taemin Lee},
year = {2024},
url = {https://github.com/nlpai-lab/KURE}
},
@misc{KoE5,
author = {NLP & AI Lab and Human-Inspired AI research},
title = {KoE5: A New Dataset and Model for Improving Korean Embedding Performance},
year = {2024},
publisher = {Youngjoon Jang, Junyoung Son, Taemin Lee},
journal = {GitHub repository},
howpublished = {\url{https://drive.google.com/file/d/1wB02XGFH5v18iJYSYB0oJkWFYxH0ftoJ/view}},
}
```
```
================================================
FILE: eval/evaluate.py
================================================
"""Benchmarking all datasets constituting the MTEB Korean leaderboard & average scores"""
from __future__ import annotations
import os
import logging
from multiprocessing import Process, current_process
import torch
from sentence_transformers import SentenceTransformer
from sentence_transformers.models import StaticEmbedding
import mteb
from mteb import MTEB, get_tasks
from mteb.encoder_interface import PromptType
from mteb.models.sentence_transformer_wrapper import SentenceTransformerWrapper
from mteb.models.instruct_wrapper import instruct_wrapper
import argparse
from dotenv import load_dotenv
from setproctitle import setproctitle
import traceback
import logging
load_dotenv() # for OPENAI
parser = argparse.ArgumentParser(description="Extract contexts")
parser.add_argument('--quantize', default=False, type=bool, help='quantize embeddings')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("main")
TASK_LIST_CLASSIFICATION = []
TASK_LIST_CLUSTERING = []
TASK_LIST_PAIR_CLASSIFICATION = []
TASK_LIST_RERANKING = []
TASK_LIST_RETRIEVAL = [
"Ko-StrategyQA",
"AutoRAGRetrieval",
"MIRACLRetrieval", # 시간이 오래 걸림 주의
"PublicHealthQA",
"BelebeleRetrieval",
"MrTidyRetrieval", # 시간이 오래 걸림 주의
"MultiLongDocRetrieval",
"XPQARetrieval",
"Tatoeba"
]
TASK_LIST_STS = []
TASK_LIST = (
TASK_LIST_CLASSIFICATION
+ TASK_LIST_CLUSTERING
+ TASK_LIST_PAIR_CLASSIFICATION
+ TASK_LIST_RERANKING
+ TASK_LIST_RETRIEVAL
+ TASK_LIST_STS
)
# MIRACL, MrTidy는 평가 시 시간이 오래 걸리기 때문에, 태스크별로 나누어 multiprocessing으로 평가합니다.
# 필요 시 GPU 번호를 다르게 조정해 주세요.
TASK_LIST_RETRIEVAL_GPU_MAPPING = {
0: [
"Ko-StrategyQA",
"AutoRAGRetrieval",
"PublicHealthQA",
"BelebeleRetrieval",
"XPQARetrieval",
"MultiLongDocRetrieval",
],
1: ["MIRACLRetrieval"],
2: ["MrTidyRetrieval"],
}
model_names = [
# my_model_directory
]
model_names = [
# "Salesforce/SFR-Embedding-2_R", # 4096
# "Alibaba-NLP/gte-Qwen2-7B-instruct", # 8192
# "BAAI/bge-multilingual-gemma2", # 8192
# "intfloat/e5-mistral-7b-instruct", # 32768
# "intfloat/multilingual-e5-large-instruct", # 512
# "openai/text-embedding-3-large", # 8191
# "Alibaba-NLP/gte-multilingual-base",
# "intfloat/multilingual-e5-base", # 512
# "intfloat/multilingual-e5-large", # 512
# "jinaai/jina-embeddings-v3", # 8192
# "jhgan/ko-sroberta-multitask", # 128
# "BAAI/bge-m3", # 8192
# "nlpai-lab/KoE5", # 512
# "dragonkue/BGE-m3-ko", # 8192
# "Snowflake/snowflake-arctic-embed-l-v2.0", # 8192,
# "nlpai-lab/KURE-v1", # 8192,
"nomic-ai/nomic-embed-text-v2-moe"
] + model_names
def evaluate_model(model_name, gpu_id, tasks):
try:
# Set the environment variable for the specific GPU
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
model = None
if not os.path.exists(model_name): # hf에 등록된 모델의 경우
if "m2v" in model_name: # model2vec의 경우: 모델명에 m2v를 포함시켜주어야 model2vec 모델로 인식합니다.
static_embedding = StaticEmbedding.from_model2vec(model_name)
model = SentenceTransformer(modules=[static_embedding], model_kwargs={"attn_implementation": "sdpa"})
else:
if model_name == "nlpai-lab/KoE5" or model_name == "KU-HIAI-ONTHEIT/ontheit-large-v1_1" :
# mE5 기반의 모델이므로, 해당 프롬프트를 추가시킵니다.
model_prompts = {
PromptType.query.value: "query: ",
PromptType.passage.value: "passage: ",
}
model = SentenceTransformerWrapper(model=model_name, model_prompts=model_prompts, model_kwargs={"attn_implementation": "sdpa"})
elif model_name == "nomic-ai/nomic-embed-text-v2-moe":
model_prompts = {
PromptType.query.value: "search_query: ",
PromptType.passage.value: "search_document: ",
}
model = SentenceTransformerWrapper(model=model_name, model_prompts=model_prompts, model_kwargs={"attn_implementation": "sdpa"}, trust_remote_code=True)
elif model_name == "BAAI/bge-multilingual-gemma2":
# mbge-gemma2의 경우, mteb에서 지원하지 않습니다. 따라서, instruct_wrapper를 사용합니다.
instruction_template = '<instruct>{instruction}\n<query>'
model = instruct_wrapper(
model_name_or_path=model_name,
instruction_template=instruction_template,
attn="cccc",
pooling_method="lasttoken",
mode="embedding",
torch_dtype=torch.float16,
normalized=True,
)
elif model_name == "Snowflake/snowflake-arctic-embed-l-v2.0":
# mteb에서 Snowflake 모델을 지원하지 않으므로, Snowflake에서 사용하는 "query: " prefix를 임의로 추가합니다.
model_prompts = {
PromptType.query.value: "query: ",
}
model = SentenceTransformerWrapper(model=model_name, model_prompts=model_prompts, model_kwargs={"attn_implementation": "sdpa"})
else:
# mteb에 등록된 모델의 경우, 프롬프트/prefix 등을 포함하여 평가할 수 있습니다. 등록되지 않은 경우, sentence-transformer를 사용하여 불러옵니다.
model = mteb.get_model(model_name)
else: # 직접 학습한 모델의 경우
file_name = os.path.join(model_name, "model.safetensors")
if os.path.exists(file_name):
if "m2v" in model_name: # model2vec의 경우: 모델명에 m2v를 포함시켜주어야 model2vec 모델로 인식합니다.
static_embedding = StaticEmbedding.from_model2vec(model_name)
model = SentenceTransformer(modules=[static_embedding], model_kwargs={"attn_implementation": "sdpa"})
else:
model = mteb.get_model(model_name, model_kwargs={"attn_implementation": "sdpa"})
if model:
setproctitle(f"{model_name}-{gpu_id}")
print(f"Running tasks: {tasks} / {model_name} on GPU {gpu_id} in process {current_process().name}")
evaluation = MTEB(
tasks=get_tasks(tasks=tasks, languages=["kor-Kore", "kor-Hang", "kor_Hang"])
)
# 48GB VRAM 기준 적합한 batch sizes
if "multilingual-e5" in model_name or "KoE5" in model_name or "ontheit" in model_name or "nomic" in model_name:
batch_size = 512
elif "jina" in model_name:
batch_size = 8
elif "bge-m3" in model_name or "Snowflake" in model_name:
batch_size = 32
elif "gemma2" in model_name:
batch_size = 256
elif "Salesforce" in model_name:
batch_size = 128
else:
batch_size = 64
if args.quantize: # quantized model의 경우
evaluation.run(
model,
output_folder=f"results/{model_name}-quantized",
encode_kwargs={"batch_size": batch_size, "precision": "binary"},
)
else:
evaluation.run(
model,
output_folder=f"results/{model_name}",
encode_kwargs={"batch_size": batch_size},
)
except Exception as ex:
print(ex)
traceback.print_exc()
if __name__ == "__main__":
processes = []
for gpu_id, tasks in TASK_LIST_RETRIEVAL_GPU_MAPPING.items():
for model_name in model_names:
p = Process(target=evaluate_model, args=(model_name, gpu_id, tasks))
p.start()
processes.append(p)
for p in processes:
p.join()
================================================
FILE: eval/leaderboard.py
================================================
import streamlit as st
import os
import json
import pandas as pd
st.set_page_config(layout="wide")
def app():
data = {}
avg_data = {} # average score를 저장하기 위한 dictionary
tasks = [
"Ko-StrategyQA",
"AutoRAGRetrieval",
"MIRACLRetrieval",
"PublicHealthQA",
"BelebeleRetrieval",
"MrTidyRetrieval",
"MultiLongDocRetrieval",
"XPQARetrieval"
]
top_k_types = ["top1", "top3", "top5", "top10"]
score_types = {
"top1": ["recall_at_1", "precision_at_1", "ndcg_at_1"],
"top3": ["recall_at_3", "precision_at_3", "ndcg_at_3"],
"top5": ["recall_at_5", "precision_at_5", "ndcg_at_5"],
"top10": ["recall_at_10", "precision_at_10", "ndcg_at_10"],
}
# 각 작업에 대한 데이터를 초기화
for task in tasks:
data[task] = {top_k: [] for top_k in top_k_types}
root_dir = "results"
# 데이터가 저장되어 있는 디렉토리의 모든 하위 폴더를 순회하면서 json 파일을 읽습니다.
for subdir, dirs, files in os.walk(root_dir):
for file in files:
for task in tasks:
if file == task + ".json":
with open(os.path.join(subdir, file)) as f:
d = json.load(f)
for top_k in top_k_types:
results = {}
for score in score_types[top_k]:
if "dev" in d["scores"] and "test" not in d["scores"]:
results[score] = d["scores"]["dev"][0][score]
elif "test" in d["scores"] and "dev" not in d["scores"]:
results[score] = d["scores"]["test"][0][score]
else:
# dev, test를 모두 가지고 있는 평가 데이터셋을 위함
results[score] = (d["scores"]["dev"][0][score] + d["scores"]["test"][0][score]) / 2
# f1 score 직접 계산
f1_score = (
2 * (results[score_types[top_k][1]] * results[score_types[top_k][0]]) / (results[score_types[top_k][1]]+ results[score_types[top_k][0]])
if (results[score_types[top_k][1]]+ results[score_types[top_k][0]])> 0
else 0
)
data[task][top_k].append(
(
os.path.relpath(subdir, root_dir),
results[score_types[top_k][0]],
results[score_types[top_k][1]],
results[score_types[top_k][2]],
f1_score,
)
)
# 각 작업에 대해 top1, top3, top5, top10 점수 표시
for task in tasks:
st.markdown(f"# {task}")
for top_k in top_k_types:
st.markdown(f"## {top_k.capitalize()} Scores")
df = pd.DataFrame(
data[task][top_k],
columns=[
"Subdir",
f"Recall_{top_k}",
f"Precision_{top_k}",
f"NDCG_{top_k}",
f"F1_{top_k}",
],
)
df = df.sort_values(by=f"NDCG_{top_k}", ascending=False)
st.dataframe(df, use_container_width=True)
# 각 모델의 평균 점수 계산
for subdir, recall, precision, ndcg, f1 in data[task][top_k]:
if subdir not in avg_data:
avg_data[subdir] = {
k: [[], [], [], []] for k in top_k_types
}
avg_data[subdir][top_k][0].append(recall)
avg_data[subdir][top_k][1].append(precision)
avg_data[subdir][top_k][2].append(ndcg)
avg_data[subdir][top_k][3].append(f1)
# 각 모델 별 평균 점수 계산 후 출력
st.markdown("# Average Scores")
for top_k in top_k_types:
avg_results = []
for model in avg_data:
recall_avg = (
sum(avg_data[model][top_k][0]) / len(avg_data[model][top_k][0])
if avg_data[model][top_k][0]
else 0
)
precision_avg = (
sum(avg_data[model][top_k][1]) / len(avg_data[model][top_k][1])
if avg_data[model][top_k][1]
else 0
)
ndcg_avg = (
sum(avg_data[model][top_k][2]) / len(avg_data[model][top_k][2])
if avg_data[model][top_k][2]
else 0
)
f1_avg = (
sum(avg_data[model][top_k][3]) / len(avg_data[model][top_k][3])
if avg_data[model][top_k][3]
else 0
)
avg_results.append([model, recall_avg, precision_avg, ndcg_avg, f1_avg])
avg_df = pd.DataFrame(
avg_results,
columns=[
"Model",
f"Average Recall_{top_k}",
f"Average Precision_{top_k}",
f"Average NDCG_{top_k}",
f"Average F1_{top_k}",
],
)
avg_df = avg_df.sort_values(by=f"Average NDCG_{top_k}", ascending=False)
st.markdown(f"## {top_k.capitalize()} Average Scores")
st.dataframe(avg_df, use_container_width=True)
if __name__ == "__main__":
app()
================================================
FILE: eval/requirements.txt
================================================
mteb
python-dotenv
streamlit
setproctitle
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/AutoRAGRetrieval.json
================================================
{
"dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c",
"evaluation_time": 846.0815389156342,
"kg_co2_emissions": null,
"mteb_version": "1.19.4",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"kor-Hang"
],
"main_score": 0.76682,
"map_at_1": 0.60526,
"map_at_10": 0.71514,
"map_at_100": 0.71846,
"map_at_1000": 0.71854,
"map_at_20": 0.71719,
"map_at_3": 0.69444,
"map_at_5": 0.7019,
"mrr_at_1": 0.6052631578947368,
"mrr_at_10": 0.7151350598719021,
"mrr_at_100": 0.7184641134066749,
"mrr_at_1000": 0.7185366086944813,
"mrr_at_20": 0.7171920671920672,
"mrr_at_3": 0.6944444444444446,
"mrr_at_5": 0.7019005847953218,
"nauc_map_at_1000_diff1": 0.7644331655442053,
"nauc_map_at_1000_max": 0.006603581947104919,
"nauc_map_at_1000_std": -0.37084270701564565,
"nauc_map_at_100_diff1": 0.7644201027610169,
"nauc_map_at_100_max": 0.0067126570840267364,
"nauc_map_at_100_std": -0.37090267941427896,
"nauc_map_at_10_diff1": 0.762543492536473,
"nauc_map_at_10_max": 0.016618337725207714,
"nauc_map_at_10_std": -0.36230366527185476,
"nauc_map_at_1_diff1": 0.8065079498138608,
"nauc_map_at_1_max": -0.008481292674104184,
"nauc_map_at_1_std": -0.3152574052168019,
"nauc_map_at_20_diff1": 0.7647616354525286,
"nauc_map_at_20_max": 0.010913309413363433,
"nauc_map_at_20_std": -0.3693593056765434,
"nauc_map_at_3_diff1": 0.749148318778548,
"nauc_map_at_3_max": -0.013924305023451019,
"nauc_map_at_3_std": -0.39203803015117983,
"nauc_map_at_5_diff1": 0.7599115394000969,
"nauc_map_at_5_max": -0.008945775041782456,
"nauc_map_at_5_std": -0.38416486703433883,
"nauc_mrr_at_1000_diff1": 0.7644331655442053,
"nauc_mrr_at_1000_max": 0.006603581947104919,
"nauc_mrr_at_1000_std": -0.37084270701564565,
"nauc_mrr_at_100_diff1": 0.7644201027610169,
"nauc_mrr_at_100_max": 0.0067126570840267364,
"nauc_mrr_at_100_std": -0.37090267941427896,
"nauc_mrr_at_10_diff1": 0.762543492536473,
"nauc_mrr_at_10_max": 0.016618337725207714,
"nauc_mrr_at_10_std": -0.36230366527185476,
"nauc_mrr_at_1_diff1": 0.8065079498138608,
"nauc_mrr_at_1_max": -0.008481292674104184,
"nauc_mrr_at_1_std": -0.3152574052168019,
"nauc_mrr_at_20_diff1": 0.7647616354525286,
"nauc_mrr_at_20_max": 0.010913309413363433,
"nauc_mrr_at_20_std": -0.3693593056765434,
"nauc_mrr_at_3_diff1": 0.749148318778548,
"nauc_mrr_at_3_max": -0.013924305023451019,
"nauc_mrr_at_3_std": -0.39203803015117983,
"nauc_mrr_at_5_diff1": 0.7599115394000969,
"nauc_mrr_at_5_max": -0.008945775041782456,
"nauc_mrr_at_5_std": -0.38416486703433883,
"nauc_ndcg_at_1000_diff1": 0.7622350239005066,
"nauc_ndcg_at_1000_max": 0.017371318088600977,
"nauc_ndcg_at_1000_std": -0.37321055393166663,
"nauc_ndcg_at_100_diff1": 0.7619593636566969,
"nauc_ndcg_at_100_max": 0.01972561053123169,
"nauc_ndcg_at_100_std": -0.37452959411278264,
"nauc_ndcg_at_10_diff1": 0.7575322927973843,
"nauc_ndcg_at_10_max": 0.07021633482084594,
"nauc_ndcg_at_10_std": -0.3375267305826439,
"nauc_ndcg_at_1_diff1": 0.8065079498138608,
"nauc_ndcg_at_1_max": -0.008481292674104184,
"nauc_ndcg_at_1_std": -0.3152574052168019,
"nauc_ndcg_at_20_diff1": 0.7652290444346898,
"nauc_ndcg_at_20_max": 0.048815872410838686,
"nauc_ndcg_at_20_std": -0.36658508429606196,
"nauc_ndcg_at_3_diff1": 0.7321065045076984,
"nauc_ndcg_at_3_max": -0.01155970025952776,
"nauc_ndcg_at_3_std": -0.4161891854409051,
"nauc_ndcg_at_5_diff1": 0.7516239012254612,
"nauc_ndcg_at_5_max": -6.352015603966895e-05,
"nauc_ndcg_at_5_std": -0.39852520659387436,
"nauc_precision_at_1000_diff1": 1.0,
"nauc_precision_at_1000_max": 1.0,
"nauc_precision_at_1000_std": 1.0,
"nauc_precision_at_100_diff1": 0.7224066624409552,
"nauc_precision_at_100_max": 0.35752776970742256,
"nauc_precision_at_100_std": -0.5637901180614636,
"nauc_precision_at_10_diff1": 0.7309271500210811,
"nauc_precision_at_10_max": 0.6817757315332228,
"nauc_precision_at_10_std": 0.00574347193803287,
"nauc_precision_at_1_diff1": 0.8065079498138608,
"nauc_precision_at_1_max": -0.008481292674104184,
"nauc_precision_at_1_std": -0.3152574052168019,
"nauc_precision_at_20_diff1": 0.8293909916468436,
"nauc_precision_at_20_max": 0.7008965455883202,
"nauc_precision_at_20_std": -0.2706904285228724,
"nauc_precision_at_3_diff1": 0.6681562238166904,
"nauc_precision_at_3_max": -0.00047679284331604674,
"nauc_precision_at_3_std": -0.5075608401380758,
"nauc_precision_at_5_diff1": 0.7204265059450241,
"nauc_precision_at_5_max": 0.046535938023941965,
"nauc_precision_at_5_std": -0.4530486350784987,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 0.7224066624409369,
"nauc_recall_at_100_max": 0.35752776970737554,
"nauc_recall_at_100_std": -0.5637901180615443,
"nauc_recall_at_10_diff1": 0.7309271500210796,
"nauc_recall_at_10_max": 0.6817757315332232,
"nauc_recall_at_10_std": 0.005743471938036119,
"nauc_recall_at_1_diff1": 0.8065079498138608,
"nauc_recall_at_1_max": -0.008481292674104184,
"nauc_recall_at_1_std": -0.3152574052168019,
"nauc_recall_at_20_diff1": 0.8293909916468486,
"nauc_recall_at_20_max": 0.700896545588321,
"nauc_recall_at_20_std": -0.27069042852286784,
"nauc_recall_at_3_diff1": 0.6681562238166918,
"nauc_recall_at_3_max": -0.0004767928433143103,
"nauc_recall_at_3_std": -0.5075608401380753,
"nauc_recall_at_5_diff1": 0.7204265059450234,
"nauc_recall_at_5_max": 0.04653593802394211,
"nauc_recall_at_5_std": -0.45304863507849813,
"ndcg_at_1": 0.60526,
"ndcg_at_10": 0.76682,
"ndcg_at_100": 0.78099,
"ndcg_at_1000": 0.78225,
"ndcg_at_20": 0.77376,
"ndcg_at_3": 0.72128,
"ndcg_at_5": 0.73524,
"precision_at_1": 0.60526,
"precision_at_10": 0.09298,
"precision_at_100": 0.00991,
"precision_at_1000": 0.001,
"precision_at_20": 0.04781,
"precision_at_3": 0.26608,
"precision_at_5": 0.16667,
"recall_at_1": 0.60526,
"recall_at_10": 0.92982,
"recall_at_100": 0.99123,
"recall_at_1000": 1.0,
"recall_at_20": 0.95614,
"recall_at_3": 0.79825,
"recall_at_5": 0.83333
}
]
},
"task_name": "AutoRAGRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/BelebeleRetrieval.json
================================================
{
"dataset_revision": "75b399394a9803252cfec289d103de462763db7c",
"evaluation_time": 537.1708543300629,
"kg_co2_emissions": null,
"mteb_version": "1.19.4",
"scores": {
"test": [
{
"hf_subset": "kor_Hang-kor_Hang",
"languages": [
"kor-Hang",
"kor-Hang"
],
"main_score": 0.94808,
"map_at_1": 0.91444,
"map_at_10": 0.93774,
"map_at_100": 0.93854,
"map_at_1000": 0.93856,
"map_at_20": 0.93803,
"map_at_3": 0.93426,
"map_at_5": 0.93648,
"mrr_at_1": 0.9144444444444444,
"mrr_at_10": 0.9377385361552029,
"mrr_at_100": 0.9385426232247298,
"mrr_at_1000": 0.9385570215532866,
"mrr_at_20": 0.9380331489498157,
"mrr_at_3": 0.9342592592592593,
"mrr_at_5": 0.9364814814814815,
"nauc_map_at_1000_diff1": 0.9341587574819448,
"nauc_map_at_1000_max": 0.7978508469331898,
"nauc_map_at_1000_std": -0.1247788227834717,
"nauc_map_at_100_diff1": 0.9341510130917275,
"nauc_map_at_100_max": 0.7978982067470314,
"nauc_map_at_100_std": -0.12453847765211529,
"nauc_map_at_10_diff1": 0.9344160375639369,
"nauc_map_at_10_max": 0.7999522285999181,
"nauc_map_at_10_std": -0.12061906912864259,
"nauc_map_at_1_diff1": 0.9313119187068765,
"nauc_map_at_1_max": 0.7544472334388295,
"nauc_map_at_1_std": -0.19743655037772562,
"nauc_map_at_20_diff1": 0.934104227715159,
"nauc_map_at_20_max": 0.7991964531277118,
"nauc_map_at_20_std": -0.12080639055631145,
"nauc_map_at_3_diff1": 0.9355558185715604,
"nauc_map_at_3_max": 0.8014531634249938,
"nauc_map_at_3_std": -0.14484948909140807,
"nauc_map_at_5_diff1": 0.9333012116411193,
"nauc_map_at_5_max": 0.7994864612511658,
"nauc_map_at_5_std": -0.1304815803872555,
"nauc_mrr_at_1000_diff1": 0.9341587574819448,
"nauc_mrr_at_1000_max": 0.7978508469331898,
"nauc_mrr_at_1000_std": -0.1247788227834717,
"nauc_mrr_at_100_diff1": 0.9341510130917275,
"nauc_mrr_at_100_max": 0.7978982067470314,
"nauc_mrr_at_100_std": -0.12453847765211529,
"nauc_mrr_at_10_diff1": 0.9344160375639369,
"nauc_mrr_at_10_max": 0.7999522285999181,
"nauc_mrr_at_10_std": -0.12061906912864259,
"nauc_mrr_at_1_diff1": 0.9313119187068765,
"nauc_mrr_at_1_max": 0.7544472334388295,
"nauc_mrr_at_1_std": -0.19743655037772562,
"nauc_mrr_at_20_diff1": 0.934104227715159,
"nauc_mrr_at_20_max": 0.7991964531277118,
"nauc_mrr_at_20_std": -0.12080639055631145,
"nauc_mrr_at_3_diff1": 0.9355558185715604,
"nauc_mrr_at_3_max": 0.8014531634249938,
"nauc_mrr_at_3_std": -0.14484948909140807,
"nauc_mrr_at_5_diff1": 0.9333012116411193,
"nauc_mrr_at_5_max": 0.7994864612511658,
"nauc_mrr_at_5_std": -0.1304815803872555,
"nauc_ndcg_at_1000_diff1": 0.9344502542270329,
"nauc_ndcg_at_1000_max": 0.8043530829653767,
"nauc_ndcg_at_1000_std": -0.10775125619222786,
"nauc_ndcg_at_100_diff1": 0.9344072352372144,
"nauc_ndcg_at_100_max": 0.8055734617052932,
"nauc_ndcg_at_100_std": -0.10129337761974706,
"nauc_ndcg_at_10_diff1": 0.9356156657712474,
"nauc_ndcg_at_10_max": 0.8171392419913772,
"nauc_ndcg_at_10_std": -0.07482350219227087,
"nauc_ndcg_at_1_diff1": 0.9313119187068765,
"nauc_ndcg_at_1_max": 0.7544472334388295,
"nauc_ndcg_at_1_std": -0.19743655037772562,
"nauc_ndcg_at_20_diff1": 0.9344540935531565,
"nauc_ndcg_at_20_max": 0.8146097937010235,
"nauc_ndcg_at_20_std": -0.07418267112495723,
"nauc_ndcg_at_3_diff1": 0.936819790095856,
"nauc_ndcg_at_3_max": 0.8178989129857199,
"nauc_ndcg_at_3_std": -0.13242670021197311,
"nauc_ndcg_at_5_diff1": 0.9321587219670335,
"nauc_ndcg_at_5_max": 0.8148177822979191,
"nauc_ndcg_at_5_std": -0.10319206027442693,
"nauc_precision_at_1000_diff1": NaN,
"nauc_precision_at_1000_max": NaN,
"nauc_precision_at_1000_std": NaN,
"nauc_precision_at_100_diff1": 0.9346405228757582,
"nauc_precision_at_100_max": 1.0,
"nauc_precision_at_100_std": 0.9346405228757582,
"nauc_precision_at_10_diff1": 0.9482570806100177,
"nauc_precision_at_10_max": 0.9679686689490625,
"nauc_precision_at_10_std": 0.36248573503474296,
"nauc_precision_at_1_diff1": 0.9313119187068765,
"nauc_precision_at_1_max": 0.7544472334388295,
"nauc_precision_at_1_std": -0.19743655037772562,
"nauc_precision_at_20_diff1": 0.9379084967320215,
"nauc_precision_at_20_max": 0.9702769996887645,
"nauc_precision_at_20_std": 0.4604419545596087,
"nauc_precision_at_3_diff1": 0.9424992219109852,
"nauc_precision_at_3_max": 0.8935444548189662,
"nauc_precision_at_3_std": -0.07892416225750032,
"nauc_precision_at_5_diff1": 0.923332295881318,
"nauc_precision_at_5_max": 0.9042258878860141,
"nauc_precision_at_5_std": 0.06276584707956435,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 0.934640522875857,
"nauc_recall_at_100_max": 1.0,
"nauc_recall_at_100_std": 0.934640522875857,
"nauc_recall_at_10_diff1": 0.9482570806100225,
"nauc_recall_at_10_max": 0.9679686689490574,
"nauc_recall_at_10_std": 0.3624857350347621,
"nauc_recall_at_1_diff1": 0.9313119187068765,
"nauc_recall_at_1_max": 0.7544472334388295,
"nauc_recall_at_1_std": -0.19743655037772562,
"nauc_recall_at_20_diff1": 0.9379084967320268,
"nauc_recall_at_20_max": 0.97027699968877,
"nauc_recall_at_20_std": 0.4604419545596044,
"nauc_recall_at_3_diff1": 0.9424992219109841,
"nauc_recall_at_3_max": 0.893544454818967,
"nauc_recall_at_3_std": -0.07892416225749478,
"nauc_recall_at_5_diff1": 0.9233322958813207,
"nauc_recall_at_5_max": 0.9042258878860211,
"nauc_recall_at_5_std": 0.06276584707957272,
"ndcg_at_1": 0.91444,
"ndcg_at_10": 0.94808,
"ndcg_at_100": 0.95193,
"ndcg_at_1000": 0.95223,
"ndcg_at_20": 0.949,
"ndcg_at_3": 0.94086,
"ndcg_at_5": 0.94492,
"precision_at_1": 0.91444,
"precision_at_10": 0.098,
"precision_at_100": 0.00998,
"precision_at_1000": 0.001,
"precision_at_20": 0.04917,
"precision_at_3": 0.32,
"precision_at_5": 0.194,
"recall_at_1": 0.91444,
"recall_at_10": 0.98,
"recall_at_100": 0.99778,
"recall_at_1000": 1.0,
"recall_at_20": 0.98333,
"recall_at_3": 0.96,
"recall_at_5": 0.97
},
{
"hf_subset": "kor_Hang-eng_Latn",
"languages": [
"kor-Hang",
"eng-Latn"
],
"main_score": 0.95397,
"map_at_1": 0.91778,
"map_at_10": 0.94328,
"map_at_100": 0.94402,
"map_at_1000": 0.94402,
"map_at_20": 0.9439,
"map_at_3": 0.9387,
"map_at_5": 0.94204,
"mrr_at_1": 0.9177777777777778,
"mrr_at_10": 0.9432760141093474,
"mrr_at_100": 0.9440192855380678,
"mrr_at_1000": 0.944022209514676,
"mrr_at_20": 0.9438995324516478,
"mrr_at_3": 0.9387037037037037,
"mrr_at_5": 0.9420370370370371,
"nauc_map_at_1000_diff1": 0.9379736356471905,
"nauc_map_at_1000_max": 0.818228835537799,
"nauc_map_at_1000_std": -0.3568158538211896,
"nauc_map_at_100_diff1": 0.9379768753993101,
"nauc_map_at_100_max": 0.8182383297830593,
"nauc_map_at_100_std": -0.35674498481109457,
"nauc_map_at_10_diff1": 0.9371832681770706,
"nauc_map_at_10_max": 0.8194699100005315,
"nauc_map_at_10_std": -0.3534046804367552,
"nauc_map_at_1_diff1": 0.9365016276781002,
"nauc_map_at_1_max": 0.7871400812577283,
"nauc_map_at_1_std": -0.3898352133646259,
"nauc_map_at_20_diff1": 0.9380057116884054,
"nauc_map_at_20_max": 0.8186263215762201,
"nauc_map_at_20_std": -0.3552421679266783,
"nauc_map_at_3_diff1": 0.9384134318379933,
"nauc_map_at_3_max": 0.8234983822330538,
"nauc_map_at_3_std": -0.36390165330986185,
"nauc_map_at_5_diff1": 0.9376907312445761,
"nauc_map_at_5_max": 0.8229708283739466,
"nauc_map_at_5_std": -0.3448556632450642,
"nauc_mrr_at_1000_diff1": 0.9379736356471905,
"nauc_mrr_at_1000_max": 0.818228835537799,
"nauc_mrr_at_1000_std": -0.3568158538211896,
"nauc_mrr_at_100_diff1": 0.9379768753993101,
"nauc_mrr_at_100_max": 0.8182383297830593,
"nauc_mrr_at_100_std": -0.35674498481109457,
"nauc_mrr_at_10_diff1": 0.9371832681770706,
"nauc_mrr_at_10_max": 0.8194699100005315,
"nauc_mrr_at_10_std": -0.3534046804367552,
"nauc_mrr_at_1_diff1": 0.9365016276781002,
"nauc_mrr_at_1_max": 0.7871400812577283,
"nauc_mrr_at_1_std": -0.3898352133646259,
"nauc_mrr_at_20_diff1": 0.9380057116884054,
"nauc_mrr_at_20_max": 0.8186263215762201,
"nauc_mrr_at_20_std": -0.3552421679266783,
"nauc_mrr_at_3_diff1": 0.9384134318379933,
"nauc_mrr_at_3_max": 0.8234983822330538,
"nauc_mrr_at_3_std": -0.36390165330986185,
"nauc_mrr_at_5_diff1": 0.9376907312445761,
"nauc_mrr_at_5_max": 0.8229708283739466,
"nauc_mrr_at_5_std": -0.3448556632450642,
"nauc_ndcg_at_1000_diff1": 0.9382191754421236,
"nauc_ndcg_at_1000_max": 0.8224201913320449,
"nauc_ndcg_at_1000_std": -0.34630458284754967,
"nauc_ndcg_at_100_diff1": 0.9384049804187261,
"nauc_ndcg_at_100_max": 0.8229542601540525,
"nauc_ndcg_at_100_std": -0.3422555904085183,
"nauc_ndcg_at_10_diff1": 0.9357447204568637,
"nauc_ndcg_at_10_max": 0.829222262177988,
"nauc_ndcg_at_10_std": -0.3281301084025004,
"nauc_ndcg_at_1_diff1": 0.9365016276781002,
"nauc_ndcg_at_1_max": 0.7871400812577283,
"nauc_ndcg_at_1_std": -0.3898352133646259,
"nauc_ndcg_at_20_diff1": 0.9386644241730514,
"nauc_ndcg_at_20_max": 0.8257288395033384,
"nauc_ndcg_at_20_std": -0.3312173812179968,
"nauc_ndcg_at_3_diff1": 0.9388035548318442,
"nauc_ndcg_at_3_max": 0.8379465154015449,
"nauc_ndcg_at_3_std": -0.3444570338354112,
"nauc_ndcg_at_5_diff1": 0.9374737365012075,
"nauc_ndcg_at_5_max": 0.8384461172346422,
"nauc_ndcg_at_5_std": -0.30419406475924055,
"nauc_precision_at_1000_diff1": NaN,
"nauc_precision_at_1000_max": NaN,
"nauc_precision_at_1000_std": NaN,
"nauc_precision_at_100_diff1": 1.0,
"nauc_precision_at_100_max": 1.0,
"nauc_precision_at_100_std": 1.0,
"nauc_precision_at_10_diff1": 0.9138266417678141,
"nauc_precision_at_10_max": 0.93191721132897,
"nauc_precision_at_10_std": -0.0321350762527383,
"nauc_precision_at_1_diff1": 0.9365016276781002,
"nauc_precision_at_1_max": 0.7871400812577283,
"nauc_precision_at_1_std": -0.3898352133646259,
"nauc_precision_at_20_diff1": 0.967320261437914,
"nauc_precision_at_20_max": 1.0,
"nauc_precision_at_20_std": 0.5262605042016776,
"nauc_precision_at_3_diff1": 0.9404408227937671,
"nauc_precision_at_3_max": 0.9062756415697625,
"nauc_precision_at_3_std": -0.24736157089098976,
"nauc_precision_at_5_diff1": 0.9352240896358476,
"nauc_precision_at_5_max": 0.9526143790849654,
"nauc_precision_at_5_std": 0.01713352007468603,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 1.0,
"nauc_recall_at_100_max": 1.0,
"nauc_recall_at_100_std": 1.0,
"nauc_recall_at_10_diff1": 0.9138266417678085,
"nauc_recall_at_10_max": 0.9319172113289713,
"nauc_recall_at_10_std": -0.03213507625271808,
"nauc_recall_at_1_diff1": 0.9365016276781002,
"nauc_recall_at_1_max": 0.7871400812577283,
"nauc_recall_at_1_std": -0.3898352133646259,
"nauc_recall_at_20_diff1": 0.9673202614379001,
"nauc_recall_at_20_max": 1.0,
"nauc_recall_at_20_std": 0.5262605042016953,
"nauc_recall_at_3_diff1": 0.9404408227937675,
"nauc_recall_at_3_max": 0.9062756415697596,
"nauc_recall_at_3_std": -0.24736157089098623,
"nauc_recall_at_5_diff1": 0.9352240896358568,
"nauc_recall_at_5_max": 0.9526143790849627,
"nauc_recall_at_5_std": 0.017133520074696214,
"ndcg_at_1": 0.91778,
"ndcg_at_10": 0.95397,
"ndcg_at_100": 0.95691,
"ndcg_at_1000": 0.95704,
"ndcg_at_20": 0.95622,
"ndcg_at_3": 0.94507,
"ndcg_at_5": 0.95104,
"precision_at_1": 0.91778,
"precision_at_10": 0.09867,
"precision_at_100": 0.00999,
"precision_at_1000": 0.001,
"precision_at_20": 0.04978,
"precision_at_3": 0.32111,
"precision_at_5": 0.19556,
"recall_at_1": 0.91778,
"recall_at_10": 0.98667,
"recall_at_100": 0.99889,
"recall_at_1000": 1.0,
"recall_at_20": 0.99556,
"recall_at_3": 0.96333,
"recall_at_5": 0.97778
},
{
"hf_subset": "eng_Latn-kor_Hang",
"languages": [
"eng-Latn",
"kor-Hang"
],
"main_score": 0.92939,
"map_at_1": 0.87778,
"map_at_10": 0.91394,
"map_at_100": 0.91475,
"map_at_1000": 0.91478,
"map_at_20": 0.91456,
"map_at_3": 0.90759,
"map_at_5": 0.9127,
"mrr_at_1": 0.8777777777777778,
"mrr_at_10": 0.9139395943562613,
"mrr_at_100": 0.9147527793478226,
"mrr_at_1000": 0.9147804033940741,
"mrr_at_20": 0.9145571747606064,
"mrr_at_3": 0.9075925925925927,
"mrr_at_5": 0.912703703703704,
"nauc_map_at_1000_diff1": 0.9034621090151944,
"nauc_map_at_1000_max": 0.8166056541545396,
"nauc_map_at_1000_std": -0.04958063210793317,
"nauc_map_at_100_diff1": 0.9034260698542368,
"nauc_map_at_100_max": 0.8166661124959635,
"nauc_map_at_100_std": -0.04932797829928419,
"nauc_map_at_10_diff1": 0.9032733282636335,
"nauc_map_at_10_max": 0.8176267163973103,
"nauc_map_at_10_std": -0.046047883529355904,
"nauc_map_at_1_diff1": 0.9129032258064524,
"nauc_map_at_1_max": 0.7948743587514139,
"nauc_map_at_1_std": -0.0959960003477958,
"nauc_map_at_20_diff1": 0.9033278279584531,
"nauc_map_at_20_max": 0.8168441424897516,
"nauc_map_at_20_std": -0.04894110672076424,
"nauc_map_at_3_diff1": 0.8994646682977531,
"nauc_map_at_3_max": 0.8155491427278718,
"nauc_map_at_3_std": -0.05738561942402418,
"nauc_map_at_5_diff1": 0.9023811940876922,
"nauc_map_at_5_max": 0.8159936946768515,
"nauc_map_at_5_std": -0.04821727179963084,
"nauc_mrr_at_1000_diff1": 0.9034621090151944,
"nauc_mrr_at_1000_max": 0.8166056541545396,
"nauc_mrr_at_1000_std": -0.04958063210793317,
"nauc_mrr_at_100_diff1": 0.9034260698542368,
"nauc_mrr_at_100_max": 0.8166661124959635,
"nauc_mrr_at_100_std": -0.04932797829928419,
"nauc_mrr_at_10_diff1": 0.9032733282636335,
"nauc_mrr_at_10_max": 0.8176267163973103,
"nauc_mrr_at_10_std": -0.046047883529355904,
"nauc_mrr_at_1_diff1": 0.9129032258064524,
"nauc_mrr_at_1_max": 0.7948743587514139,
"nauc_mrr_at_1_std": -0.0959960003477958,
"nauc_mrr_at_20_diff1": 0.9033278279584531,
"nauc_mrr_at_20_max": 0.8168441424897516,
"nauc_mrr_at_20_std": -0.04894110672076424,
"nauc_mrr_at_3_diff1": 0.8994646682977531,
"nauc_mrr_at_3_max": 0.8155491427278718,
"nauc_mrr_at_3_std": -0.05738561942402418,
"nauc_mrr_at_5_diff1": 0.9023811940876922,
"nauc_mrr_at_5_max": 0.8159936946768515,
"nauc_mrr_at_5_std": -0.04821727179963084,
"nauc_ndcg_at_1000_diff1": 0.9025162201523518,
"nauc_ndcg_at_1000_max": 0.8210647790377918,
"nauc_ndcg_at_1000_std": -0.033259752555848905,
"nauc_ndcg_at_100_diff1": 0.9019832601650294,
"nauc_ndcg_at_100_max": 0.822705958126526,
"nauc_ndcg_at_100_std": -0.025706848528490346,
"nauc_ndcg_at_10_diff1": 0.901289557068697,
"nauc_ndcg_at_10_max": 0.8284852301803537,
"nauc_ndcg_at_10_std": -0.006171497791011456,
"nauc_ndcg_at_1_diff1": 0.9129032258064524,
"nauc_ndcg_at_1_max": 0.7948743587514139,
"nauc_ndcg_at_1_std": -0.0959960003477958,
"nauc_ndcg_at_20_diff1": 0.901386120900766,
"nauc_ndcg_at_20_max": 0.8251101196933834,
"nauc_ndcg_at_20_std": -0.018990543647164154,
"nauc_ndcg_at_3_diff1": 0.8937606553985863,
"nauc_ndcg_at_3_max": 0.8227786405631717,
"nauc_ndcg_at_3_std": -0.032438022169551876,
"nauc_ndcg_at_5_diff1": 0.8991524294049409,
"nauc_ndcg_at_5_max": 0.8244725886241099,
"nauc_ndcg_at_5_std": -0.010211310284992422,
"nauc_precision_at_1000_diff1": NaN,
"nauc_precision_at_1000_max": NaN,
"nauc_precision_at_1000_std": NaN,
"nauc_precision_at_100_diff1": 0.85585901027076,
"nauc_precision_at_100_max": 1.0,
"nauc_precision_at_100_std": 0.8068394024276223,
"nauc_precision_at_10_diff1": 0.8851762927393119,
"nauc_precision_at_10_max": 0.9416433239962604,
"nauc_precision_at_10_std": 0.42265795206971,
"nauc_precision_at_1_diff1": 0.9129032258064524,
"nauc_precision_at_1_max": 0.7948743587514139,
"nauc_precision_at_1_std": -0.0959960003477958,
"nauc_precision_at_20_diff1": 0.8761042878689921,
"nauc_precision_at_20_max": 0.947209653091999,
"nauc_precision_at_20_std": 0.4398836457660034,
"nauc_precision_at_3_diff1": 0.8670494864612496,
"nauc_precision_at_3_max": 0.8558543417366978,
"nauc_precision_at_3_std": 0.08870214752567777,
"nauc_precision_at_5_diff1": 0.8773109243697506,
"nauc_precision_at_5_max": 0.8860255213196399,
"nauc_precision_at_5_std": 0.2848428260192923,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 0.8558590102707876,
"nauc_recall_at_100_max": 1.0,
"nauc_recall_at_100_std": 0.8068394024276523,
"nauc_recall_at_10_diff1": 0.8851762927393169,
"nauc_recall_at_10_max": 0.9416433239962636,
"nauc_recall_at_10_std": 0.4226579520697196,
"nauc_recall_at_1_diff1": 0.9129032258064524,
"nauc_recall_at_1_max": 0.7948743587514139,
"nauc_recall_at_1_std": -0.0959960003477958,
"nauc_recall_at_20_diff1": 0.8761042878690056,
"nauc_recall_at_20_max": 0.9472096530920127,
"nauc_recall_at_20_std": 0.4398836457660057,
"nauc_recall_at_3_diff1": 0.8670494864612522,
"nauc_recall_at_3_max": 0.8558543417366946,
"nauc_recall_at_3_std": 0.08870214752567522,
"nauc_recall_at_5_diff1": 0.8773109243697501,
"nauc_recall_at_5_max": 0.8860255213196395,
"nauc_recall_at_5_std": 0.28484282601929706,
"ndcg_at_1": 0.87778,
"ndcg_at_10": 0.92939,
"ndcg_at_100": 0.93337,
"ndcg_at_1000": 0.93397,
"ndcg_at_20": 0.93163,
"ndcg_at_3": 0.91708,
"ndcg_at_5": 0.92626,
"precision_at_1": 0.87778,
"precision_at_10": 0.09767,
"precision_at_100": 0.00996,
"precision_at_1000": 0.001,
"precision_at_20": 0.04928,
"precision_at_3": 0.31481,
"precision_at_5": 0.19333,
"recall_at_1": 0.87778,
"recall_at_10": 0.97667,
"recall_at_100": 0.99556,
"recall_at_1000": 1.0,
"recall_at_20": 0.98556,
"recall_at_3": 0.94444,
"recall_at_5": 0.96667
}
]
},
"task_name": "BelebeleRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/Ko-StrategyQA.json
================================================
{
"dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c",
"evaluation_time": 9451.070049762726,
"kg_co2_emissions": null,
"mteb_version": "1.19.4",
"scores": {
"dev": [
{
"hf_subset": "default",
"languages": [
"kor-Hang"
],
"main_score": 0.8108,
"map_at_1": 0.49102,
"map_at_10": 0.76711,
"map_at_100": 0.77193,
"map_at_1000": 0.77217,
"map_at_20": 0.7704,
"map_at_3": 0.72297,
"map_at_5": 0.75609,
"mrr_at_1": 0.7567567567567568,
"mrr_at_10": 0.8184329418704417,
"mrr_at_100": 0.8199197488873609,
"mrr_at_1000": 0.820035921652267,
"mrr_at_20": 0.8191797589412719,
"mrr_at_3": 0.8124999999999999,
"mrr_at_5": 0.816216216216216,
"nauc_map_at_1000_diff1": 0.5860412800077854,
"nauc_map_at_1000_max": 0.5688615375646618,
"nauc_map_at_1000_std": 0.14232936442918717,
"nauc_map_at_100_diff1": 0.5862654281094708,
"nauc_map_at_100_max": 0.5693298489926267,
"nauc_map_at_100_std": 0.14289907790694886,
"nauc_map_at_10_diff1": 0.58259296969473,
"nauc_map_at_10_max": 0.5700053608148513,
"nauc_map_at_10_std": 0.1378994523753573,
"nauc_map_at_1_diff1": 0.632693918726285,
"nauc_map_at_1_max": 0.19506800049379597,
"nauc_map_at_1_std": -0.11312532728724334,
"nauc_map_at_20_diff1": 0.5849545323148008,
"nauc_map_at_20_max": 0.5699622379553662,
"nauc_map_at_20_std": 0.14174612526369168,
"nauc_map_at_3_diff1": 0.5695775561789569,
"nauc_map_at_3_max": 0.5001255236964945,
"nauc_map_at_3_std": 0.058437237332709745,
"nauc_map_at_5_diff1": 0.5759990071437616,
"nauc_map_at_5_max": 0.5586872929226386,
"nauc_map_at_5_std": 0.10931034514013828,
"nauc_mrr_at_1000_diff1": 0.6709973477564796,
"nauc_mrr_at_1000_max": 0.6282079865303426,
"nauc_mrr_at_1000_std": 0.2182570851932003,
"nauc_mrr_at_100_diff1": 0.6711170760242097,
"nauc_mrr_at_100_max": 0.628458884294849,
"nauc_mrr_at_100_std": 0.21873771212546203,
"nauc_mrr_at_10_diff1": 0.6708764756266502,
"nauc_mrr_at_10_max": 0.6309082959700567,
"nauc_mrr_at_10_std": 0.22073367752871997,
"nauc_mrr_at_1_diff1": 0.6512317513672372,
"nauc_mrr_at_1_max": 0.4938058683502111,
"nauc_mrr_at_1_std": 0.043607255153568676,
"nauc_mrr_at_20_diff1": 0.6708019682498405,
"nauc_mrr_at_20_max": 0.6295377015786073,
"nauc_mrr_at_20_std": 0.21860680581568911,
"nauc_mrr_at_3_diff1": 0.6749605035448772,
"nauc_mrr_at_3_max": 0.6308357607971359,
"nauc_mrr_at_3_std": 0.2123955334390889,
"nauc_mrr_at_5_diff1": 0.6707744820554383,
"nauc_mrr_at_5_max": 0.6326256995891799,
"nauc_mrr_at_5_std": 0.22072942902869305,
"nauc_ndcg_at_1000_diff1": 0.6193636658978984,
"nauc_ndcg_at_1000_max": 0.630697544541978,
"nauc_ndcg_at_1000_std": 0.2386622609694652,
"nauc_ndcg_at_100_diff1": 0.6249180286049728,
"nauc_ndcg_at_100_max": 0.6433073743489643,
"nauc_ndcg_at_100_std": 0.2567146683561388,
"nauc_ndcg_at_10_diff1": 0.6136536266134186,
"nauc_ndcg_at_10_max": 0.6489710228693197,
"nauc_ndcg_at_10_std": 0.24420829169202138,
"nauc_ndcg_at_1_diff1": 0.6512317513672372,
"nauc_ndcg_at_1_max": 0.4938058683502111,
"nauc_ndcg_at_1_std": 0.043607255153568676,
"nauc_ndcg_at_20_diff1": 0.6187284093130772,
"nauc_ndcg_at_20_max": 0.6481730608510515,
"nauc_ndcg_at_20_std": 0.24951834109610457,
"nauc_ndcg_at_3_diff1": 0.5955050016913125,
"nauc_ndcg_at_3_max": 0.5693055062275632,
"nauc_ndcg_at_3_std": 0.13404496786641223,
"nauc_ndcg_at_5_diff1": 0.6018163792854944,
"nauc_ndcg_at_5_max": 0.629072716288407,
"nauc_ndcg_at_5_std": 0.19753897996008837,
"nauc_precision_at_1000_diff1": -0.344189042020911,
"nauc_precision_at_1000_max": 0.06475013447391754,
"nauc_precision_at_1000_std": 0.23917858332558542,
"nauc_precision_at_100_diff1": -0.23914321934560318,
"nauc_precision_at_100_max": 0.1849182738696575,
"nauc_precision_at_100_std": 0.32206416448630437,
"nauc_precision_at_10_diff1": -0.16380898142137407,
"nauc_precision_at_10_max": 0.2985983037666543,
"nauc_precision_at_10_std": 0.33963783741465275,
"nauc_precision_at_1_diff1": 0.6512317513672372,
"nauc_precision_at_1_max": 0.4938058683502111,
"nauc_precision_at_1_std": 0.043607255153568676,
"nauc_precision_at_20_diff1": -0.18782381198510406,
"nauc_precision_at_20_max": 0.25344887941275157,
"nauc_precision_at_20_std": 0.3301393828208045,
"nauc_precision_at_3_diff1": -0.03798706450288487,
"nauc_precision_at_3_max": 0.41546117059530224,
"nauc_precision_at_3_std": 0.2841280644973789,
"nauc_precision_at_5_diff1": -0.11961371289466115,
"nauc_precision_at_5_max": 0.34773826390289747,
"nauc_precision_at_5_std": 0.3053693435767689,
"nauc_recall_at_1000_diff1": 0.6975728443917598,
"nauc_recall_at_1000_max": 0.9852414530686907,
"nauc_recall_at_1000_std": 0.9563648851837822,
"nauc_recall_at_100_diff1": 0.7357693746397532,
"nauc_recall_at_100_max": 0.970368714583283,
"nauc_recall_at_100_std": 0.8305978541126534,
"nauc_recall_at_10_diff1": 0.630960245824474,
"nauc_recall_at_10_max": 0.8657233835111986,
"nauc_recall_at_10_std": 0.5369361718154417,
"nauc_recall_at_1_diff1": 0.632693918726285,
"nauc_recall_at_1_max": 0.19506800049379597,
"nauc_recall_at_1_std": -0.11312532728724334,
"nauc_recall_at_20_diff1": 0.6587733846215134,
"nauc_recall_at_20_max": 0.9010277052483276,
"nauc_recall_at_20_std": 0.608188922151966,
"nauc_recall_at_3_diff1": 0.5766836091129506,
"nauc_recall_at_3_max": 0.6032568623874444,
"nauc_recall_at_3_std": 0.1834427343207938,
"nauc_recall_at_5_diff1": 0.5898441814477909,
"nauc_recall_at_5_max": 0.7572960798154303,
"nauc_recall_at_5_std": 0.34333987643488223,
"ndcg_at_1": 0.75676,
"ndcg_at_10": 0.8108,
"ndcg_at_100": 0.82549,
"ndcg_at_1000": 0.83146,
"ndcg_at_20": 0.81772,
"ndcg_at_3": 0.7644,
"ndcg_at_5": 0.79491,
"precision_at_1": 0.75676,
"precision_at_10": 0.16318,
"precision_at_100": 0.0176,
"precision_at_1000": 0.00185,
"precision_at_20": 0.08412,
"precision_at_3": 0.45946,
"precision_at_5": 0.30811,
"recall_at_1": 0.49102,
"recall_at_10": 0.87588,
"recall_at_100": 0.92891,
"recall_at_1000": 0.9676,
"recall_at_20": 0.89667,
"recall_at_3": 0.77636,
"recall_at_5": 0.838
}
]
},
"task_name": "Ko-StrategyQA"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MIRACLRetrieval.json
================================================
{
"dataset_revision": "main",
"evaluation_time": 529443.3452823162,
"kg_co2_emissions": null,
"mteb_version": "1.19.4",
"scores": {
"dev": [
{
"hf_subset": "ko",
"languages": [
"kor-Kore"
],
"main_score": 0.53375,
"map_at_1": 0.32154,
"map_at_10": 0.44673,
"map_at_100": 0.46751,
"map_at_1000": 0.46925,
"map_at_20": 0.45789,
"map_at_3": 0.40325,
"map_at_5": 0.42409,
"mrr_at_1": 0.5023474178403756,
"mrr_at_10": 0.6107105596542215,
"mrr_at_100": 0.6154599409080139,
"mrr_at_1000": 0.6155662638623027,
"mrr_at_20": 0.6140300142709334,
"mrr_at_3": 0.5829420970266039,
"mrr_at_5": 0.5989045383411579,
"nauc_map_at_1000_diff1": 0.36031682483777533,
"nauc_map_at_1000_max": 0.45390972288889114,
"nauc_map_at_1000_std": 0.13734187164243297,
"nauc_map_at_100_diff1": 0.3600105915368106,
"nauc_map_at_100_max": 0.4542942414546183,
"nauc_map_at_100_std": 0.13738385626349392,
"nauc_map_at_10_diff1": 0.37024010443796035,
"nauc_map_at_10_max": 0.45045624061818007,
"nauc_map_at_10_std": 0.11853986467680915,
"nauc_map_at_1_diff1": 0.4706687514348616,
"nauc_map_at_1_max": 0.3785598180609727,
"nauc_map_at_1_std": 0.019557004686001363,
"nauc_map_at_20_diff1": 0.36274647010544186,
"nauc_map_at_20_max": 0.4536888030295927,
"nauc_map_at_20_std": 0.12753823479536147,
"nauc_map_at_3_diff1": 0.38115606736353363,
"nauc_map_at_3_max": 0.4144717952993478,
"nauc_map_at_3_std": 0.09752636322817537,
"nauc_map_at_5_diff1": 0.37467387847101574,
"nauc_map_at_5_max": 0.43567861778923944,
"nauc_map_at_5_std": 0.10313974453104882,
"nauc_mrr_at_1000_diff1": 0.2852977064023687,
"nauc_mrr_at_1000_max": 0.398570864440513,
"nauc_mrr_at_1000_std": 0.20335934910198356,
"nauc_mrr_at_100_diff1": 0.2853112466469182,
"nauc_mrr_at_100_max": 0.39850219473785614,
"nauc_mrr_at_100_std": 0.20357699621251318,
"nauc_mrr_at_10_diff1": 0.2858037102372598,
"nauc_mrr_at_10_max": 0.39935268233483895,
"nauc_mrr_at_10_std": 0.20273336747754533,
"nauc_mrr_at_1_diff1": 0.35781125255772933,
"nauc_mrr_at_1_max": 0.3817075728005826,
"nauc_mrr_at_1_std": 0.15291749551166287,
"nauc_mrr_at_20_diff1": 0.28526035685683987,
"nauc_mrr_at_20_max": 0.399936960823763,
"nauc_mrr_at_20_std": 0.20434386179383293,
"nauc_mrr_at_3_diff1": 0.2894201723559571,
"nauc_mrr_at_3_max": 0.3844481810664284,
"nauc_mrr_at_3_std": 0.20303269450222397,
"nauc_mrr_at_5_diff1": 0.28541917051266164,
"nauc_mrr_at_5_max": 0.40044268989170173,
"nauc_mrr_at_5_std": 0.19709143657059006,
"nauc_ndcg_at_1000_diff1": 0.3077962815159766,
"nauc_ndcg_at_1000_max": 0.46609651208669883,
"nauc_ndcg_at_1000_std": 0.21539275920415737,
"nauc_ndcg_at_100_diff1": 0.3016914461573391,
"nauc_ndcg_at_100_max": 0.4713105342695115,
"nauc_ndcg_at_100_std": 0.2263944047923428,
"nauc_ndcg_at_10_diff1": 0.32321710435700557,
"nauc_ndcg_at_10_max": 0.4674512727191553,
"nauc_ndcg_at_10_std": 0.16902846173636296,
"nauc_ndcg_at_1_diff1": 0.35781125255772933,
"nauc_ndcg_at_1_max": 0.3817075728005826,
"nauc_ndcg_at_1_std": 0.15291749551166287,
"nauc_ndcg_at_20_diff1": 0.3012113569809693,
"nauc_ndcg_at_20_max": 0.47150220023564965,
"nauc_ndcg_at_20_std": 0.18771797034660115,
"nauc_ndcg_at_3_diff1": 0.3344897853051733,
"nauc_ndcg_at_3_max": 0.41103017207276366,
"nauc_ndcg_at_3_std": 0.16543980238878778,
"nauc_ndcg_at_5_diff1": 0.32596617209752865,
"nauc_ndcg_at_5_max": 0.44257031745396613,
"nauc_ndcg_at_5_std": 0.15271626934313168,
"nauc_precision_at_1000_diff1": -0.19240298740616193,
"nauc_precision_at_1000_max": -0.09319519616564716,
"nauc_precision_at_1000_std": 0.19996903865573903,
"nauc_precision_at_100_diff1": -0.20633671484264068,
"nauc_precision_at_100_max": -0.005120399255030897,
"nauc_precision_at_100_std": 0.29184767615959145,
"nauc_precision_at_10_diff1": -0.1063489415799065,
"nauc_precision_at_10_max": 0.24302859238436084,
"nauc_precision_at_10_std": 0.2998399174212745,
"nauc_precision_at_1_diff1": 0.35781125255772933,
"nauc_precision_at_1_max": 0.3817075728005826,
"nauc_precision_at_1_std": 0.15291749551166287,
"nauc_precision_at_20_diff1": -0.18982635217381524,
"nauc_precision_at_20_max": 0.15302492246844374,
"nauc_precision_at_20_std": 0.30148495252965213,
"nauc_precision_at_3_diff1": 0.051384039218975025,
"nauc_precision_at_3_max": 0.3415094720366865,
"nauc_precision_at_3_std": 0.3050973666769741,
"nauc_precision_at_5_diff1": -0.05159355651665696,
"nauc_precision_at_5_max": 0.2895404001134896,
"nauc_precision_at_5_std": 0.2870935784123534,
"nauc_recall_at_1000_diff1": -0.02540042414067673,
"nauc_recall_at_1000_max": 0.6914512479556674,
"nauc_recall_at_1000_std": 0.6952669376653589,
"nauc_recall_at_100_diff1": 0.06431828985652863,
"nauc_recall_at_100_max": 0.49768612041645394,
"nauc_recall_at_100_std": 0.5235483628705813,
"nauc_recall_at_10_diff1": 0.21299696608876725,
"nauc_recall_at_10_max": 0.46908580333257077,
"nauc_recall_at_10_std": 0.20166417606311324,
"nauc_recall_at_1_diff1": 0.4706687514348616,
"nauc_recall_at_1_max": 0.3785598180609727,
"nauc_recall_at_1_std": 0.019557004686001363,
"nauc_recall_at_20_diff1": 0.11674748568996735,
"nauc_recall_at_20_max": 0.4664820858052285,
"nauc_recall_at_20_std": 0.24984091736066763,
"nauc_recall_at_3_diff1": 0.2913781914497467,
"nauc_recall_at_3_max": 0.3799568988978954,
"nauc_recall_at_3_std": 0.13544116333758804,
"nauc_recall_at_5_diff1": 0.25704180765800555,
"nauc_recall_at_5_max": 0.43482488959261517,
"nauc_recall_at_5_std": 0.12481012511593734,
"ndcg_at_1": 0.50235,
"ndcg_at_10": 0.53375,
"ndcg_at_100": 0.59893,
"ndcg_at_1000": 0.61892,
"ndcg_at_20": 0.55959,
"ndcg_at_3": 0.48512,
"ndcg_at_5": 0.5,
"precision_at_1": 0.50235,
"precision_at_10": 0.12629,
"precision_at_100": 0.02056,
"precision_at_1000": 0.00246,
"precision_at_20": 0.07535,
"precision_at_3": 0.26917,
"precision_at_5": 0.19812,
"recall_at_1": 0.32154,
"recall_at_10": 0.62643,
"recall_at_100": 0.84398,
"recall_at_1000": 0.95114,
"recall_at_20": 0.69904,
"recall_at_3": 0.46787,
"recall_at_5": 0.5294
}
]
},
"task_name": "MIRACLRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MrTidyRetrieval.json
================================================
{
"dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7",
"evaluation_time": 523189.4379520416,
"kg_co2_emissions": null,
"mteb_version": "1.20.0",
"scores": {
"test": [
{
"hf_subset": "korean",
"languages": [
"kor-Kore"
],
"main_score": 0.46571,
"map_at_1": 0.27514,
"map_at_10": 0.39939,
"map_at_100": 0.41129,
"map_at_1000": 0.41168,
"map_at_20": 0.4066,
"map_at_3": 0.36448,
"map_at_5": 0.38421,
"mrr_at_1": 0.3016627078384798,
"mrr_at_10": 0.4210279757191871,
"mrr_at_100": 0.43137655437218475,
"mrr_at_1000": 0.4316617049477868,
"mrr_at_20": 0.42775991444354394,
"mrr_at_3": 0.39152810768012675,
"mrr_at_5": 0.40779889152810783,
"nauc_map_at_1000_diff1": 0.33293097264950955,
"nauc_map_at_1000_max": 0.3130945117290807,
"nauc_map_at_1000_std": 0.15320103165009255,
"nauc_map_at_100_diff1": 0.3329715076297884,
"nauc_map_at_100_max": 0.3129594820935785,
"nauc_map_at_100_std": 0.15338062022944135,
"nauc_map_at_10_diff1": 0.32887360247558767,
"nauc_map_at_10_max": 0.3108233813084867,
"nauc_map_at_10_std": 0.1546675312359396,
"nauc_map_at_1_diff1": 0.36727145137891976,
"nauc_map_at_1_max": 0.2835205184523079,
"nauc_map_at_1_std": 0.08284292069749208,
"nauc_map_at_20_diff1": 0.3312437874430616,
"nauc_map_at_20_max": 0.3132079222147372,
"nauc_map_at_20_std": 0.15360937950563022,
"nauc_map_at_3_diff1": 0.3359312089046139,
"nauc_map_at_3_max": 0.30787457325441847,
"nauc_map_at_3_std": 0.13849369378070164,
"nauc_map_at_5_diff1": 0.3399343435870189,
"nauc_map_at_5_max": 0.31547727388019076,
"nauc_map_at_5_std": 0.1446015036644677,
"nauc_mrr_at_1000_diff1": 0.34708453070365647,
"nauc_mrr_at_1000_max": 0.3327969140637137,
"nauc_mrr_at_1000_std": 0.16830876389746163,
"nauc_mrr_at_100_diff1": 0.3470604278609808,
"nauc_mrr_at_100_max": 0.3326620162101879,
"nauc_mrr_at_100_std": 0.16837297579442742,
"nauc_mrr_at_10_diff1": 0.3437214867731117,
"nauc_mrr_at_10_max": 0.3316098836220819,
"nauc_mrr_at_10_std": 0.1690527564620172,
"nauc_mrr_at_1_diff1": 0.3767527634408593,
"nauc_mrr_at_1_max": 0.3048804875104381,
"nauc_mrr_at_1_std": 0.10983635052464556,
"nauc_mrr_at_20_diff1": 0.3452672537657949,
"nauc_mrr_at_20_max": 0.33298105123957994,
"nauc_mrr_at_20_std": 0.1682974971024297,
"nauc_mrr_at_3_diff1": 0.349791941798552,
"nauc_mrr_at_3_max": 0.3344859077948214,
"nauc_mrr_at_3_std": 0.16473453655024795,
"nauc_mrr_at_5_diff1": 0.3537503263120288,
"nauc_mrr_at_5_max": 0.3353023453737103,
"nauc_mrr_at_5_std": 0.1662874495535466,
"nauc_ndcg_at_1000_diff1": 0.32908917733637794,
"nauc_ndcg_at_1000_max": 0.32613113076041883,
"nauc_ndcg_at_1000_std": 0.18277112937224538,
"nauc_ndcg_at_100_diff1": 0.329842647374385,
"nauc_ndcg_at_100_max": 0.3204276831537327,
"nauc_ndcg_at_100_std": 0.18695185786004517,
"nauc_ndcg_at_10_diff1": 0.308389151053235,
"nauc_ndcg_at_10_max": 0.31744735058957213,
"nauc_ndcg_at_10_std": 0.18928893750159045,
"nauc_ndcg_at_1_diff1": 0.3767527634408593,
"nauc_ndcg_at_1_max": 0.3048804875104381,
"nauc_ndcg_at_1_std": 0.10983635052464556,
"nauc_ndcg_at_20_diff1": 0.3142097652257374,
"nauc_ndcg_at_20_max": 0.3256140956842127,
"nauc_ndcg_at_20_std": 0.18825616208232035,
"nauc_ndcg_at_3_diff1": 0.32710712769551964,
"nauc_ndcg_at_3_max": 0.31887549789970226,
"nauc_ndcg_at_3_std": 0.1612799742774508,
"nauc_ndcg_at_5_diff1": 0.33316107860910377,
"nauc_ndcg_at_5_max": 0.32772903954728455,
"nauc_ndcg_at_5_std": 0.16985213318205114,
"nauc_precision_at_1000_diff1": 0.05823992403845274,
"nauc_precision_at_1000_max": 0.1253829515929163,
"nauc_precision_at_1000_std": 0.09610589777902924,
"nauc_precision_at_100_diff1": 0.14105102943548245,
"nauc_precision_at_100_max": 0.1440230375138777,
"nauc_precision_at_100_std": 0.17203917643895358,
"nauc_precision_at_10_diff1": 0.18331538719235707,
"nauc_precision_at_10_max": 0.28980912626029914,
"nauc_precision_at_10_std": 0.2805987246660366,
"nauc_precision_at_1_diff1": 0.3767527634408593,
"nauc_precision_at_1_max": 0.3048804875104381,
"nauc_precision_at_1_std": 0.10983635052464556,
"nauc_precision_at_20_diff1": 0.15619077660826267,
"nauc_precision_at_20_max": 0.2832576972655273,
"nauc_precision_at_20_std": 0.24994856873135776,
"nauc_precision_at_3_diff1": 0.30310011086092725,
"nauc_precision_at_3_max": 0.3488172732208653,
"nauc_precision_at_3_std": 0.2316356138771506,
"nauc_precision_at_5_diff1": 0.28818187989791355,
"nauc_precision_at_5_max": 0.34918912698140314,
"nauc_precision_at_5_std": 0.2403851697874218,
"nauc_recall_at_1000_diff1": 0.252208053700925,
"nauc_recall_at_1000_max": 0.539409617920156,
"nauc_recall_at_1000_std": 0.7654850168562043,
"nauc_recall_at_100_diff1": 0.31153747808028986,
"nauc_recall_at_100_max": 0.2589106974778827,
"nauc_recall_at_100_std": 0.4433876652036297,
"nauc_recall_at_10_diff1": 0.20384518056559342,
"nauc_recall_at_10_max": 0.29640975881808485,
"nauc_recall_at_10_std": 0.2876325943932951,
"nauc_recall_at_1_diff1": 0.36727145137891976,
"nauc_recall_at_1_max": 0.2835205184523079,
"nauc_recall_at_1_std": 0.08284292069749208,
"nauc_recall_at_20_diff1": 0.20428489732555838,
"nauc_recall_at_20_max": 0.3310372371548431,
"nauc_recall_at_20_std": 0.3143954063567378,
"nauc_recall_at_3_diff1": 0.2765682171149457,
"nauc_recall_at_3_max": 0.3119664143708647,
"nauc_recall_at_3_std": 0.19408329545045494,
"nauc_recall_at_5_diff1": 0.28498223121657856,
"nauc_recall_at_5_max": 0.3281071536534065,
"nauc_recall_at_5_std": 0.2149027801698047,
"ndcg_at_1": 0.30166,
"ndcg_at_10": 0.46571,
"ndcg_at_100": 0.52079,
"ndcg_at_1000": 0.53041,
"ndcg_at_20": 0.4905,
"ndcg_at_3": 0.39923,
"ndcg_at_5": 0.43194,
"precision_at_1": 0.30166,
"precision_at_10": 0.07292,
"precision_at_100": 0.01043,
"precision_at_1000": 0.00113,
"precision_at_20": 0.04228,
"precision_at_3": 0.1734,
"precision_at_5": 0.12209,
"recall_at_1": 0.27514,
"recall_at_10": 0.64846,
"recall_at_100": 0.89351,
"recall_at_1000": 0.96793,
"recall_at_20": 0.7403,
"recall_at_3": 0.47427,
"recall_at_5": 0.54988
}
]
},
"task_name": "MrTidyRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MultiLongDocRetrieval.json
================================================
{
"dataset_revision": "d67138e705d963e346253a80e59676ddb418810a",
"evaluation_time": 6735.936784029007,
"kg_co2_emissions": null,
"mteb_version": "1.20.0",
"scores": {
"dev": [
{
"hf_subset": "ko",
"languages": [
"kor-Hang"
],
"main_score": 0.30313,
"map_at_1": 0.21,
"map_at_10": 0.27097,
"map_at_100": 0.27948,
"map_at_1000": 0.28033,
"map_at_20": 0.27614,
"map_at_3": 0.255,
"map_at_5": 0.2635,
"mrr_at_1": 0.21,
"mrr_at_10": 0.27097023809523807,
"mrr_at_100": 0.27947751541679616,
"mrr_at_1000": 0.2803278246284526,
"mrr_at_20": 0.27614094647182885,
"mrr_at_3": 0.25499999999999995,
"mrr_at_5": 0.26349999999999996,
"nauc_map_at_1000_diff1": 0.35873770357574136,
"nauc_map_at_1000_max": 0.31134425197944254,
"nauc_map_at_1000_std": -0.06704458741305393,
"nauc_map_at_100_diff1": 0.3586784080926965,
"nauc_map_at_100_max": 0.311717869958562,
"nauc_map_at_100_std": -0.06674245676360822,
"nauc_map_at_10_diff1": 0.35860247103967324,
"nauc_map_at_10_max": 0.3032036458545803,
"nauc_map_at_10_std": -0.0813804882991818,
"nauc_map_at_1_diff1": 0.4261168384879724,
"nauc_map_at_1_max": 0.32412466150410946,
"nauc_map_at_1_std": -0.10102617701553508,
"nauc_map_at_20_diff1": 0.35724833739937045,
"nauc_map_at_20_max": 0.31083006589902196,
"nauc_map_at_20_std": -0.06922956842820864,
"nauc_map_at_3_diff1": 0.3684035495695084,
"nauc_map_at_3_max": 0.2922831830708324,
"nauc_map_at_3_std": -0.08888473191164713,
"nauc_map_at_5_diff1": 0.3678964185766755,
"nauc_map_at_5_max": 0.30087480560348767,
"nauc_map_at_5_std": -0.08768089007739847,
"nauc_mrr_at_1000_diff1": 0.35873770357574136,
"nauc_mrr_at_1000_max": 0.31134425197944254,
"nauc_mrr_at_1000_std": -0.06704458741305393,
"nauc_mrr_at_100_diff1": 0.3586784080926965,
"nauc_mrr_at_100_max": 0.311717869958562,
"nauc_mrr_at_100_std": -0.06674245676360822,
"nauc_mrr_at_10_diff1": 0.35860247103967324,
"nauc_mrr_at_10_max": 0.3032036458545803,
"nauc_mrr_at_10_std": -0.0813804882991818,
"nauc_mrr_at_1_diff1": 0.4261168384879724,
"nauc_mrr_at_1_max": 0.32412466150410946,
"nauc_mrr_at_1_std": -0.10102617701553508,
"nauc_mrr_at_20_diff1": 0.35724833739937045,
"nauc_mrr_at_20_max": 0.31083006589902196,
"nauc_mrr_at_20_std": -0.06922956842820864,
"nauc_mrr_at_3_diff1": 0.3684035495695084,
"nauc_mrr_at_3_max": 0.2922831830708324,
"nauc_mrr_at_3_std": -0.08888473191164713,
"nauc_mrr_at_5_diff1": 0.3678964185766755,
"nauc_mrr_at_5_max": 0.30087480560348767,
"nauc_mrr_at_5_std": -0.08768089007739847,
"nauc_ndcg_at_1000_diff1": 0.3352554616023902,
"nauc_ndcg_at_1000_max": 0.32663083442920143,
"nauc_ndcg_at_1000_std": -0.01311576117979229,
"nauc_ndcg_at_100_diff1": 0.3318237851782861,
"nauc_ndcg_at_100_max": 0.33657772487092585,
"nauc_ndcg_at_100_std": -0.005489102690649493,
"nauc_ndcg_at_10_diff1": 0.3314404267618865,
"nauc_ndcg_at_10_max": 0.30633427477190284,
"nauc_ndcg_at_10_std": -0.06522869923869197,
"nauc_ndcg_at_1_diff1": 0.4261168384879724,
"nauc_ndcg_at_1_max": 0.32412466150410946,
"nauc_ndcg_at_1_std": -0.10102617701553508,
"nauc_ndcg_at_20_diff1": 0.3283232800788365,
"nauc_ndcg_at_20_max": 0.33313347262163,
"nauc_ndcg_at_20_std": -0.02484722727801561,
"nauc_ndcg_at_3_diff1": 0.35287080579869623,
"nauc_ndcg_at_3_max": 0.2868857758777876,
"nauc_ndcg_at_3_std": -0.08231335061500795,
"nauc_ndcg_at_5_diff1": 0.35322102269226135,
"nauc_ndcg_at_5_max": 0.3024221792638072,
"nauc_ndcg_at_5_std": -0.07959401367124273,
"nauc_precision_at_1000_diff1": 0.1879961649089168,
"nauc_precision_at_1000_max": 0.3998465963566634,
"nauc_precision_at_1000_std": 0.49311601150527246,
"nauc_precision_at_100_diff1": 0.23821275795609478,
"nauc_precision_at_100_max": 0.44484401197135665,
"nauc_precision_at_100_std": 0.26764756577194854,
"nauc_precision_at_10_diff1": 0.2552144786810836,
"nauc_precision_at_10_max": 0.317691509184575,
"nauc_precision_at_10_std": -0.017563202383024983,
"nauc_precision_at_1_diff1": 0.4261168384879724,
"nauc_precision_at_1_max": 0.32412466150410946,
"nauc_precision_at_1_std": -0.10102617701553508,
"nauc_precision_at_20_diff1": 0.2454380345408315,
"nauc_precision_at_20_max": 0.41575801604860807,
"nauc_precision_at_20_std": 0.12646039031800008,
"nauc_precision_at_3_diff1": 0.31220954795099287,
"nauc_precision_at_3_max": 0.2737508466279064,
"nauc_precision_at_3_std": -0.06407548232643295,
"nauc_precision_at_5_diff1": 0.31606566022809685,
"nauc_precision_at_5_max": 0.309862218999275,
"nauc_precision_at_5_std": -0.05685279187817251,
"nauc_recall_at_1000_diff1": 0.1879961649089169,
"nauc_recall_at_1000_max": 0.3998465963566645,
"nauc_recall_at_1000_std": 0.4931160115052724,
"nauc_recall_at_100_diff1": 0.23821275795609434,
"nauc_recall_at_100_max": 0.4448440119713569,
"nauc_recall_at_100_std": 0.26764756577194837,
"nauc_recall_at_10_diff1": 0.25521447868108316,
"nauc_recall_at_10_max": 0.31769150918457467,
"nauc_recall_at_10_std": -0.017563202383025025,
"nauc_recall_at_1_diff1": 0.4261168384879724,
"nauc_recall_at_1_max": 0.32412466150410946,
"nauc_recall_at_1_std": -0.10102617701553508,
"nauc_recall_at_20_diff1": 0.24543803454083146,
"nauc_recall_at_20_max": 0.4157580160486079,
"nauc_recall_at_20_std": 0.12646039031800027,
"nauc_recall_at_3_diff1": 0.31220954795099265,
"nauc_recall_at_3_max": 0.2737508466279063,
"nauc_recall_at_3_std": -0.06407548232643298,
"nauc_recall_at_5_diff1": 0.31606566022809673,
"nauc_recall_at_5_max": 0.30986221899927474,
"nauc_recall_at_5_std": -0.05685279187817247,
"ndcg_at_1": 0.21,
"ndcg_at_10": 0.30313,
"ndcg_at_100": 0.34874,
"ndcg_at_1000": 0.37803,
"ndcg_at_20": 0.32123,
"ndcg_at_3": 0.27036,
"ndcg_at_5": 0.28521,
"precision_at_1": 0.21,
"precision_at_10": 0.0405,
"precision_at_100": 0.0063,
"precision_at_1000": 0.00088,
"precision_at_20": 0.02375,
"precision_at_3": 0.105,
"precision_at_5": 0.07,
"recall_at_1": 0.21,
"recall_at_10": 0.405,
"recall_at_100": 0.63,
"recall_at_1000": 0.875,
"recall_at_20": 0.475,
"recall_at_3": 0.315,
"recall_at_5": 0.35
}
],
"test": [
{
"hf_subset": "ko",
"languages": [
"kor-Hang"
],
"main_score": 0.29368,
"map_at_1": 0.23,
"map_at_10": 0.27265,
"map_at_100": 0.2815,
"map_at_1000": 0.28244,
"map_at_20": 0.27838,
"map_at_3": 0.25917,
"map_at_5": 0.26867,
"mrr_at_1": 0.23,
"mrr_at_10": 0.27264880952380954,
"mrr_at_100": 0.281497391346387,
"mrr_at_1000": 0.2824432364298307,
"mrr_at_20": 0.2783757250661276,
"mrr_at_3": 0.25916666666666666,
"mrr_at_5": 0.26866666666666666,
"nauc_map_at_1000_diff1": 0.5693273819450104,
"nauc_map_at_1000_max": 0.5806473230147973,
"nauc_map_at_1000_std": -0.060757043584923086,
"nauc_map_at_100_diff1": 0.5693786847659337,
"nauc_map_at_100_max": 0.5807719113443569,
"nauc_map_at_100_std": -0.06089628585980827,
"nauc_map_at_10_diff1": 0.5711855003420948,
"nauc_map_at_10_max": 0.5777140106516642,
"nauc_map_at_10_std": -0.07319092362315573,
"nauc_map_at_1_diff1": 0.6205289730174953,
"nauc_map_at_1_max": 0.5944463934351216,
"nauc_map_at_1_std": -0.08557679108006284,
"nauc_map_at_20_diff1": 0.5724183931836933,
"nauc_map_at_20_max": 0.5807674045138349,
"nauc_map_at_20_std": -0.06504509217669452,
"nauc_map_at_3_diff1": 0.5816747629910118,
"nauc_map_at_3_max": 0.5802199578516652,
"nauc_map_at_3_std": -0.08412296119192691,
"nauc_map_at_5_diff1": 0.5773259359315182,
"nauc_map_at_5_max": 0.5791643172847812,
"nauc_map_at_5_std": -0.07712777414534365,
"nauc_mrr_at_1000_diff1": 0.5693273819450104,
"nauc_mrr_at_1000_max": 0.5806473230147973,
"nauc_mrr_at_1000_std": -0.060757043584923086,
"nauc_mrr_at_100_diff1": 0.5693786847659337,
"nauc_mrr_at_100_max": 0.5807719113443569,
"nauc_mrr_at_100_std": -0.06089628585980827,
"nauc_mrr_at_10_diff1": 0.5711855003420948,
"nauc_mrr_at_10_max": 0.5777140106516642,
"nauc_mrr_at_10_std": -0.07319092362315573,
"nauc_mrr_at_1_diff1": 0.6205289730174953,
"nauc_mrr_at_1_max": 0.5944463934351216,
"nauc_mrr_at_1_std": -0.08557679108006284,
"nauc_mrr_at_20_diff1": 0.5724183931836933,
"nauc_mrr_at_20_max": 0.5807674045138349,
"nauc_mrr_at_20_std": -0.06504509217669452,
"nauc_mrr_at_3_diff1": 0.5816747629910118,
"nauc_mrr_at_3_max": 0.5802199578516652,
"nauc_mrr_at_3_std": -0.08412296119192691,
"nauc_mrr_at_5_diff1": 0.5773259359315182,
"nauc_mrr_at_5_max": 0.5791643172847812,
"nauc_mrr_at_5_std": -0.07712777414534365,
"nauc_ndcg_at_1000_diff1": 0.5362013372269028,
"nauc_ndcg_at_1000_max": 0.5832488209073998,
"nauc_ndcg_at_1000_std": -0.001090633497059146,
"nauc_ndcg_at_100_diff1": 0.5350372618108911,
"nauc_ndcg_at_100_max": 0.584824061919428,
"nauc_ndcg_at_100_std": 0.00032361783827936655,
"nauc_ndcg_at_10_diff1": 0.5479495957959611,
"nauc_ndcg_at_10_max": 0.5725374735471958,
"nauc_ndcg_at_10_std": -0.061335475446542505,
"nauc_ndcg_at_1_diff1": 0.6205289730174953,
"nauc_ndcg_at_1_max": 0.5944463934351216,
"nauc_ndcg_at_1_std": -0.08557679108006284,
"nauc_ndcg_at_20_diff1": 0.5548679294963687,
"nauc_ndcg_at_20_max": 0.5874811607238918,
"nauc_ndcg_at_20_std": -0.02590888789912424,
"nauc_ndcg_at_3_diff1": 0.570878278999106,
"nauc_ndcg_at_3_max": 0.5777343931413231,
"nauc_ndcg_at_3_std": -0.08385814742901972,
"nauc_ndcg_at_5_diff1": 0.5630976166667157,
"nauc_ndcg_at_5_max": 0.5759898239262172,
"nauc_ndcg_at_5_std": -0.07118431240108142,
"nauc_precision_at_1000_diff1": 0.2822715643852065,
"nauc_precision_at_1000_max": 0.6049556236379151,
"nauc_precision_at_1000_std": 0.5230251729256822,
"nauc_precision_at_100_diff1": 0.39846672618740214,
"nauc_precision_at_100_max": 0.604278114324083,
"nauc_precision_at_100_std": 0.2741446353863069,
"nauc_precision_at_10_diff1": 0.4793667663884957,
"nauc_precision_at_10_max": 0.5578845851047082,
"nauc_precision_at_10_std": -0.02404476985251651,
"nauc_precision_at_1_diff1": 0.6205289730174953,
"nauc_precision_at_1_max": 0.5944463934351216,
"nauc_precision_at_1_std": -0.08557679108006284,
"nauc_precision_at_20_diff1": 0.509670660642965,
"nauc_precision_at_20_max": 0.6209520068094021,
"nauc_precision_at_20_std": 0.1194722713284881,
"nauc_precision_at_3_diff1": 0.5415367498592096,
"nauc_precision_at_3_max": 0.5714530994601875,
"nauc_precision_at_3_std": -0.08318338530005635,
"nauc_precision_at_5_diff1": 0.524031100637723,
"nauc_precision_at_5_max": 0.5678262837666922,
"nauc_precision_at_5_std": -0.053414280198639245,
"nauc_recall_at_1000_diff1": 0.2822715643852057,
"nauc_recall_at_1000_max": 0.604955623637915,
"nauc_recall_at_1000_std": 0.523025172925681,
"nauc_recall_at_100_diff1": 0.3984667261874016,
"nauc_recall_at_100_max": 0.6042781143240835,
"nauc_recall_at_100_std": 0.2741446353863071,
"nauc_recall_at_10_diff1": 0.47936676638849496,
"nauc_recall_at_10_max": 0.557884585104708,
"nauc_recall_at_10_std": -0.024044769852516724,
"nauc_recall_at_1_diff1": 0.6205289730174953,
"nauc_recall_at_1_max": 0.5944463934351216,
"nauc_recall_at_1_std": -0.08557679108006284,
"nauc_recall_at_20_diff1": 0.5096706606429648,
"nauc_recall_at_20_max": 0.620952006809402,
"nauc_recall_at_20_std": 0.11947227132848824,
"nauc_recall_at_3_diff1": 0.5415367498592092,
"nauc_recall_at_3_max": 0.5714530994601871,
"nauc_recall_at_3_std": -0.08318338530005628,
"nauc_recall_at_5_diff1": 0.5240311006377228,
"nauc_recall_at_5_max": 0.567826283766692,
"nauc_recall_at_5_std": -0.053414280198639634,
"ndcg_at_1": 0.23,
"ndcg_at_10": 0.29368,
"ndcg_at_100": 0.34147,
"ndcg_at_1000": 0.37178,
"ndcg_at_20": 0.31579,
"ndcg_at_3": 0.2672,
"ndcg_at_5": 0.28399,
"precision_at_1": 0.23,
"precision_at_10": 0.036,
"precision_at_100": 0.00595,
"precision_at_1000": 0.00085,
"precision_at_20": 0.0225,
"precision_at_3": 0.09667,
"precision_at_5": 0.066,
"recall_at_1": 0.23,
"recall_at_10": 0.36,
"recall_at_100": 0.595,
"recall_at_1000": 0.845,
"recall_at_20": 0.45,
"recall_at_3": 0.29,
"recall_at_5": 0.33
}
]
},
"task_name": "MultiLongDocRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/PublicHealthQA.json
================================================
{
"dataset_revision": "main",
"evaluation_time": 46.12740516662598,
"kg_co2_emissions": null,
"mteb_version": "1.19.4",
"scores": {
"test": [
{
"hf_subset": "korean",
"languages": [
"kor-Hang"
],
"main_score": 0.85844,
"map_at_1": 0.68831,
"map_at_10": 0.81196,
"map_at_100": 0.81196,
"map_at_1000": 0.81196,
"map_at_20": 0.81196,
"map_at_3": 0.80303,
"map_at_5": 0.80303,
"mrr_at_1": 0.6883116883116883,
"mrr_at_10": 0.8119562976705834,
"mrr_at_100": 0.8119562976705834,
"mrr_at_1000": 0.8119562976705834,
"mrr_at_20": 0.8119562976705834,
"mrr_at_3": 0.8030303030303031,
"mrr_at_5": 0.8030303030303031,
"nauc_map_at_1000_diff1": 0.7306499143837317,
"nauc_map_at_1000_max": 0.3237895227119708,
"nauc_map_at_1000_std": -0.16620176663433442,
"nauc_map_at_100_diff1": 0.7306499143837317,
"nauc_map_at_100_max": 0.3237895227119708,
"nauc_map_at_100_std": -0.16620176663433442,
"nauc_map_at_10_diff1": 0.7306499143837317,
"nauc_map_at_10_max": 0.3237895227119708,
"nauc_map_at_10_std": -0.16620176663433442,
"nauc_map_at_1_diff1": 0.7422916035829402,
"nauc_map_at_1_max": 0.3482164359211589,
"nauc_map_at_1_std": -0.09165116325682032,
"nauc_map_at_20_diff1": 0.7306499143837317,
"nauc_map_at_20_max": 0.3237895227119708,
"nauc_map_at_20_std": -0.16620176663433442,
"nauc_map_at_3_diff1": 0.727274397793811,
"nauc_map_at_3_max": 0.3229930586432481,
"nauc_map_at_3_std": -0.18695188162216977,
"nauc_map_at_5_diff1": 0.727274397793811,
"nauc_map_at_5_max": 0.3229930586432481,
"nauc_map_at_5_std": -0.18695188162216977,
"nauc_mrr_at_1000_diff1": 0.7306499143837317,
"nauc_mrr_at_1000_max": 0.3237895227119708,
"nauc_mrr_at_1000_std": -0.16620176663433442,
"nauc_mrr_at_100_diff1": 0.7306499143837317,
"nauc_mrr_at_100_max": 0.3237895227119708,
"nauc_mrr_at_100_std": -0.16620176663433442,
"nauc_mrr_at_10_diff1": 0.7306499143837317,
"nauc_mrr_at_10_max": 0.3237895227119708,
"nauc_mrr_at_10_std": -0.16620176663433442,
"nauc_mrr_at_1_diff1": 0.7422916035829402,
"nauc_mrr_at_1_max": 0.3482164359211589,
"nauc_mrr_at_1_std": -0.09165116325682032,
"nauc_mrr_at_20_diff1": 0.7306499143837317,
"nauc_mrr_at_20_max": 0.3237895227119708,
"nauc_mrr_at_20_std": -0.16620176663433442,
"nauc_mrr_at_3_diff1": 0.727274397793811,
"nauc_mrr_at_3_max": 0.3229930586432481,
"nauc_mrr_at_3_std": -0.18695188162216977,
"nauc_mrr_at_5_diff1": 0.727274397793811,
"nauc_mrr_at_5_max": 0.3229930586432481,
"nauc_mrr_at_5_std": -0.18695188162216977,
"nauc_ndcg_at_1000_diff1": 0.7298622798036085,
"nauc_ndcg_at_1000_max": 0.32081770832924095,
"nauc_ndcg_at_1000_std": -0.17269361184898482,
"nauc_ndcg_at_100_diff1": 0.7298622798036085,
"nauc_ndcg_at_100_max": 0.32081770832924095,
"nauc_ndcg_at_100_std": -0.17269361184898482,
"nauc_ndcg_at_10_diff1": 0.7298622798036085,
"nauc_ndcg_at_10_max": 0.32081770832924095,
"nauc_ndcg_at_10_std": -0.17269361184898482,
"nauc_ndcg_at_1_diff1": 0.7422916035829402,
"nauc_ndcg_at_1_max": 0.3482164359211589,
"nauc_ndcg_at_1_std": -0.09165116325682032,
"nauc_ndcg_at_20_diff1": 0.7298622798036085,
"nauc_ndcg_at_20_max": 0.32081770832924095,
"nauc_ndcg_at_20_std": -0.17269361184898482,
"nauc_ndcg_at_3_diff1": 0.7219783130912044,
"nauc_ndcg_at_3_max": 0.31347352334832335,
"nauc_ndcg_at_3_std": -0.22725218013410522,
"nauc_ndcg_at_5_diff1": 0.7219783130912044,
"nauc_ndcg_at_5_max": 0.31347352334832335,
"nauc_ndcg_at_5_std": -0.22725218013410522,
"nauc_precision_at_1000_diff1": NaN,
"nauc_precision_at_1000_max": NaN,
"nauc_precision_at_1000_std": NaN,
"nauc_precision_at_100_diff1": 1.0,
"nauc_precision_at_100_max": 1.0,
"nauc_precision_at_100_std": 1.0,
"nauc_precision_at_10_diff1": 1.0,
"nauc_precision_at_10_max": 1.0,
"nauc_precision_at_10_std": 1.0,
"nauc_precision_at_1_diff1": 0.7422916035829402,
"nauc_precision_at_1_max": 0.3482164359211589,
"nauc_precision_at_1_std": -0.09165116325682032,
"nauc_precision_at_20_diff1": 1.0,
"nauc_precision_at_20_max": 1.0,
"nauc_precision_at_20_std": 1.0,
"nauc_precision_at_3_diff1": 0.6835820481559275,
"nauc_precision_at_3_max": 0.24373783838225974,
"nauc_precision_at_3_std": -0.5271426588640465,
"nauc_precision_at_5_diff1": 0.6835820481559289,
"nauc_precision_at_5_max": 0.24373783838226237,
"nauc_precision_at_5_std": -0.5271426588640427,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": NaN,
"nauc_recall_at_100_max": NaN,
"nauc_recall_at_100_std": NaN,
"nauc_recall_at_10_diff1": NaN,
"nauc_recall_at_10_max": NaN,
"nauc_recall_at_10_std": NaN,
"nauc_recall_at_1_diff1": 0.7422916035829402,
"nauc_recall_at_1_max": 0.3482164359211589,
"nauc_recall_at_1_std": -0.09165116325682032,
"nauc_recall_at_20_diff1": NaN,
"nauc_recall_at_20_max": NaN,
"nauc_recall_at_20_std": NaN,
"nauc_recall_at_3_diff1": 0.6835820481559286,
"nauc_recall_at_3_max": 0.24373783838225746,
"nauc_recall_at_3_std": -0.5271426588640481,
"nauc_recall_at_5_diff1": 0.6835820481559286,
"nauc_recall_at_5_max": 0.24373783838225746,
"nauc_recall_at_5_std": -0.5271426588640481,
"ndcg_at_1": 0.68831,
"ndcg_at_10": 0.85844,
"ndcg_at_100": 0.85844,
"ndcg_at_1000": 0.85844,
"ndcg_at_20": 0.85844,
"ndcg_at_3": 0.83719,
"ndcg_at_5": 0.83719,
"precision_at_1": 0.68831,
"precision_at_10": 0.1,
"precision_at_100": 0.01,
"precision_at_1000": 0.001,
"precision_at_20": 0.05,
"precision_at_3": 0.31169,
"precision_at_5": 0.18701,
"recall_at_1": 0.68831,
"recall_at_10": 1.0,
"recall_at_100": 1.0,
"recall_at_1000": 1.0,
"recall_at_20": 1.0,
"recall_at_3": 0.93506,
"recall_at_5": 0.93506
}
]
},
"task_name": "PublicHealthQA"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/XPQARetrieval.json
================================================
{
"dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f",
"evaluation_time": 373.6595883369446,
"kg_co2_emissions": null,
"mteb_version": "1.20.0",
"scores": {
"test": [
{
"hf_subset": "kor-kor",
"languages": [
"kor-Hang",
"kor-Hang"
],
"main_score": 0.37866,
"map_at_1": 0.21493,
"map_at_10": 0.3305,
"map_at_100": 0.34654,
"map_at_1000": 0.34759,
"map_at_20": 0.33866,
"map_at_3": 0.2948,
"map_at_5": 0.31486,
"mrr_at_1": 0.2996941896024465,
"mrr_at_10": 0.36733168292801305,
"mrr_at_100": 0.3793843923095916,
"mrr_at_1000": 0.380081832466756,
"mrr_at_20": 0.37381202387628654,
"mrr_at_3": 0.34378185524974525,
"mrr_at_5": 0.35616717635066264,
"nauc_map_at_1000_diff1": 0.39605398696420363,
"nauc_map_at_1000_max": 0.0412782945923716,
"nauc_map_at_1000_std": -0.3673566334388843,
"nauc_map_at_100_diff1": 0.395789867336939,
"nauc_map_at_100_max": 0.04155953451877115,
"nauc_map_at_100_std": -0.36651315519418337,
"nauc_map_at_10_diff1": 0.3977984105897261,
"nauc_map_at_10_max": 0.03743921340462259,
"nauc_map_at_10_std": -0.37518750626502256,
"nauc_map_at_1_diff1": 0.446152501423059,
"nauc_map_at_1_max": 0.02663440208773937,
"nauc_map_at_1_std": -0.33965214961688095,
"nauc_map_at_20_diff1": 0.39625562343728227,
"nauc_map_at_20_max": 0.038924158809186135,
"nauc_map_at_20_std": -0.37210104220524026,
"nauc_map_at_3_diff1": 0.39380176890037777,
"nauc_map_at_3_max": 0.03416038244220423,
"nauc_map_at_3_std": -0.3681811178017894,
"nauc_map_at_5_diff1": 0.39571643464140555,
"nauc_map_at_5_max": 0.03721875189175598,
"nauc_map_at_5_std": -0.37458573833597125,
"nauc_mrr_at_1000_diff1": 0.4204049613713672,
"nauc_mrr_at_1000_max": 0.04473616768863933,
"nauc_mrr_at_1000_std": -0.3683979005894126,
"nauc_mrr_at_100_diff1": 0.42024967745486136,
"nauc_mrr_at_100_max": 0.04499165061867992,
"nauc_mrr_at_100_std": -0.3677313105342792,
"nauc_mrr_at_10_diff1": 0.4213875590775421,
"nauc_mrr_at_10_max": 0.03916398461276921,
"nauc_mrr_at_10_std": -0.3760084893557857,
"nauc_mrr_at_1_diff1": 0.43496360123353245,
"nauc_mrr_at_1_max": 0.05114237339724553,
"nauc_mrr_at_1_std": -0.3737822614588633,
"nauc_mrr_at_20_diff1": 0.42058757596953117,
"nauc_mrr_at_20_max": 0.042627432553596564,
"nauc_mrr_at_20_std": -0.37140474184158695,
"nauc_mrr_at_3_diff1": 0.42186574684849265,
"nauc_mrr_at_3_max": 0.03772375422449969,
"nauc_mrr_at_3_std": -0.3779124477466279,
"nauc_mrr_at_5_diff1": 0.4199331499830262,
"nauc_mrr_at_5_max": 0.03760842836763998,
"nauc_mrr_at_5_std": -0.37814230032915724,
"nauc_ndcg_at_1000_diff1": 0.3978473800651747,
"nauc_ndcg_at_1000_max": 0.053960682039080894,
"nauc_ndcg_at_1000_std": -0.3476679618966263,
"nauc_ndcg_at_100_diff1": 0.3936901716079827,
"nauc_ndcg_at_100_max": 0.06975428508696647,
"nauc_ndcg_at_100_std": -0.3176601369260679,
"nauc_ndcg_at_10_diff1": 0.4015378876968567,
"nauc_ndcg_at_10_max": 0.03791666857344486,
"nauc_ndcg_at_10_std": -0.37436111440335995,
"nauc_ndcg_at_1_diff1": 0.4302105091963213,
"nauc_ndcg_at_1_max": 0.04725753243883758,
"nauc_ndcg_at_1_std": -0.3773456769429174,
"nauc_ndcg_at_20_diff1": 0.3962429416823341,
"nauc_ndcg_at_20_max": 0.04389393336467113,
"nauc_ndcg_at_20_std": -0.3630483112178299,
"nauc_ndcg_at_3_diff1": 0.3950615196047281,
"nauc_ndcg_at_3_max": 0.030305615511054654,
"nauc_ndcg_at_3_std": -0.37394136253274857,
"nauc_ndcg_at_5_diff1": 0.3963854674719409,
"nauc_ndcg_at_5_max": 0.034625800368698355,
"nauc_ndcg_at_5_std": -0.375950275291491,
"nauc_precision_at_1000_diff1": -0.034626884552404966,
"nauc_precision_at_1000_max": 0.07690224605888045,
"nauc_precision_at_1000_std": 0.021871172442492676,
"nauc_precision_at_100_diff1": 0.0714801115254476,
"nauc_precision_at_100_max": 0.13315561541052745,
"nauc_precision_at_100_std": 0.03444855372602813,
"nauc_precision_at_10_diff1": 0.23573365255125925,
"nauc_precision_at_10_max": 0.035876362649886924,
"nauc_precision_at_10_std": -0.29724374204707604,
"nauc_precision_at_1_diff1": 0.4302105091963213,
"nauc_precision_at_1_max": 0.04725753243883758,
"nauc_precision_at_1_std": -0.3773456769429174,
"nauc_precision_at_20_diff1": 0.1874260473657467,
"nauc_precision_at_20_max": 0.04232031468123627,
"nauc_precision_at_20_std": -0.23356189942493982,
"nauc_precision_at_3_diff1": 0.26903273918480025,
"nauc_precision_at_3_max": 0.04616211001781429,
"nauc_precision_at_3_std": -0.3359670496699033,
"nauc_precision_at_5_diff1": 0.2533080980168239,
"nauc_precision_at_5_max": 0.0447726069996935,
"nauc_precision_at_5_std": -0.32679138663754764,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 0.310564964535451,
"nauc_recall_at_100_max": 0.30173839624071874,
"nauc_recall_at_100_std": 0.11377334515723407,
"nauc_recall_at_10_diff1": 0.37982695676707556,
"nauc_recall_at_10_max": 0.042492451912603056,
"nauc_recall_at_10_std": -0.35231447642883357,
"nauc_recall_at_1_diff1": 0.446152501423059,
"nauc_recall_at_1_max": 0.02663440208773937,
"nauc_recall_at_1_std": -0.33965214961688095,
"nauc_recall_at_20_diff1": 0.35638430589180825,
"nauc_recall_at_20_max": 0.06328227941791602,
"nauc_recall_at_20_std": -0.3113052824785763,
"nauc_recall_at_3_diff1": 0.3712633595908005,
"nauc_recall_at_3_max": 0.02494288594337995,
"nauc_recall_at_3_std": -0.35548445928432737,
"nauc_recall_at_5_diff1": 0.36571414475870745,
"nauc_recall_at_5_max": 0.029103482587579426,
"nauc_recall_at_5_std": -0.3588705476558524,
"ndcg_at_1": 0.30122,
"ndcg_at_10": 0.37866,
"ndcg_at_100": 0.45423,
"ndcg_at_1000": 0.4792,
"ndcg_at_20": 0.40376,
"ndcg_at_3": 0.3221,
"ndcg_at_5": 0.34735,
"precision_at_1": 0.30122,
"precision_at_10": 0.07691,
"precision_at_100": 0.01303,
"precision_at_1000": 0.00156,
"precision_at_20": 0.0458,
"precision_at_3": 0.17431,
"precision_at_5": 0.12508,
"recall_at_1": 0.21493,
"recall_at_10": 0.4843,
"recall_at_100": 0.8169,
"recall_at_1000": 1.0,
"recall_at_20": 0.56944,
"recall_at_3": 0.33876,
"recall_at_5": 0.40008
},
{
"hf_subset": "eng-kor",
"languages": [
"eng-Latn",
"kor-Hang"
],
"main_score": 0.39995,
"map_at_1": 0.19177,
"map_at_10": 0.33984,
"map_at_100": 0.35699,
"map_at_1000": 0.35838,
"map_at_20": 0.34825,
"map_at_3": 0.28717,
"map_at_5": 0.3174,
"mrr_at_1": 0.3318042813455658,
"mrr_at_10": 0.40260545604582293,
"mrr_at_100": 0.41334522944107477,
"mrr_at_1000": 0.4138877971938659,
"mrr_at_20": 0.4083959893443803,
"mrr_at_3": 0.374362895005097,
"mrr_at_5": 0.38934760448521916,
"nauc_map_at_1000_diff1": 0.41058617945545617,
"nauc_map_at_1000_max": -0.0013002952537210077,
"nauc_map_at_1000_std": -0.402768625964047,
"nauc_map_at_100_diff1": 0.4103859458849651,
"nauc_map_at_100_max": -0.0024929692398992275,
"nauc_map_at_100_std": -0.40403076519830755,
"nauc_map_at_10_diff1": 0.4066546282104827,
"nauc_map_at_10_max": -0.002486845640082756,
"nauc_map_at_10_std": -0.4053493086971202,
"nauc_map_at_1_diff1": 0.46831829280036646,
"nauc_map_at_1_max": -0.014474441932302181,
"nauc_map_at_1_std": -0.34551612668189086,
"nauc_map_at_20_diff1": 0.4101210254500855,
"nauc_map_at_20_max": -0.0047810042551561975,
"nauc_map_at_20_std": -0.4079539137908949,
"nauc_map_at_3_diff1": 0.4391849418232014,
"nauc_map_at_3_max": 0.012395764653712059,
"nauc_map_at_3_std": -0.38991991301597145,
"nauc_map_at_5_diff1": 0.41917501735931717,
"nauc_map_at_5_max": 0.008385330503281803,
"nauc_map_at_5_std": -0.3995164775690684,
"nauc_mrr_at_1000_diff1": 0.40920660076706167,
"nauc_mrr_at_1000_max": 0.0012385941769198715,
"nauc_mrr_at_1000_std": -0.3932741446560242,
"nauc_mrr_at_100_diff1": 0.40904930361090497,
"nauc_mrr_at_100_max": 0.0010593221401256512,
"nauc_mrr_at_100_std": -0.39335860583329924,
"nauc_mrr_at_10_diff1": 0.40766999280728256,
"nauc_mrr_at_10_max": -8.088705493833043e-05,
"nauc_mrr_at_10_std": -0.39530918705522655,
"nauc_mrr_at_1_diff1": 0.4427688268960664,
"nauc_mrr_at_1_max": 0.01633880940031036,
"nauc_mrr_at_1_std": -0.38611719450212045,
"nauc_mrr_at_20_diff1": 0.4090902574437497,
"nauc_mrr_at_20_max": -0.0009201069513837534,
"nauc_mrr_at_20_std": -0.39607998114446913,
"nauc_mrr_at_3_diff1": 0.41695142105898064,
"nauc_mrr_at_3_max": 0.004057669729456041,
"nauc_mrr_at_3_std": -0.39551144383536135,
"nauc_mrr_at_5_diff1": 0.4070267126250566,
"nauc_mrr_at_5_max": 0.005278939293701363,
"nauc_mrr_at_5_std": -0.3919125656637338,
"nauc_ndcg_at_1000_diff1": 0.40215131033707746,
"nauc_ndcg_at_1000_max": 0.001258704325013927,
"nauc_ndcg_at_1000_std": -0.3908957262121216,
"nauc_ndcg_at_100_diff1": 0.39832291711852713,
"nauc_ndcg_at_100_max": -0.004799637066320755,
"nauc_ndcg_at_100_std": -0.393164829192417,
"nauc_ndcg_at_10_diff1": 0.38899882407459435,
"nauc_ndcg_at_10_max": -0.018537873597670097,
"nauc_ndcg_at_10_std": -0.4141314712129482,
"nauc_ndcg_at_1_diff1": 0.4427688268960664,
"nauc_ndcg_at_1_max": 0.01633880940031036,
"nauc_ndcg_at_1_std": -0.38611719450212045,
"nauc_ndcg_at_20_diff1": 0.39736311864081186,
"nauc_ndcg_at_20_max": -0.026621311940814282,
"nauc_ndcg_at_20_std": -0.4212133721923276,
"nauc_ndcg_at_3_diff1": 0.4190616958577465,
"nauc_ndcg_at_3_max": 0.009013169509649496,
"nauc_ndcg_at_3_std": -0.3993861491322945,
"nauc_ndcg_at_5_diff1": 0.4032274550560342,
"nauc_ndcg_at_5_max": 0.0024637382960664166,
"nauc_ndcg_at_5_std": -0.40340444594494707,
"nauc_precision_at_1000_diff1": 0.004259057835218737,
"nauc_precision_at_1000_max": 0.08826356081485175,
"nauc_precision_at_1000_std": 0.01338165737445073,
"nauc_precision_at_100_diff1": 0.06902501449184535,
"nauc_precision_at_100_max": 0.05582969368179055,
"nauc_precision_at_100_std": -0.08454217226602033,
"nauc_precision_at_10_diff1": 0.17246682812480735,
"nauc_precision_at_10_max": -0.012406289944737326,
"nauc_precision_at_10_std": -0.3006788219394346,
"nauc_precision_at_1_diff1": 0.4427688268960664,
"nauc_precision_at_1_max": 0.01633880940031036,
"nauc_precision_at_1_std": -0.38611719450212045,
"nauc_precision_at_20_diff1": 0.163521463625998,
"nauc_precision_at_20_max": -0.01981955957289997,
"nauc_precision_at_20_std": -0.27855391812427455,
"nauc_precision_at_3_diff1": 0.3015896868111652,
"nauc_precision_at_3_max": 0.038691751764205154,
"nauc_precision_at_3_std": -0.347123228838273,
"nauc_precision_at_5_diff1": 0.23111191170926612,
"nauc_precision_at_5_max": 0.02370344878705259,
"nauc_precision_at_5_std": -0.32399634526515875,
"nauc_recall_at_1000_diff1": 0.5647183521144965,
"nauc_recall_at_1000_max": 0.41208649490966276,
"nauc_recall_at_1000_std": -0.15695523713683882,
"nauc_recall_at_100_diff1": 0.3156808945250338,
"nauc_recall_at_100_max": 0.01423885208773468,
"nauc_recall_at_100_std": -0.2687000214523801,
"nauc_recall_at_10_diff1": 0.3193138140628656,
"nauc_recall_at_10_max": -0.06363018392286901,
"nauc_recall_at_10_std": -0.4226385831868655,
"nauc_recall_at_1_diff1": 0.46831829280036646,
"nauc_recall_at_1_max": -0.014474441932302181,
"nauc_recall_at_1_std": -0.34551612668189086,
"nauc_recall_at_20_diff1": 0.3433179839518331,
"nauc_recall_at_20_max": -0.1038140150525632,
"nauc_recall_at_20_std": -0.45289277852456233,
"nauc_recall_at_3_diff1": 0.39994036352425977,
"nauc_recall_at_3_max": -0.005326393503024048,
"nauc_recall_at_3_std": -0.3869023976730874,
"nauc_recall_at_5_diff1": 0.3578112098305799,
"nauc_recall_at_5_max": -0.006546757260117465,
"nauc_recall_at_5_std": -0.39532613786244336,
"ndcg_at_1": 0.3318,
"ndcg_at_10": 0.39995,
"ndcg_at_100": 0.47688,
"ndcg_at_1000": 0.50029,
"ndcg_at_20": 0.42508,
"ndcg_at_3": 0.33655,
"ndcg_at_5": 0.35826,
"precision_at_1": 0.3318,
"precision_at_10": 0.10489,
"precision_at_100": 0.01665,
"precision_at_1000": 0.00195,
"precision_at_20": 0.06063,
"precision_at_3": 0.21764,
"precision_at_5": 0.16483,
"recall_at_1": 0.19177,
"recall_at_10": 0.51012,
"recall_at_100": 0.83907,
"recall_at_1000": 0.99503,
"recall_at_20": 0.59455,
"recall_at_3": 0.32989,
"recall_at_5": 0.40344
},
{
"hf_subset": "kor-eng",
"languages": [
"kor-Hang",
"eng-Latn"
],
"main_score": 0.36948,
"map_at_1": 0.20615,
"map_at_10": 0.3216,
"map_at_100": 0.33743,
"map_at_1000": 0.33861,
"map_at_20": 0.32978,
"map_at_3": 0.2846,
"map_at_5": 0.3068,
"mrr_at_1": 0.2980456026058632,
"mrr_at_10": 0.3613728607621115,
"mrr_at_100": 0.37275227119258564,
"mrr_at_1000": 0.3734207310023052,
"mrr_at_20": 0.3672744150330474,
"mrr_at_3": 0.3390336590662324,
"mrr_at_5": 0.351900108577633,
"nauc_map_at_1000_diff1": 0.4373721133535326,
"nauc_map_at_1000_max": 0.09733685887316677,
"nauc_map_at_1000_std": -0.309625544370664,
"nauc_map_at_100_diff1": 0.4370183083119422,
"nauc_map_at_100_max": 0.0972950650230623,
"nauc_map_at_100_std": -0.3090029910345518,
"nauc_map_at_10_diff1": 0.4391928166766258,
"nauc_map_at_10_max": 0.09307475831631143,
"nauc_map_at_10_std": -0.3162571191444478,
"nauc_map_at_1_diff1": 0.5602577371730281,
"nauc_map_at_1_max": 0.10724644675399574,
"nauc_map_at_1_std": -0.277598300172147,
"nauc_map_at_20_diff1": 0.43948424825376897,
"nauc_map_at_20_max": 0.0930331336171994,
"nauc_map_at_20_std": -0.31738766465049795,
"nauc_map_at_3_diff1": 0.46090849293075453,
"nauc_map_at_3_max": 0.10879690061773899,
"nauc_map_at_3_std": -0.29945481511490535,
"nauc_map_at_5_diff1": 0.43825072507014,
"nauc_map_at_5_max": 0.09575663793164427,
"nauc_map_at_5_std": -0.3117128053942495,
"nauc_mrr_at_1000_diff1": 0.46421231294131865,
"nauc_mrr_at_1000_max": 0.10977960061397075,
"nauc_mrr_at_1000_std": -0.3030153340987944,
"nauc_mrr_at_100_diff1": 0.46392796505574907,
"nauc_mrr_at_100_max": 0.11010970849782215,
"nauc_mrr_at_100_std": -0.30223842219228625,
"nauc_mrr_at_10_diff1": 0.46498471958808024,
"nauc_mrr_at_10_max": 0.106646067543003,
"nauc_mrr_at_10_std": -0.30795365867667474,
"nauc_mrr_at_1_diff1": 0.500778242420633,
"nauc_mrr_at_1_max": 0.13070668371304484,
"nauc_mrr_at_1_std": -0.30514994152834823,
"nauc_mrr_at_20_diff1": 0.465026746737862,
"nauc_mrr_at_20_max": 0.10683571107326713,
"nauc_mrr_at_20_std": -0.30811545466865076,
"nauc_mrr_at_3_diff1": 0.4718357358171834,
"nauc_mrr_at_3_max": 0.11748827081331539,
"nauc_mrr_at_3_std": -0.30066474511851476,
"nauc_mrr_at_5_diff1": 0.46458638326622315,
"nauc_mrr_at_5_max": 0.1046713557604587,
"nauc_mrr_at_5_std": -0.308084580457927,
"nauc_ndcg_at_1000_diff1": 0.4283433731130498,
"nauc_ndcg_at_1000_max": 0.10949293288269658,
"nauc_ndcg_at_1000_std": -0.28754953984153664,
"nauc_ndcg_at_100_diff1": 0.41417211580021823,
"nauc_ndcg_at_100_max": 0.11602320148783662,
"nauc_ndcg_at_100_std": -0.2606633885264539,
"nauc_ndcg_at_10_diff1": 0.43018721793998604,
"nauc_ndcg_at_10_max": 0.08274497479111372,
"nauc_ndcg_at_10_std": -0.32042032605870274,
"nauc_ndcg_at_1_diff1": 0.49473135350997494,
"nauc_ndcg_at_1_max": 0.13578656720000074,
"nauc_ndcg_at_1_std": -0.29611939726083886,
"nauc_ndcg_at_20_diff1": 0.43105443007993605,
"nauc_ndcg_at_20_max": 0.0827979056933635,
"nauc_ndcg_at_20_std": -0.3232816484442792,
"nauc_ndcg_at_3_diff1": 0.45036219779939474,
"nauc_ndcg_at_3_max": 0.10478070893705232,
"nauc_ndcg_at_3_std": -0.30016964568421,
"nauc_ndcg_at_5_diff1": 0.4281601682293345,
"nauc_ndcg_at_5_max": 0.08520122895762401,
"nauc_ndcg_at_5_std": -0.31392015201203394,
"nauc_precision_at_1000_diff1": -0.10680404046939247,
"nauc_precision_at_1000_max": 0.1781045855411332,
"nauc_precision_at_1000_std": 0.10293066129639211,
"nauc_precision_at_100_diff1": -0.02641496096230086,
"nauc_precision_at_100_max": 0.18623465151680246,
"nauc_precision_at_100_std": 0.08975371905085901,
"nauc_precision_at_10_diff1": 0.16369753143889806,
"nauc_precision_at_10_max": 0.047241852955354345,
"nauc_precision_at_10_std": -0.25690527620071385,
"nauc_precision_at_1_diff1": 0.49473135350997494,
"nauc_precision_at_1_max": 0.13578656720000074,
"nauc_precision_at_1_std": -0.29611939726083886,
"nauc_precision_at_20_diff1": 0.14351541443712884,
"nauc_precision_at_20_max": 0.06193074008380701,
"nauc_precision_at_20_std": -0.21514301134983735,
"nauc_precision_at_3_diff1": 0.25417523312181733,
"nauc_precision_at_3_max": 0.11194211250906363,
"nauc_precision_at_3_std": -0.2606506300756569,
"nauc_precision_at_5_diff1": 0.1848561701587436,
"nauc_precision_at_5_max": 0.07073984164943047,
"nauc_precision_at_5_std": -0.2682731675801018,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 0.20706354222903484,
"nauc_recall_at_100_max": 0.21365885661701703,
"nauc_recall_at_100_std": 0.09035637562095138,
"nauc_recall_at_10_diff1": 0.3677163354241613,
"nauc_recall_at_10_max": 0.03575033957027068,
"nauc_recall_at_10_std": -0.3280144585910673,
"nauc_recall_at_1_diff1": 0.5602577371730281,
"nauc_recall_at_1_max": 0.10724644675399574,
"nauc_recall_at_1_std": -0.277598300172147,
"nauc_recall_at_20_diff1": 0.3642988806351914,
"nauc_recall_at_20_max": 0.026840971376837584,
"nauc_recall_at_20_std": -0.34325729113061004,
"nauc_recall_at_3_diff1": 0.4275675122137623,
"nauc_recall_at_3_max": 0.09106384319877968,
"nauc_recall_at_3_std": -0.28650617926616817,
"nauc_recall_at_5_diff1": 0.3698909986822292,
"nauc_recall_at_5_max": 0.03906080211614774,
"nauc_recall_at_5_std": -0.3178511477594843,
"ndcg_at_1": 0.29967,
"ndcg_at_10": 0.36948,
"ndcg_at_100": 0.44298,
"ndcg_at_1000": 0.47161,
"ndcg_at_20": 0.39425,
"ndcg_at_3": 0.31568,
"ndcg_at_5": 0.34151,
"precision_at_1": 0.29967,
"precision_at_10": 0.07948,
"precision_at_100": 0.01355,
"precision_at_1000": 0.00167,
"precision_at_20": 0.04707,
"precision_at_3": 0.18024,
"precision_at_5": 0.13192,
"recall_at_1": 0.20615,
"recall_at_10": 0.46939,
"recall_at_100": 0.78699,
"recall_at_1000": 1.0,
"recall_at_20": 0.55504,
"recall_at_3": 0.32724,
"recall_at_5": 0.39265
}
]
},
"task_name": "XPQARetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/model_meta.json
================================================
{"name": "Alibaba-NLP/gte-Qwen2-7B-instruct", "revision": "e26182b2122f4435e8b3ebecbf363990f409b45b", "release_date": "2024-06-15", "languages": null, "n_parameters": 7613000000, "memory_usage": null, "max_tokens": null, "embed_dim": 3584, "license": "apache-2.0", "open_weights": true, "public_training_data": null, "public_training_code": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct", "similarity_fn_name": "cosine", "use_instructions": true, "zero_shot_benchmarks": null, "loader": "instruct_wrapper"}
================================================
FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/AutoRAGRetrieval.json
================================================
{
"dataset_revision": "fd7df84ac089bbec763b1c6bb1b56e985df5cc5c",
"evaluation_time": 9.638917446136475,
"kg_co2_emissions": null,
"mteb_version": "1.19.4",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"kor-Hang"
],
"main_score": 0.77108,
"map_at_1": 0.59649,
"map_at_10": 0.71667,
"map_at_100": 0.71923,
"map_at_1000": 0.71923,
"map_at_20": 0.71793,
"map_at_3": 0.69298,
"map_at_5": 0.7057,
"mrr_at_1": 0.5964912280701754,
"mrr_at_10": 0.7166666666666666,
"mrr_at_100": 0.7192301247857292,
"mrr_at_1000": 0.7192301247857292,
"mrr_at_20": 0.717925795349618,
"mrr_at_3": 0.6929824561403509,
"mrr_at_5": 0.7057017543859649,
"nauc_map_at_1000_diff1": 0.5936517099361166,
"nauc_map_at_1000_max": 0.23025820800907956,
"nauc_map_at_1000_std": -0.44830854310298024,
"nauc_map_at_100_diff1": 0.5936517099361166,
"nauc_map_at_100_max": 0.23025820800907956,
"nauc_map_at_100_std": -0.44830854310298024,
"nauc_map_at_10_diff1": 0.5893271930047639,
"nauc_map_at_10_max": 0.2276498001316117,
"nauc_map_at_10_std": -0.4551546095194692,
"nauc_map_at_1_diff1": 0.6110833815898389,
"nauc_map_at_1_max": 0.1908294474342627,
"nauc_map_at_1_std": -0.35314996932177417,
"nauc_map_at_20_diff1": 0.59448055498058,
"nauc_map_at_20_max": 0.2329357588721152,
"nauc_map_at_20_std": -0.45255932263784404,
"nauc_map_at_3_diff1": 0.5938383228262359,
"nauc_map_at_3_max": 0.23164372605869038,
"nauc_map_at_3_std": -0.43564852034913903,
"nauc_map_at_5_diff1": 0.5900023435482211,
"nauc_map_at_5_max": 0.23294445889550133,
"nauc_map_at_5_std": -0.45567242172858874,
"nauc_mrr_at_1000_diff1": 0.5936517099361166,
"nauc_mrr_at_1000_max": 0.23025820800907956,
"nauc_mrr_at_1000_std": -0.44830854310298024,
"nauc_mrr_at_100_diff1": 0.5936517099361166,
"nauc_mrr_at_100_max": 0.23025820800907956,
"nauc_mrr_at_100_std": -0.44830854310298024,
"nauc_mrr_at_10_diff1": 0.5893271930047639,
"nauc_mrr_at_10_max": 0.2276498001316117,
"nauc_mrr_at_10_std": -0.4551546095194692,
"nauc_mrr_at_1_diff1": 0.6110833815898389,
"nauc_mrr_at_1_max": 0.1908294474342627,
"nauc_mrr_at_1_std": -0.35314996932177417,
"nauc_mrr_at_20_diff1": 0.59448055498058,
"nauc_mrr_at_20_max": 0.2329357588721152,
"nauc_mrr_at_20_std": -0.45255932263784404,
"nauc_mrr_at_3_diff1": 0.5938383228262359,
"nauc_mrr_at_3_max": 0.23164372605869038,
"nauc_mrr_at_3_std": -0.43564852034913903,
"nauc_mrr_at_5_diff1": 0.5900023435482211,
"nauc_mrr_at_5_max": 0.23294445889550133,
"nauc_mrr_at_5_std": -0.45567242172858874,
"nauc_ndcg_at_1000_diff1": 0.5925722016847822,
"nauc_ndcg_at_1000_max": 0.2381630983055214,
"nauc_ndcg_at_1000_std": -0.467474281511242,
"nauc_ndcg_at_100_diff1": 0.5925722016847822,
"nauc_ndcg_at_100_max": 0.2381630983055214,
"nauc_ndcg_at_100_std": -0.467474281511242,
"nauc_ndcg_at_10_diff1": 0.5800090322494245,
"nauc_ndcg_at_10_max": 0.24203689898525843,
"nauc_ndcg_at_10_std": -0.5118177090163054,
"nauc_ndcg_at_1_diff1": 0.6110833815898389,
"nauc_ndcg_at_1_max": 0.1908294474342627,
"nauc_ndcg_at_1_std": -0.35314996932177417,
"nauc_ndcg_at_20_diff1": 0.598047860353563,
"nauc_ndcg_at_20_max": 0.25996203653945654,
"nauc_ndcg_at_20_std": -0.5029431420978402,
"nauc_ndcg_at_3_diff1": 0.5896443334804378,
"nauc_ndcg_at_3_max": 0.2439244231908955,
"nauc_ndcg_at_3_std": -0.4669491517657949,
"nauc_ndcg_at_5_diff1": 0.5816884009461497,
"nauc_ndcg_at_5_max": 0.2509765225930184,
"nauc_ndcg_at_5_std": -0.5086643597687968,
"nauc_precision_at_1000_diff1": 1.0,
"nauc_precision_at_1000_max": 1.0,
"nauc_precision_at_1000_std": 1.0,
"nauc_precision_at_100_diff1": 1.0,
"nauc_precision_at_100_max": 1.0,
"nauc_precision_at_100_std": 1.0,
"nauc_precision_at_10_diff1": 0.47965320075378115,
"nauc_precision_at_10_max": 0.40503318457167936,
"nauc_precision_at_10_std": -1.143569297810643,
"nauc_precision_at_1_diff1": 0.6110833815898389,
"nauc_precision_at_1_max": 0.1908294474342627,
"nauc_precision_at_1_std": -0.35314996932177417,
"nauc_precision_at_20_diff1": 0.7008965455883202,
"nauc_precision_at_20_max": 0.7183536566232184,
"nauc_precision_at_20_std": -1.2588569287895695,
"nauc_precision_at_3_diff1": 0.5731811728893021,
"nauc_precision_at_3_max": 0.2938957496013745,
"nauc_precision_at_3_std": -0.5982375555469349,
"nauc_precision_at_5_diff1": 0.5358095787755137,
"nauc_precision_at_5_max": 0.35334774502959576,
"nauc_precision_at_5_std": -0.8077434531545805,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": NaN,
"nauc_recall_at_100_max": NaN,
"nauc_recall_at_100_std": NaN,
"nauc_recall_at_10_diff1": 0.4796532007537782,
"nauc_recall_at_10_max": 0.40503318457167875,
"nauc_recall_at_10_std": -1.1435692978106466,
"nauc_recall_at_1_diff1": 0.6110833815898389,
"nauc_recall_at_1_max": 0.1908294474342627,
"nauc_recall_at_1_std": -0.35314996932177417,
"nauc_recall_at_20_diff1": 0.700896545588321,
"nauc_recall_at_20_max": 0.7183536566232177,
"nauc_recall_at_20_std": -1.2588569287895657,
"nauc_recall_at_3_diff1": 0.5731811728893031,
"nauc_recall_at_3_max": 0.2938957496013749,
"nauc_recall_at_3_std": -0.5982375555469336,
"nauc_recall_at_5_diff1": 0.5358095787755134,
"nauc_recall_at_5_max": 0.3533477450295955,
"nauc_recall_at_5_std": -0.8077434531545817,
"ndcg_at_1": 0.59649,
"ndcg_at_10": 0.77108,
"ndcg_at_100": 0.78411,
"ndcg_at_1000": 0.78411,
"ndcg_at_20": 0.77556,
"ndcg_at_3": 0.72452,
"ndcg_at_5": 0.7468,
"precision_at_1": 0.59649,
"precision_at_10": 0.09386,
"precision_at_100": 0.01,
"precision_at_1000": 0.001,
"precision_at_20": 0.04781,
"precision_at_3": 0.27193,
"precision_at_5": 0.17368,
"recall_at_1": 0.59649,
"recall_at_10": 0.9386,
"recall_at_100": 1.0,
"recall_at_1000": 1.0,
"recall_at_20": 0.95614,
"recall_at_3": 0.81579,
"recall_at_5": 0.86842
}
]
},
"task_name": "AutoRAGRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/BelebeleRetrieval.json
================================================
{
"dataset_revision": "75b399394a9803252cfec289d103de462763db7c",
"evaluation_time": 9.514682531356812,
"kg_co2_emissions": null,
"mteb_version": "1.20.0",
"scores": {
"test": [
{
"hf_subset": "kor_Hang-kor_Hang",
"languages": [
"kor-Hang",
"kor-Hang"
],
"main_score": 0.8796,
"map_at_1": 0.80333,
"map_at_10": 0.85849,
"map_at_100": 0.86055,
"map_at_1000": 0.86059,
"map_at_20": 0.85972,
"map_at_3": 0.85074,
"map_at_5": 0.85674,
"mrr_at_1": 0.8033333333333333,
"mrr_at_10": 0.8584872134038802,
"mrr_at_100": 0.8605501771786044,
"mrr_at_1000": 0.860594347604656,
"mrr_at_20": 0.859719015958051,
"mrr_at_3": 0.8507407407407409,
"mrr_at_5": 0.8567407407407408,
"nauc_map_at_1000_diff1": 0.8651361278493194,
"nauc_map_at_1000_max": 0.7418586175399554,
"nauc_map_at_1000_std": 0.013734735612603912,
"nauc_map_at_100_diff1": 0.8651555699121707,
"nauc_map_at_100_max": 0.7419304322492464,
"nauc_map_at_100_std": 0.013970732162235726,
"nauc_map_at_10_diff1": 0.8642127452490442,
"nauc_map_at_10_max": 0.7423022391957348,
"nauc_map_at_10_std": 0.013077755502568747,
"nauc_map_at_1_diff1": 0.8783845490368515,
"nauc_map_at_1_max": 0.7144604734281541,
"nauc_map_at_1_std": -0.03075479037358894,
"nauc_map_at_20_diff1": 0.8653809000316373,
"nauc_map_at_20_max": 0.7427858268462051,
"nauc_map_at_20_std": 0.015015758160884355,
"nauc_map_at_3_diff1": 0.8595602194780724,
"nauc_map_at_3_max": 0.7380410878857894,
"nauc_map_at_3_std": 0.0037690970412170128,
"nauc_map_at_5_diff1": 0.8639913582671778,
"nauc_map_at_5_max": 0.7441652104318379,
"nauc_map_at_5_std": 0.019813631471681307,
"nauc_mrr_at_1000_diff1": 0.8651361278493194,
"nauc_mrr_at_1000_max": 0.7418586175399554,
"nauc_mrr_at_1000_std": 0.013734735612603912,
"nauc_mrr_at_100_diff1": 0.8651555699121707,
"nauc_mrr_at_100_max": 0.7419304322492464,
"nauc_mrr_at_100_std": 0.013970732162235726,
"nauc_mrr_at_10_diff1": 0.8642127452490442,
"nauc_mrr_at_10_max": 0.7423022391957348,
"nauc_mrr_at_10_std": 0.013077755502568747,
"nauc_mrr_at_1_diff1": 0.8783845490368515,
"nauc_mrr_at_1_max": 0.7144604734281541,
"nauc_mrr_at_1_std": -0.03075479037358894,
"nauc_mrr_at_20_diff1": 0.8653809000316373,
"nauc_mrr_at_20_max": 0.7427858268462051,
"nauc_mrr_at_20_std": 0.015015758160884355,
"nauc_mrr_at_3_diff1": 0.8595602194780724,
"nauc_mrr_at_3_max": 0.7380410878857894,
"nauc_mrr_at_3_std": 0.0037690970412170128,
"nauc_mrr_at_5_diff1": 0.8639913582671778,
"nauc_mrr_at_5_max": 0.7441652104318379,
"nauc_mrr_at_5_std": 0.019813631471681307,
"nauc_ndcg_at_1000_diff1": 0.8640736382725115,
"nauc_ndcg_at_1000_max": 0.7476194763678936,
"nauc_ndcg_at_1000_std": 0.023164955045697332,
"nauc_ndcg_at_100_diff1": 0.8646708926516866,
"nauc_ndcg_at_100_max": 0.7497271398718169,
"nauc_ndcg_at_100_std": 0.030462267429729645,
"nauc_ndcg_at_10_diff1": 0.8601464938487059,
"nauc_ndcg_at_10_max": 0.7537308843825129,
"nauc_ndcg_at_10_std": 0.02743877031929201,
"nauc_ndcg_at_1_diff1": 0.8783845490368515,
"nauc_ndcg_at_1_max": 0.7144604734281541,
"nauc_ndcg_at_1_std": -0.03075479037358894,
"nauc_ndcg_at_20_diff1": 0.8660678949081592,
"nauc_ndcg_at_20_max": 0.756589310017133,
"nauc_ndcg_at_20_std": 0.03472573440205783,
"nauc_ndcg_at_3_diff1": 0.8518734277875382,
"nauc_ndcg_at_3_max": 0.7460751997142202,
"nauc_ndcg_at_3_std": 0.01431613860324495,
"nauc_ndcg_at_5_diff1": 0.8600816216068748,
"nauc_ndcg_at_5_max": 0.7588431787096619,
"nauc_ndcg_at_5_std": 0.04660310694125582,
"nauc_precision_at_1000_diff1": NaN,
"nauc_precision_at_1000_max": NaN,
"nauc_precision_at_1000_std": NaN,
"nauc_precision_at_100_diff1": 0.9253034547152017,
"nauc_precision_at_100_max": 0.962651727357627,
"nauc_precision_at_100_std": 0.7736427904495127,
"nauc_precision_at_10_diff1": 0.834376888010103,
"nauc_precision_at_10_max": 0.8285091814503543,
"nauc_precision_at_10_std": 0.1116237344611039,
"nauc_precision_at_1_diff1": 0.8783845490368515,
"nauc_precision_at_1_max": 0.7144604734281541,
"nauc_precision_at_1_std": -0.03075479037358894,
"nauc_precision_at_20_diff1": 0.8951580632252828,
"nauc_precision_at_20_max": 0.8967587034813862,
"nauc_precision_at_20_std": 0.22232893157262623,
"nauc_precision_at_3_diff1": 0.8191105899724272,
"nauc_precision_at_3_max": 0.7795691920178943,
"nauc_precision_at_3_std": 0.05760211061168791,
"nauc_precision_at_5_diff1": 0.8404324692840104,
"nauc_precision_at_5_max": 0.8426852222370399,
"nauc_precision_at_5_std": 0.20104338031508923,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 0.9253034547152172,
"nauc_recall_at_100_max": 0.962651727357617,
"nauc_recall_at_100_std": 0.7736427904495109,
"nauc_recall_at_10_diff1": 0.8343768880101047,
"nauc_recall_at_10_max": 0.8285091814503583,
"nauc_recall_at_10_std": 0.1116237344611009,
"nauc_recall_at_1_diff1": 0.8783845490368515,
"nauc_recall_at_1_max": 0.7144604734281541,
"nauc_recall_at_1_std": -0.03075479037358894,
"nauc_recall_at_20_diff1": 0.8951580632252891,
"nauc_recall_at_20_max": 0.8967587034813924,
"nauc_recall_at_20_std": 0.22232893157262798,
"nauc_recall_at_3_diff1": 0.8191105899724227,
"nauc_recall_at_3_max": 0.7795691920178919,
"nauc_recall_at_3_std": 0.057602110611686456,
"nauc_recall_at_5_diff1": 0.8404324692840107,
"nauc_recall_at_5_max": 0.8426852222370445,
"nauc_recall_at_5_std": 0.2010433803150886,
"ndcg_at_1": 0.80333,
"ndcg_at_10": 0.8796,
"ndcg_at_100": 0.88994,
"ndcg_at_1000": 0.89097,
"ndcg_at_20": 0.88409,
"ndcg_at_3": 0.86465,
"ndcg_at_5": 0.87532,
"precision_at_1": 0.80333,
"precision_at_10": 0.09433,
"precision_at_100": 0.00992,
"precision_at_1000": 0.001,
"precision_at_20": 0.04806,
"precision_at_3": 0.30148,
"precision_at_5": 0.186,
"recall_at_1": 0.80333,
"recall_at_10": 0.94333,
"recall_at_100": 0.99222,
"recall_at_1000": 1.0,
"recall_at_20": 0.96111,
"recall_at_3": 0.90444,
"recall_at_5": 0.93
},
{
"hf_subset": "kor_Hang-eng_Latn",
"languages": [
"kor-Hang",
"eng-Latn"
],
"main_score": 0.88062,
"map_at_1": 0.79778,
"map_at_10": 0.85643,
"map_at_100": 0.85821,
"map_at_1000": 0.85825,
"map_at_20": 0.85771,
"map_at_3": 0.84815,
"map_at_5": 0.85343,
"mrr_at_1": 0.7977777777777778,
"mrr_at_10": 0.856428130511464,
"mrr_at_100": 0.8582147049962549,
"mrr_at_1000": 0.8582509822841501,
"mrr_at_20": 0.8577066769468734,
"mrr_at_3": 0.8481481481481484,
"mrr_at_5": 0.8534259259259264,
"nauc_map_at_1000_diff1": 0.829546134330904,
"nauc_map_at_1000_max": 0.6325855821857873,
"nauc_map_at_1000_std": -0.08592721118228824,
"nauc_map_at_100_diff1": 0.8295521080594748,
"nauc_map_at_100_max": 0.6326847487835539,
"nauc_map_at_100_std": -0.0856605907756247,
"nauc_map_at_10_diff1": 0.8288997350230023,
"nauc_map_at_10_max": 0.6333210130677666,
"nauc_map_at_10_std": -0.08382181759549567,
"nauc_map_at_1_diff1": 0.8469104756035148,
"nauc_map_at_1_max": 0.6136072240228472,
"nauc_map_at_1_std": -0.10831081787516371,
"nauc_map_at_20_diff1": 0.8291637416330593,
"nauc_map_at_20_max": 0.6330502230134387,
"nauc_map_at_20_std": -0.08618019653766029,
"nauc_map_at_3_diff1": 0.827284260162407,
"nauc_map_at_3_max": 0.6325155136406339,
"nauc_map_at_3_std": -0.08899414161844547,
"nauc_map_at_5_diff1": 0.8268611228256288,
"nauc_map_at_5_max": 0.6324112783164382,
"nauc_map_at_5_std": -0.08103465516566621,
"nauc_mrr_at_1000_diff1": 0.829546134330904,
"nauc_mrr_at_1000_max": 0.6325855821857873,
"nauc_mrr_at_1000_std": -0.08592721118228824,
"nauc_mrr_at_100_diff1": 0.8295521080594748,
"nauc_mrr_at_100_max": 0.6326847487835539,
"nauc_mrr_at_100_std": -0.0856605907756247,
"nauc_mrr_at_10_diff1": 0.8288997350230023,
"nauc_mrr_at_10_max": 0.6333210130677666,
"nauc_mrr_at_10_std": -0.08382181759549567,
"nauc_mrr_at_1_diff1": 0.8469104756035148,
"nauc_mrr_at_1_max": 0.6136072240228472,
"nauc_mrr_at_1_std": -0.10831081787516371,
"nauc_mrr_at_20_diff1": 0.8291637416330593,
"nauc_mrr_at_20_max": 0.6330502230134387,
"nauc_mrr_at_20_std": -0.08618019653766029,
"nauc_mrr_at_3_diff1": 0.827284260162407,
"nauc_mrr_at_3_max": 0.6325155136406339,
"nauc_mrr_at_3_std": -0.08899414161844547,
"nauc_mrr_at_5_diff1": 0.8268611228256288,
"nauc_mrr_at_5_max": 0.6324112783164382,
"nauc_mrr_at_5_std": -0.08103465516566621,
"nauc_ndcg_at_1000_diff1": 0.827275101042018,
"nauc_ndcg_at_1000_max": 0.6368223960440613,
"nauc_ndcg_at_1000_std": -0.0795689943247917,
"nauc_ndcg_at_100_diff1": 0.8275142613783739,
"nauc_ndcg_at_100_max": 0.6394490798501035,
"nauc_ndcg_at_100_std": -0.07252899389683454,
"nauc_ndcg_at_10_diff1": 0.8236698658167095,
"nauc_ndcg_at_10_max": 0.643210997098357,
"nauc_ndcg_at_10_std": -0.0682725779951663,
"nauc_ndcg_at_1_diff1": 0.8469104756035148,
"nauc_ndcg_at_1_max": 0.6136072240228472,
"nauc_ndcg_at_1_std": -0.10831081787516371,
"nauc_ndcg_at_20_diff1": 0.8243330368189321,
"nauc_ndcg_at_20_max": 0.6425555639374455,
"nauc_ndcg_at_20_std": -0.07771461296268094,
"nauc_ndcg_at_3_diff1": 0.8196030620594603,
"nauc_ndcg_at_3_max": 0.6396350400369007,
"nauc_ndcg_at_3_std": -0.07909721619527248,
"nauc_ndcg_at_5_diff1": 0.8181558557674538,
"nauc_ndcg_at_5_max": 0.6395382719190128,
"nauc_ndcg_at_5_std": -0.06251199626346327,
"nauc_precision_at_1000_diff1": NaN,
"nauc_precision_at_1000_max": NaN,
"nauc_precision_at_1000_std": NaN,
"nauc_precision_at_100_diff1": 0.8627450980391947,
"nauc_precision_at_100_max": 1.0,
"nauc_precision_at_100_std": 0.8921568627450869,
"nauc_precision_at_10_diff1": 0.7854751656760247,
"nauc_precision_at_10_max": 0.7316731570677019,
"nauc_precision_at_10_std": 0.0690601443829509,
"nauc_precision_at_1_diff1": 0.8469104756035148,
"nauc_precision_at_1_max": 0.6136072240228472,
"nauc_precision_at_1_std": -0.10831081787516371,
"nauc_precision_at_20_diff1": 0.7666705571117243,
"nauc_precision_at_20_max": 0.784702769996881,
"nauc_precision_at_20_std": 0.017098506069089116,
"nauc_precision_at_3_diff1": 0.7846249610955512,
"nauc_precision_at_3_max": 0.6719057993567797,
"nauc_precision_at_3_std": -0.032783483763876776,
"nauc_precision_at_5_diff1": 0.7670095360548549,
"nauc_precision_at_5_max": 0.6802589888414331,
"nauc_precision_at_5_std": 0.0515911282545789,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 0.8627450980392323,
"nauc_recall_at_100_max": 1.0,
"nauc_recall_at_100_std": 0.892156862745096,
"nauc_recall_at_10_diff1": 0.7854751656760276,
"nauc_recall_at_10_max": 0.7316731570677064,
"nauc_recall_at_10_std": 0.0690601443829571,
"nauc_recall_at_1_diff1": 0.8469104756035148,
"nauc_recall_at_1_max": 0.6136072240228472,
"nauc_recall_at_1_std": -0.10831081787516371,
"nauc_recall_at_20_diff1": 0.7666705571117347,
"nauc_recall_at_20_max": 0.7847027699968866,
"nauc_recall_at_20_std": 0.017098506069086708,
"nauc_recall_at_3_diff1": 0.7846249610955486,
"nauc_recall_at_3_max": 0.6719057993567795,
"nauc_recall_at_3_std": -0.03278348376387673,
"nauc_recall_at_5_diff1": 0.7670095360548592,
"nauc_recall_at_5_max": 0.6802589888414381,
"nauc_recall_at_5_std": 0.05159112825458121,
"ndcg_at_1": 0.79778,
"ndcg_at_10": 0.88062,
"ndcg_at_100": 0.88921,
"ndcg_at_1000": 0.88998,
"ndcg_at_20": 0.88536,
"ndcg_at_3": 0.86407,
"ndcg_at_5": 0.8734,
"precision_at_1": 0.79778,
"precision_at_10": 0.09544,
"precision_at_100": 0.00994,
"precision_at_1000": 0.001,
"precision_at_20": 0.04867,
"precision_at_3": 0.30333,
"precision_at_5": 0.18644,
"recall_at_1": 0.79778,
"recall_at_10": 0.95444,
"recall_at_100": 0.99444,
"recall_at_1000": 1.0,
"recall_at_20": 0.97333,
"recall_at_3": 0.91,
"recall_at_5": 0.93222
},
{
"hf_subset": "eng_Latn-kor_Hang",
"languages": [
"eng-Latn",
"kor-Hang"
],
"main_score": 0.85657,
"map_at_1": 0.75889,
"map_at_10": 0.82638,
"map_at_100": 0.82792,
"map_at_1000": 0.82798,
"map_at_20": 0.82724,
"map_at_3": 0.81167,
"map_at_5": 0.82194,
"mrr_at_1": 0.7588888888888888,
"mrr_at_10": 0.8263791887125224,
"mrr_at_100": 0.8279172664600791,
"mrr_at_1000": 0.8279781231588634,
"mrr_at_20": 0.8272371670392827,
"mrr_at_3": 0.8116666666666666,
"mrr_at_5": 0.8219444444444447,
"nauc_map_at_1000_diff1": 0.8080662924334656,
"nauc_map_at_1000_max": 0.7137610309322288,
"nauc_map_at_1000_std": 0.03889981948933412,
"nauc_map_at_100_diff1": 0.8080206640656972,
"nauc_map_at_100_max": 0.713844142428308,
"nauc_map_at_100_std": 0.03925475947340508,
"nauc_map_at_10_diff1": 0.8085438358710668,
"nauc_map_at_10_max": 0.7147965670336995,
"nauc_map_at_10_std": 0.041601046039760524,
"nauc_map_at_1_diff1": 0.8148941927542931,
"nauc_map_at_1_max": 0.6885576833277857,
"nauc_map_at_1_std": -0.005981815090553283,
"nauc_map_at_20_diff1": 0.8079968971636359,
"nauc_map_at_20_max": 0.7144601462940647,
"nauc_map_at_20_std": 0.04099096241352047,
"nauc_map_at_3_diff1": 0.8077983411024191,
"nauc_map_at_3_max": 0.717892878661459,
"nauc_map_at_3_std": 0.03975692189966652,
"nauc_map_at_5_diff1": 0.8063046003130837,
"nauc_map_at_5_max": 0.7106092932047338,
"nauc_map_at_5_std": 0.03282925356427449,
"nauc_mrr_at_1000_diff1": 0.8080662924334656,
"nauc_mrr_at_1000_max": 0.7137610309322288,
"nauc_mrr_at_1000_std": 0.03889981948933412,
"nauc_mrr_at_100_diff1": 0.8080206640656972,
"nauc_mrr_at_100_max": 0.713844142428308,
"nauc_mrr_at_100_std": 0.03925475947340508,
"nauc_mrr_at_10_diff1": 0.8085438358710668,
"nauc_mrr_at_10_max": 0.7147965670336995,
"nauc_mrr_at_10_std": 0.041601046039760524,
"nauc_mrr_at_1_diff1": 0.8148941927542931,
"nauc_mrr_at_1_max": 0.6885576833277857,
"nauc_mrr_at_1_std": -0.005981815090553283,
"nauc_mrr_at_20_diff1": 0.8079968971636359,
"nauc_mrr_at_20_max": 0.7144601462940647,
"nauc_mrr_at_20_std": 0.04099096241352047,
"nauc_mrr_at_3_diff1": 0.8077983411024191,
"nauc_mrr_at_3_max": 0.717892878661459,
"nauc_mrr_at_3_std": 0.03975692189966652,
"nauc_mrr_at_5_diff1": 0.8063046003130837,
"nauc_mrr_at_5_max": 0.7106092932047338,
"nauc_mrr_at_5_std": 0.03282925356427449,
"nauc_ndcg_at_1000_diff1": 0.8071936485236224,
"nauc_ndcg_at_1000_max": 0.7194181195131821,
"nauc_ndcg_at_1000_std": 0.05342224213986593,
"nauc_ndcg_at_100_diff1": 0.8061779133250535,
"nauc_ndcg_at_100_max": 0.7219947254747765,
"nauc_ndcg_at_100_std": 0.06398427568254675,
"nauc_ndcg_at_10_diff1": 0.8084808695754142,
"nauc_ndcg_at_10_max": 0.7284930166428097,
"nauc_ndcg_at_10_std": 0.07720960494895679,
"nauc_ndcg_at_1_diff1": 0.8148941927542931,
"nauc_ndcg_at_1_max": 0.6885576833277857,
"nauc_ndcg_at_1_std": -0.005981815090553283,
"nauc_ndcg_at_20_diff1": 0.8062310856840769,
"nauc_ndcg_at_20_max": 0.7277722658852183,
"nauc_ndcg_at_20_std": 0.07684336868935904,
"nauc_ndcg_at_3_diff1": 0.8059262738497057,
"nauc_ndcg_at_3_max": 0.7294359748625586,
"nauc_ndcg_at_3_std": 0.06181714689020046,
"nauc_ndcg_at_5_diff1": 0.8028769320851481,
"nauc_ndcg_at_5_max": 0.7164772902430775,
"nauc_ndcg_at_5_std": 0.05087262789435781,
"nauc_precision_at_1000_diff1": NaN,
"nauc_precision_at_1000_max": NaN,
"nauc_precision_at_1000_std": NaN,
"nauc_precision_at_100_diff1": 0.7288048552754708,
"nauc_precision_at_100_max": 0.9423436041083034,
"nauc_precision_at_100_std": 0.9554154995331525,
"nauc_precision_at_10_diff1": 0.813175640626619,
"nauc_precision_at_10_max": 0.8620915032679718,
"nauc_precision_at_10_std": 0.4347235190372405,
"nauc_precision_at_1_diff1": 0.8148941927542931,
"nauc_precision_at_1_max": 0.6885576833277857,
"nauc_precision_at_1_std": -0.005981815090553283,
"nauc_precision_at_20_diff1": 0.7854122040973202,
"nauc_precision_at_20_max": 0.8985280386664453,
"nauc_precision_at_20_std": 0.5554457076948398,
"nauc_precision_at_3_diff1": 0.7987855205754054,
"nauc_precision_at_3_max": 0.7759908703488474,
"nauc_precision_at_3_std": 0.15259740259740234,
"nauc_precision_at_5_diff1": 0.7832790773967219,
"nauc_precision_at_5_max": 0.7469086733792593,
"nauc_precision_at_5_std": 0.15426350720468127,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": 0.7288048552754446,
"nauc_recall_at_100_max": 0.9423436041083036,
"nauc_recall_at_100_std": 0.9554154995331435,
"nauc_recall_at_10_diff1": 0.8131756406266185,
"nauc_recall_at_10_max": 0.8620915032679746,
"nauc_recall_at_10_std": 0.4347235190372419,
"nauc_recall_at_1_diff1": 0.8148941927542931,
"nauc_recall_at_1_max": 0.6885576833277857,
"nauc_recall_at_1_std": -0.005981815090553283,
"nauc_recall_at_20_diff1": 0.7854122040973217,
"nauc_recall_at_20_max": 0.8985280386664473,
"nauc_recall_at_20_std": 0.5554457076948423,
"nauc_recall_at_3_diff1": 0.7987855205754045,
"nauc_recall_at_3_max": 0.7759908703488476,
"nauc_recall_at_3_std": 0.15259740259740326,
"nauc_recall_at_5_diff1": 0.7832790773967254,
"nauc_recall_at_5_max": 0.7469086733792616,
"nauc_recall_at_5_std": 0.15426350720468285,
"ndcg_at_1": 0.75889,
"ndcg_at_10": 0.85657,
"ndcg_at_100": 0.86461,
"ndcg_at_1000": 0.86607,
"ndcg_at_20": 0.85967,
"ndcg_at_3": 0.8276,
"ndcg_at_5": 0.84601,
"precision_at_1": 0.75889,
"precision_at_10": 0.095,
"precision_at_100": 0.00989,
"precision_at_1000": 0.001,
"precision_at_20": 0.04811,
"precision_at_3": 0.29111,
"precision_at_5": 0.18356,
"recall_at_1": 0.75889,
"recall_at_10": 0.95,
"recall_at_100": 0.98889,
"recall_at_1000": 1.0,
"recall_at_20": 0.96222,
"recall_at_3": 0.87333,
"recall_at_5": 0.91778
}
]
},
"task_name": "BelebeleRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/Ko-StrategyQA.json
================================================
{
"dataset_revision": "d243889a3eb6654029dbd7e7f9319ae31d58f97c",
"evaluation_time": 47.95084285736084,
"kg_co2_emissions": null,
"mteb_version": "1.19.4",
"scores": {
"dev": [
{
"hf_subset": "default",
"languages": [
"kor-Hang"
],
"main_score": 0.75121,
"map_at_1": 0.45135,
"map_at_10": 0.69709,
"map_at_100": 0.70379,
"map_at_1000": 0.7042,
"map_at_20": 0.70167,
"map_at_3": 0.65496,
"map_at_5": 0.68422,
"mrr_at_1": 0.7010135135135135,
"mrr_at_10": 0.7737069658944656,
"mrr_at_100": 0.7755940101379746,
"mrr_at_1000": 0.775762024387044,
"mrr_at_20": 0.7747121401629913,
"mrr_at_3": 0.764921171171171,
"mrr_at_5": 0.7705799549549547,
"nauc_map_at_1000_diff1": 0.5519197145137611,
"nauc_map_at_1000_max": 0.6067402935995151,
"nauc_map_at_1000_std": 0.1594828873545819,
"nauc_map_at_100_diff1": 0.552117983876564,
"nauc_map_at_100_max": 0.6071358257277069,
"nauc_map_at_100_std": 0.15992886906044776,
"nauc_map_at_10_diff1": 0.5482356659883542,
"nauc_map_at_10_max": 0.6036640161616742,
"nauc_map_at_10_std": 0.14948304159183426,
"nauc_map_at_1_diff1": 0.5834434025577803,
"nauc_map_at_1_max": 0.299451623991129,
"nauc_map_at_1_std": -0.022953282119810403,
"nauc_map_at_20_diff1": 0.551019680772447,
"nauc_map_at_20_max": 0.6074107021103508,
"nauc_map_at_20_std": 0.15853657489864967,
"nauc_map_at_3_diff1": 0.5248726081949865,
"nauc_map_at_3_max": 0.5594477813787555,
"nauc_map_at_3_std": 0.08416007387108534,
"nauc_map_at_5_diff1": 0.5373092416579366,
"nauc_map_at_5_max": 0.5899555474257632,
"nauc_map_at_5_std": 0.12732982461525347,
"nauc_mrr_at_1000_diff1": 0.6630500026079771,
"nauc_mrr_at_1000_max": 0.6736685608978698,
"nauc_mrr_at_1000_std": 0.2320878485491545,
"nauc_mrr_at_100_diff1": 0.663057226112171,
"nauc_mrr_at_100_max": 0.6738925445530085,
"nauc_mrr_at_100_std": 0.2325917862628659,
"nauc_mrr_at_10_diff1": 0.6636967780082375,
"nauc_mrr_at_10_max": 0.6762588126119293,
"nauc_mrr_at_10_std": 0.23291273028624304,
"nauc_mrr_at_1_diff1": 0.661500115820817,
"nauc_mrr_at_1_max": 0.5770263549598156,
"nauc_mrr_at_1_std": 0.1313742842575525,
"nauc_mrr_at_20_diff1": 0.662962026964006,
"nauc_mrr_at_20_max": 0.6747824530605779,
"nauc_mrr_at_20_std": 0.23226267353927715,
"nauc_mrr_at_3_diff1": 0.658707138510419,
"nauc_mrr_at_3_max": 0.6720413722313505,
"nauc_mrr_at_3_std": 0.214088474893056,
"nauc_mrr_at_5_diff1": 0.6603441608824964,
"nauc_mrr_at_5_max": 0.6736745604944314,
"nauc_mrr_at_5_std": 0.22846499218561858,
"nauc_ndcg_at_1000_diff1": 0.5907551369979055,
"nauc_ndcg_at_1000_max": 0.6615877024334198,
"nauc_ndcg_at_1000_std": 0.2393015359142359,
"nauc_ndcg_at_100_diff1": 0.5937442728212519,
"nauc_ndcg_at_100_max": 0.670641015014622,
"nauc_ndcg_at_100_std": 0.25487773336698044,
"nauc_ndcg_at_10_diff1": 0.5831817543877047,
"nauc_ndcg_at_10_max": 0.6643418532861225,
"nauc_ndcg_at_10_std": 0.22326362761367885,
"nauc_ndcg_at_1_diff1": 0.661500115820817,
"nauc_ndcg_at_1_max": 0.5770263549598156,
"nauc_ndcg_at_1_std": 0.1313742842575525,
"nauc_ndcg_at_20_diff1": 0.5899026096075843,
"nauc_ndcg_at_20_max": 0.673272767215356,
"nauc_ndcg_at_20_std": 0.24635138529614536,
"nauc_ndcg_at_3_diff1": 0.5479702226955419,
"nauc_ndcg_at_3_max": 0.6059582416432887,
"nauc_ndcg_at_3_std": 0.12743795996465662,
"nauc_ndcg_at_5_diff1": 0.5628160195267607,
"nauc_ndcg_at_5_max": 0.6411398480758634,
"nauc_ndcg_at_5_std": 0.18438351907666847,
"nauc_precision_at_1000_diff1": -0.2004685252782137,
"nauc_precision_at_1000_max": 0.07063855451998648,
"nauc_precision_at_1000_std": 0.23065877466350113,
"nauc_precision_at_100_diff1": -0.07861027277114246,
"nauc_precision_at_100_max": 0.21703569000810188,
"nauc_precision_at_100_std": 0.31518785228088636,
"nauc_precision_at_10_diff1": 0.012468448328005923,
"nauc_precision_at_10_max": 0.35106876700614753,
"nauc_precision_at_10_std": 0.30589978246233046,
"nauc_precision_at_1_diff1": 0.661500115820817,
"nauc_precision_at_1_max": 0.5770263549598156,
"nauc_precision_at_1_std": 0.1313742842575525,
"nauc_precision_at_20_diff1": -0.015991023617958375,
"nauc_precision_at_20_max": 0.31120661076351014,
"nauc_precision_at_20_std": 0.32862281412296374,
"nauc_precision_at_3_diff1": 0.10652022166353019,
"nauc_precision_at_3_max": 0.4643801024890446,
"nauc_precision_at_3_std": 0.21573365606067008,
"nauc_precision_at_5_diff1": 0.04752964400629679,
"nauc_precision_at_5_max": 0.3993637294469268,
"nauc_precision_at_5_std": 0.26849236828347134,
"nauc_recall_at_1000_diff1": 0.6310537710228703,
"nauc_recall_at_1000_max": 0.9077919776775288,
"nauc_recall_at_1000_std": 0.7655930243673666,
"nauc_recall_at_100_diff1": 0.6368032349603382,
"nauc_recall_at_100_max": 0.8803880729883224,
"nauc_recall_at_100_std": 0.6979867070823739,
"nauc_recall_at_10_diff1": 0.5650349723481966,
"nauc_recall_at_10_max": 0.7654270251851792,
"nauc_recall_at_10_std": 0.3773682882526592,
"nauc_recall_at_1_diff1": 0.5834434025577803,
"nauc_recall_at_1_max": 0.299451623991129,
"nauc_recall_at_1_std": -0.022953282119810403,
"nauc_recall_at_20_diff1": 0.5996092880814857,
"nauc_recall_at_20_max": 0.829226851926464,
"nauc_recall_at_20_std": 0.5101268399413494,
"nauc_recall_at_3_diff1": 0.49205076291344274,
"nauc_recall_at_3_max": 0.6109051461139894,
"nauc_recall_at_3_std": 0.12072253500359617,
"nauc_recall_at_5_diff1": 0.5088705234664951,
"nauc_recall_at_5_max": 0.6817156339665192,
"nauc_recall_at_5_std": 0.24410069255370517,
"ndcg_at_1": 0.70101,
"ndcg_at_10": 0.75121,
"ndcg_at_100": 0.77256,
"ndcg_at_1000": 0.78115,
"ndcg_at_20": 0.76145,
"ndcg_at_3": 0.70056,
"ndcg_at_5": 0.73113,
"precision_at_1": 0.70101,
"precision_at_10": 0.15152,
"precision_at_100": 0.01698,
"precision_at_1000": 0.00183,
"precision_at_20": 0.07956,
"precision_at_3": 0.41667,
"precision_at_5": 0.27939,
"recall_at_1": 0.45135,
"recall_at_10": 0.82298,
"recall_at_100": 0.90386,
"recall_at_1000": 0.9587,
"recall_at_20": 0.85468,
"recall_at_3": 0.71127,
"recall_at_5": 0.77506
}
]
},
"task_name": "Ko-StrategyQA"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MIRACLRetrieval.json
================================================
{
"dataset_revision": "main",
"evaluation_time": 4725.387804031372,
"kg_co2_emissions": null,
"mteb_version": "1.20.0",
"scores": {
"dev": [
{
"hf_subset": "ko",
"languages": [
"kor-Kore"
],
"main_score": 0.62697,
"map_at_1": 0.38205,
"map_at_10": 0.54289,
"map_at_100": 0.56555,
"map_at_1000": 0.56697,
"map_at_20": 0.55681,
"map_at_3": 0.48344,
"map_at_5": 0.51568,
"mrr_at_1": 0.6103286384976526,
"mrr_at_10": 0.7039291303375809,
"mrr_at_100": 0.7082462092431728,
"mrr_at_1000": 0.7084151998358826,
"mrr_at_20": 0.7072567444280695,
"mrr_at_3": 0.6862284820031298,
"mrr_at_5": 0.69679186228482,
"nauc_map_at_1000_diff1": 0.3880663627782523,
"nauc_map_at_1000_max": 0.26875837358990384,
"nauc_map_at_1000_std": -0.02915658141808151,
"nauc_map_at_100_diff1": 0.3877255300606842,
"nauc_map_at_100_max": 0.2676573499357481,
"nauc_map_at_100_std": -0.03003797795589652,
"nauc_map_at_10_diff1": 0.3967106291403696,
"nauc_map_at_10_max": 0.2507505837032776,
"nauc_map_at_10_std": -0.051781293185286445,
"nauc_map_at_1_diff1": 0.41809526232663874,
"nauc_map_at_1_max": 0.08779534853304669,
"nauc_map_at_1_std": -0.17386942942712602,
"nauc_map_at_20_diff1": 0.3915764765548958,
"nauc_map_at_20_max": 0.25739603266273464,
"nauc_map_at_20_std": -0.041442559021572345,
"nauc_map_at_3_diff1": 0.42231719725329264,
"nauc_map_at_3_max": 0.18371697143417534,
"nauc_map_at_3_std": -0.12726929425612019,
"nauc_map_at_5_diff1": 0.398872122466833,
"nauc_map_at_5_max": 0.2183229034565644,
"nauc_map_at_5_std": -0.08595999951698406,
"nauc_mrr_at_1000_diff1": 0.4088929543341095,
"nauc_mrr_at_1000_max": 0.4960270065168594,
"nauc_mrr_at_1000_std": 0.1986308055671587,
"nauc_mrr_at_100_diff1": 0.40884542440189453,
"nauc_mrr_at_100_max": 0.496298929979252,
"nauc_mrr_at_100_std": 0.19901507136436228,
"nauc_mrr_at_10_diff1": 0.4095868941932426,
"nauc_mrr_at_10_max": 0.5016475476247045,
"nauc_mrr_at_10_std": 0.20295922598033728,
"nauc_mrr_at_1_diff1": 0.42845609791848677,
"nauc_mrr_at_1_max": 0.40065137930038525,
"nauc_mrr_at_1_std": 0.10762285365210995,
"nauc_mrr_at_20_diff1": 0.4083082899116937,
"nauc_mrr_at_20_max": 0.49640091643714074,
"nauc_mrr_at_20_std": 0.20091163621165942,
"nauc_mrr_at_3_diff1": 0.4157753000024607,
"nauc_mrr_at_3_max": 0.49681140521620853,
"nauc_mrr_at_3_std": 0.18299091495164696,
"nauc_mrr_at_5_diff1": 0.4024032183255278,
"nauc_mrr_at_5_max": 0.4965190104753864,
"nauc_mrr_at_5_std": 0.1973793570513383,
"nauc_ndcg_at_1000_diff1": 0.37373031518754046,
"nauc_ndcg_at_1000_max": 0.37607986329290355,
"nauc_ndcg_at_1000_std": 0.0908324185246208,
"nauc_ndcg_at_100_diff1": 0.3717491373566511,
"nauc_ndcg_at_100_max": 0.37240022954155677,
"nauc_ndcg_at_100_std": 0.08618214219538683,
"nauc_ndcg_at_10_diff1": 0.39086793566356887,
"nauc_ndcg_at_10_max": 0.33466796739784904,
"nauc_ndcg_at_10_std": 0.030749916588024073,
"nauc_ndcg_at_1_diff1": 0.42845609791848677,
"nauc_ndcg_at_1_max": 0.40065137930038525,
"nauc_ndcg_at_1_std": 0.10762285365210995,
"nauc_ndcg_at_20_diff1": 0.38054633502568497,
"nauc_ndcg_at_20_max": 0.3438995134967677,
"nauc_ndcg_at_20_std": 0.0562318258312696,
"nauc_ndcg_at_3_diff1": 0.399795423235567,
"nauc_ndcg_at_3_max": 0.33702985956781256,
"nauc_ndcg_at_3_std": 0.011897751562643624,
"nauc_ndcg_at_5_diff1": 0.37602006444813374,
"nauc_ndcg_at_5_max": 0.32301355137987625,
"nauc_ndcg_at_5_std": 0.014210821857346219,
"nauc_precision_at_1000_diff1": -0.2325905952276443,
"nauc_precision_at_1000_max": 0.26803251700559394,
"nauc_precision_at_1000_std": 0.40373084311985863,
"nauc_precision_at_100_diff1": -0.22361371198039076,
"nauc_precision_at_100_max": 0.30222103314451515,
"nauc_precision_at_100_std": 0.42174055990118403,
"nauc_precision_at_10_diff1": -0.10782271786242098,
"nauc_precision_at_10_max": 0.3906023360764504,
"nauc_precision_at_10_std": 0.41343020041282674,
"nauc_precision_at_1_diff1": 0.42845609791848677,
"nauc_precision_at_1_max": 0.40065137930038525,
"nauc_precision_at_1_std": 0.10762285365210995,
"nauc_precision_at_20_diff1": -0.16620077076923456,
"nauc_precision_at_20_max": 0.3540662038450409,
"nauc_precision_at_20_std": 0.4385742655258709,
"nauc_precision_at_3_diff1": 0.0643330951436578,
"nauc_precision_at_3_max": 0.4708826625517317,
"nauc_precision_at_3_std": 0.3150688758870942,
"nauc_precision_at_5_diff1": -0.0791402282451768,
"nauc_precision_at_5_max": 0.410837502746928,
"nauc_precision_at_5_std": 0.3739032437456903,
"nauc_recall_at_1000_diff1": 0.1461707557767423,
"nauc_recall_at_1000_max": 0.7392773551034885,
"nauc_recall_at_1000_std": 0.596389402600282,
"nauc_recall_at_100_diff1": 0.2603605715058474,
"nauc_recall_at_100_max": 0.4411100758773747,
"nauc_recall_at_100_std": 0.20665759469441622,
"nauc_recall_at_10_diff1": 0.3496804304290531,
"nauc_recall_at_10_max": 0.2969594791669843,
"nauc_recall_at_10_std": -0.0024174736235818084,
"nauc_recall_at_1_diff1": 0.41809526232663874,
"nauc_recall_at_1_max": 0.08779534853304669,
"nauc_recall_at_1_std": -0.17386942942712602,
"nauc_recall_at_20_diff1": 0.3034341078071222,
"nauc_recall_at_20_max": 0.29415428741234795,
"nauc_recall_at_20_std": 0.0621874859031154,
"nauc_recall_at_3_diff1": 0.4114352782373673,
"nauc_recall_at_3_max": 0.184362694523637,
"nauc_recall_at_3_std": -0.1315379775711195,
"nauc_recall_at_5_diff1": 0.3507401455620195,
"nauc_recall_at_5_max": 0.2305916637080133,
"nauc_recall_at_5_std": -0.0646045286797751,
"ndcg_at_1": 0.61033,
"ndcg_at_10": 0.62697,
"ndcg_at_100": 0.6861,
"ndcg_at_1000": 0.70173,
"ndcg_at_20": 0.65736,
"ndcg_at_3": 0.58892,
"ndcg_at_5": 0.60065,
"precision_at_1": 0.61033,
"precision_at_10": 0.15681,
"precision_at_100": 0.02244,
"precision_at_1000": 0.00253,
"precision_at_20": 0.09202,
"precision_at_3": 0.33959,
"precision_at_5": 0.2554,
"recall_at_1": 0.38205,
"recall_at_10": 0.70019,
"recall_at_100": 0.89421,
"recall_at_1000": 0.97981,
"recall_at_20": 0.78979,
"recall_at_3": 0.53982,
"recall_at_5": 0.6097
}
]
},
"task_name": "MIRACLRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MrTidyRetrieval.json
================================================
{
"dataset_revision": "fc24a3ce8f09746410daee3d5cd823ff7a0675b7",
"evaluation_time": 4789.847511768341,
"kg_co2_emissions": null,
"mteb_version": "1.20.0",
"scores": {
"test": [
{
"hf_subset": "korean",
"languages": [
"kor-Kore"
],
"main_score": 0.56464,
"map_at_1": 0.38836,
"map_at_10": 0.50553,
"map_at_100": 0.51348,
"map_at_1000": 0.51388,
"map_at_20": 0.51026,
"map_at_3": 0.47816,
"map_at_5": 0.4954,
"mrr_at_1": 0.42042755344418054,
"mrr_at_10": 0.5310343852505374,
"mrr_at_100": 0.5367455539072286,
"mrr_at_1000": 0.5370753449207969,
"mrr_at_20": 0.5347435394689465,
"mrr_at_3": 0.5079176563737136,
"mrr_at_5": 0.5206254948535236,
"nauc_map_at_1000_diff1": 0.25716543859897323,
"nauc_map_at_1000_max": 0.28005192170293824,
"nauc_map_at_1000_std": 0.03836511543988043,
"nauc_map_at_100_diff1": 0.25715155161945596,
"nauc_map_at_100_max": 0.2801515462923894,
"nauc_map_at_100_std": 0.03847960284606766,
"nauc_map_at_10_diff1": 0.25825901715325766,
"nauc_map_at_10_max": 0.2774051051549839,
"nauc_map_at_10_std": 0.030341349544946543,
"nauc_map_at_1_diff1": 0.32573416780172126,
"nauc_map_at_1_max": 0.31247776894025886,
"nauc_map_at_1_std": 0.031249381404232498,
"nauc_map_at_20_diff1": 0.257080817902839,
"nauc_map_at_20_max": 0.2782100911069668,
"nauc_map_at_20_std": 0.03451940941925907,
"nauc_map_at_3_diff1": 0.26845525923224,
"nauc_map_at_3_max": 0.2642457306385882,
"nauc_map_at_3_std": 0.015335480167255127,
"nauc_map_at_5_diff1": 0.25278758474435037,
"nauc_map_at_5_max": 0.27504315746221053,
"nauc_map_at_5_std": 0.029983842134557192,
"nauc_mrr_at_1000_diff1": 0.26088870787930196,
"nauc_mrr_at_1000_max": 0.2912226938065716,
"nauc_mrr_at_1000_std": 0.04285053316771879,
"nauc_mrr_at_100_diff1": 0.2609646228530708,
"nauc_mrr_at_100_max": 0.2914631566489875,
"nauc_mrr_at_100_std": 0.0430522291485771,
"nauc_mrr_at_10_diff1": 0.2609882291729948,
"nauc_mrr_at_10_max": 0.28934197519266663,
"nauc_mrr_at_10_std": 0.03827684254515211,
"nauc_mrr_at_1_diff1": 0.3214764381617425,
"nauc_mrr_at_1_max": 0.3109742940946706,
"nauc_mrr_at_1_std": 0.03263481415776953,
"nauc_mrr_at_20_diff1": 0.2603922334686195,
"nauc_mrr_at_20_max": 0.2900484255899885,
"nauc_mrr_at_20_std": 0.040662887127286745,
"nauc_mrr_at_3_diff1": 0.2654485903005151,
"nauc_mrr_at_3_max": 0.2779252830401248,
"nauc_mrr_at_3_std": 0.02424182001973642,
"nauc_mrr_at_5_diff1": 0.2539868410845816,
"nauc_mrr_at_5_max": 0.28505476937588964,
"nauc_mrr_at_5_std": 0.03596238739186569,
"nauc_ndcg_at_1000_diff1": 0.23557354884221127,
"nauc_ndcg_at_1000_max": 0.29032978771135,
"nauc_ndcg_at_1000_std": 0.06997467082862376,
"nauc_ndcg_at_100_diff1": 0.23571006906147526,
"nauc_ndcg_at_100_max": 0.29522278193987794,
"nauc_ndcg_at_100_std": 0.07461791164210363,
"nauc_ndcg_at_10_diff1": 0.23718143213771353,
"nauc_ndcg_at_10_max": 0.27862306219326183,
"nauc_ndcg_at_10_std": 0.036243938954022954,
"nauc_ndcg_at_1_diff1": 0.3214764381617425,
"nauc_ndcg_at_1_max": 0.3109742940946706,
"nauc_ndcg_at_1_std": 0.03263481415776953,
"nauc_ndcg_at_20_diff1": 0.23384962103391074,
"nauc_ndcg_at_20_max": 0.2823622591128196,
"nauc_ndcg_at_20_std": 0.05073943905013697,
"nauc_ndcg_at_3_diff1": 0.2505995080003967,
"nauc_ndcg_at_3_max": 0.2550849070629041,
"nauc_ndcg_at_3_std": 0.012143397163197251,
"nauc_ndcg_at_5_diff1": 0.22439685641544171,
"nauc_ndcg_at_5_max": 0.2716990904719191,
"nauc_ndcg_at_5_std": 0.03575325421970202,
"nauc_precision_at_1000_diff1": -0.13230947244366867,
"nauc_precision_at_1000_max": 0.050305125840942856,
"nauc_precision_at_1000_std": 0.1764339714748889,
"nauc_precision_at_100_diff1": -0.06222382942833803,
"nauc_precision_at_100_max": 0.13842447314423142,
"nauc_precision_at_100_std": 0.20705592120058777,
"nauc_precision_at_10_diff1": 0.05514361830417003,
"nauc_precision_at_10_max": 0.18701739784935376,
"nauc_precision_at_10_std": 0.04650331876462832,
"nauc_precision_at_1_diff1": 0.3214764381617425,
"nauc_precision_at_1_max": 0.3109742940946706,
"nauc_precision_at_1_std": 0.03263481415776953,
"nauc_precision_at_20_diff1": -0.00617052929258315,
"nauc_precision_at_20_max": 0.1565272393710742,
"nauc_precision_at_20_std": 0.1100427948727622,
"nauc_precision_at_3_diff1": 0.1468202987011345,
"nauc_precision_at_3_max": 0.1849822470919322,
"nauc_precision_at_3_std": -0.010667626125281129,
"nauc_precision_at_5_diff1": 0.05497589349994129,
"nauc_precision_at_5_max": 0.1879791183562475,
"nauc_precision_at_5_std": 0.0444158747825215,
"nauc_recall_at_1000_diff1": -0.03852211432611442,
"nauc_recall_at_1000_max": 0.41798503899896616,
"nauc_recall_at_1000_std": 0.6213555554385645,
"nauc_recall_at_100_diff1": 0.10092135478885442,
"nauc_recall_at_100_max": 0.4381086975662203,
"nauc_recall_at_100_std": 0.4171683707601491,
"nauc_recall_at_10_diff1": 0.15252772800619863,
"nauc_recall_at_10_max": 0.26507602202914265,
"nauc_recall_at_10_std": 0.045423224965121856,
"nauc_recall_at_1_diff1": 0.32573416780172126,
"nauc_recall_at_1_max": 0.31247776894025886,
"nauc_recall_at_1_std": 0.031249381404232498,
"nauc_recall_at_20_diff1": 0.13064148113752028,
"nauc_recall_at_20_max": 0.2901054578717061,
"nauc_recall_at_20_std": 0.12413182063867068,
"nauc_recall_at_3_diff1": 0.19680821832881057,
"nauc_recall_at_3_max": 0.20838567453412998,
"nauc_recall_at_3_std": -0.00682971262625349,
"nauc_recall_at_5_diff1": 0.12554959294600906,
"nauc_recall_at_5_max": 0.24624746390881122,
"nauc_recall_at_5_std": 0.04863875266602324,
"ndcg_at_1": 0.42043,
"ndcg_at_10": 0.56464,
"ndcg_at_100": 0.59861,
"ndcg_at_1000": 0.60804,
"ndcg_at_20": 0.57975,
"ndcg_at_3": 0.51325,
"ndcg_at_5": 0.54051,
"precision_at_1": 0.42043,
"precision_at_10": 0.08076,
"precision_at_100": 0.01007,
"precision_at_1000": 0.00109,
"precision_at_20": 0.04418,
"precision_at_3": 0.21615,
"precision_at_5": 0.14537,
"recall_at_1": 0.38836,
"recall_at_10": 0.71971,
"recall_at_100": 0.86659,
"recall_at_1000": 0.93666,
"recall_at_20": 0.77395,
"recall_at_3": 0.58591,
"recall_at_5": 0.64766
}
]
},
"task_name": "MrTidyRetrieval"
}
================================================
FILE: eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MultiLongDocRetrieval.json
================================================
{
"dataset_revision": "d67138e705d963e346253a80e59676ddb418810a",
"evaluation_time": 4641.1542456150055,
"kg_co2_emissions": null,
"mteb_version": "1.20.0",
"scores": {
"dev": [
{
"hf_subset": "ko",
"languages": [
"kor-Hang"
],
"main_score": 0.48402,
"map_at_1": 0.36,
"map_at_10": 0.44135,
"map_at_100": 0.45013,
"map_at_1000": 0.45073,
"map_at_20": 0.44591,
"map_at_3": 0.42167,
"map_at_5": 0.42967,
"mrr_at_1": 0.36,
"mrr_at_10": 0.4413511904761906,
"mrr_at_100": 0.450129543381387,
"mrr_at_1000": 0.45073324355439376,
"mrr_at_20": 0.44590807700211743,
"mrr_at_3": 0.4216666666666667,
"mrr_at_5": 0.4296666666666667,
"nauc_map_at_1000_diff1": 0.653999576097392,
"nauc_map_at_1000_max": 0.5915981156693542,
"nauc_map_at_1000_std": 0.04204665806066081,
"nauc_map_at_100_diff1": 0.6537378762654921,
"nauc_map_at_100_max": 0.591719062921236,
"nauc_map_at_100_std": 0.042690974962164625,
"nauc_map_at_10_diff1": 0.652369406457342,
"nauc_map_at_10_max": 0.5909253802982799,
"nauc_map_at_10_std": 0.036022144467421524,
"nauc_map_at_1_diff1": 0.7126436781609194,
"nauc_map_at_1_max": 0.5840418831680049,
"nauc_map_at_1_std": -0.021833831942476276,
"nauc_map_at_20_diff1": 0.6520302323881799,
"nauc_map_at_20_max": 0.5908525887352375,
"nauc_map_at_20_std": 0.04020584504644928,
"nauc_map_at_3_diff1": 0.6640690747768079,
"nauc_map_at_3_max": 0.5918378557726748,
"nauc_map_at_3_std": 0.027992687337536,
"nauc_map_at_5_diff1": 0.6522396910722045,
"nauc_map_at_5_max": 0.591985080836136,
"nauc_map_at_5_std": 0.040401896772712935,
"nauc_mrr_at_1000_diff1": 0.653999576097392,
"nauc_mrr_at_1000_max": 0.5915981156693542,
"nauc_mrr_at_1000_std": 0.04204665806066081,
"nauc_mrr_at_100_diff1": 0.6537378762654921,
"nauc_mrr_at_100_max": 0.591719062921236,
"nauc_mrr_at_100_std": 0.042690974962164625,
"nauc_mrr_at_10_diff1": 0.652369406457342,
"nauc_mrr_at_10_max": 0.5909253802982799,
"nauc_mrr_at_10_std": 0.036022144467421524,
"nauc_mrr_at_1_diff1": 0.7126436781609194,
"nauc_mrr_at_1_max": 0.5840418831680049,
"nauc_mrr_at_1_std": -0.021833831942476276,
"nauc_mrr_at_20_diff1": 0.6520302323881799,
"nauc_mrr_at_20_max": 0.5908525887352375,
"nauc_mrr_at_20_std": 0.04020584504644928,
"nauc_mrr_at_3_diff1": 0.6640690747768079,
"nauc_mrr_at_3_max": 0.5918378557726748,
"nauc_mrr_at_3_std": 0.027992687337536,
"nauc_mrr_at_5_diff1": 0.6522396910722045,
"nauc_mrr_at_5_max": 0.591985080836136,
"nauc_mrr_at_5_std": 0.040401896772712935,
"nauc_ndcg_at_1000_diff1": 0.6432566916474228,
"nauc_ndcg_at_1000_max": 0.5998470404188347,
"nauc_ndcg_at_1000_std": 0.07410684021566737,
"nauc_ndcg_at_100_diff1": 0.6360589988421134,
"nauc_ndcg_at_100_max": 0.6037341341497577,
"nauc_ndcg_at_100_std": 0.09785768793288292,
"nauc_ndcg_at_10_diff1": 0.6293196340250062,
"nauc_ndcg_at_10_max": 0.5982485402739869,
"nauc_ndcg_at_10_std": 0.06006582165943897,
"nauc_ndcg_at_1_diff1": 0.7126436781609194,
"nauc_ndcg_at_1_max": 0.5840418831680049,
"nauc_ndcg_at_1_std": -0.021833831942476276,
"nauc_ndcg_at_20_diff1": 0.6261813451342695,
"nauc_ndcg_at_20_max": 0.5970582271477827,
"nauc_ndcg_at_20_std": 0.07622456371047974,
"nauc_ndcg_at_3_diff1": 0.6506603473332017,
"nauc_ndcg_at_3_max": 0.6004543836287595,
"nauc_ndcg_at_3_std": 0.047787735554177944,
"nauc_ndcg_at_5_diff1": 0.6287749223122684,
"nauc_ndcg_at_5_max": 0.6005997287841836,
"nauc_ndcg_at_5_std": 0.07005047365688216,
"nauc_precision_at_1000_diff1": 0.8078120136943695,
"nauc_precision_at_1000_max": 1.0,
"nauc_precision_at_1000_std": 1.0,
"nauc_precision_at_100_diff1": 0.5412788805727321,
"nauc_precision_at_100_max": 0.7116986657988953,
"nauc_precision_at_100_std": 0.598714611129191,
"nauc_precision_at_10_diff1": 0.5518721348798572,
"nauc_precision_at_10_max": 0.6234424514660185,
"nauc_precision_at_10_std": 0.13918715195520429,
"nauc_precision_at_1_diff1": 0.7126436781609194,
"nauc_precision_at_1_max": 0.5840418831680049,
"nauc_precision_at_1_std": -0.021833831942476276,
"nauc_precision_at_20_diff1": 0.5220822971233046,
"nauc_precision_at_20_max": 0.6184366015838579,
"nauc_precision_at_20_std": 0.22843360318888078,
"nauc_precision_at_3_diff1": 0.6122251832111925,
"nauc_precision_at_3_max": 0.6275216522318455,
"nauc_precision_at_3_std": 0.10674217188541009,
"nauc_precision_at_5_diff1": 0.5556584283947462,
"nauc_precision_at_5_max": 0.6282966330640979,
"nauc_precision_at_5_std": 0.16318549585092312,
"nauc_recall_at_1000_diff1": 0.8078120136943671,
"nauc_recall_at_1000_max": 1.0,
"nauc_recall_at_1000_std": 1.0,
"nauc_recall_at_100_diff1": 0.5412788805727307,
"nauc_recall_at_100_max": 0.7116986657988936,
"nauc_recall_at_100_std": 0.5987146111291889,
"nauc_recall_at_10_diff1": 0.5518721348798571,
"nauc_recall_at_10_max": 0.6234424514660182,
"nauc_recall_at_10_std": 0.13918715195520406,
"nauc_recall_at_1_diff1": 0.7126436781609194,
"nauc_recall_at_1_max": 0.5840418831680049,
"nauc_recall_at_1_std": -0.021833831942476276,
"nauc_recall_at_20_diff1": 0.5220822971233048,
"nauc_recall_at_20_max": 0.6184366015838583,
"nauc_recall_at_20_std": 0.22843360318888178,
"nauc_recall_at_3_diff1": 0.6122251832111921,
"nauc_recall_at_3_max": 0.6275216522318453,
"nauc_recall_at_3_std": 0.10674217188540977,
"nauc_recall_at_5_diff1": 0.5556584283947461,
"nauc_recall_at_5_max": 0.6282966330640981,
"nauc_recall_at_5_std": 0.16318549585092318,
"ndcg_at_1": 0.36,
"ndcg_at_10": 0.48402,
"ndcg_at_100": 0.53127,
"ndcg_at_1000": 0.54818,
"ndcg_at_20": 0.50135,
"ndcg_at_3": 0.44178,
"ndcg_at_5": 0.4562,
"precision_at_1": 0.36,
"precision_at_10": 0.062,
"precision_at_100": 0.0085,
"precision_at_1000": 0.00099,
"precision_at_20": 0.0345,
"precision_at_3": 0.16667,
"precision_at_5": 0.107,
"recall_at_1": 0.36,
"recall_at_10": 0.62,
"recall_at_100": 0.85,
"recall_at_1000": 0.985,
"recall_at_20": 0.69,
"recall_at_3": 0.5,
"recall_at_5": 0.535
}
],
"test": [
{
"hf_subset": "ko",
"languages": [
"kor-Hang"
],
"main_score": 0.46733,
"map_at_1": 0.33,
"map_at_10": 0.42326,
"map_at_100": 0.43228,
"map_at_1000": 0.43308,
"map_at_20": 0.42878,
"map_at_3": 0.40333,
"map_at_5": 0.41608,
"mrr_at_1": 0.33,
"mrr_at_10": 0.4232619047619048,
"mrr_at_100": 0.432279471134081,
"mrr_at_1000": 0.4330762455314873,
"mrr_at_20": 0.4287762510107712,
"mrr_at_3": 0.40333333333333327,
"mrr_at_5": 0.4160833333333334,
"nauc_map_at_1000_diff1": 0.6409752471277966,
"nauc_map_at_1000_max": 0.5946002721149415,
"nauc_map_at_1000_std": 0.03492754217391834,
"nauc_map_at_100_diff1": 0.6405785551672617,
"nauc_map_at_100_max": 0.5947016987665,
"nauc_map_at_100_std": 0.03519683850581178,
"nauc_map_at_10_diff1": 0.6437977226295699,
"nauc_map_at_10_max": 0.5938053519033123,
"nauc_map_at_10_std": 0.031333520693003776,
"nauc_map_at_1_diff1": 0.7091105164356752,
"nauc_map_at_1_max": 0.6154957329818537,
"nauc_map_at_1_std": 0.014005937878606113,
"nauc_map_at_20_diff1": 0.6414751638548382,
"nauc_map_at_20_max": 0.5939793225373083,
"nauc_map_at_20_std": 0.03190726201177065,
"nauc_map_at_3_diff1": 0.6404104137741969,
"nauc_map_at_3_max": 0.5827813691071001,
"nauc_map_at_3_std": 0.008878591452145033,
"nauc_map_at_5_diff1": 0.6400465541544528,
"nauc_map_at_5_max": 0.5881739870325576,
"nauc_map_at_5_std": 0.021291218705272164,
"nauc_mrr_at_1000_diff1": 0.6409752471277966,
"nauc_mrr_at_1000_max": 0.5946002721149415,
"nauc_mrr_at_1000_std": 0.03492754217391834,
"nauc_mrr_at_100_diff1": 0.6405785551672617,
"nauc_mrr_at_100_max": 0.5947016987665,
"nauc_mrr_at_100_std": 0.03519683850581178,
"nauc_mrr_at_10_diff1": 0.6437977226295699,
"nauc_mrr_at_10_max": 0.5938053519033123,
"nauc_mrr_at_10_std": 0.031333520693003776,
"nauc_mrr_at_1_diff1": 0.7091105164356752,
"nauc_mrr_at_1_max": 0.6154957329818537,
"nauc_mrr_at_1_std": 0.014005937878606113,
"nauc_mrr_at_
gitextract_pyyh9iti/
├── .gitignore
├── LICENSE
├── README.md
├── README_EN.md
└── eval/
├── evaluate.py
├── leaderboard.py
├── requirements.txt
└── results/
├── Alibaba-NLP/
│ ├── gte-Qwen2-7B-instruct/
│ │ └── Alibaba-NLP__gte-Qwen2-7B-instruct/
│ │ └── e26182b2122f4435e8b3ebecbf363990f409b45b/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ └── gte-multilingual-base/
│ └── Alibaba-NLP__gte-multilingual-base/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── BAAI/
│ ├── bge-m3/
│ │ └── BAAI__bge-m3/
│ │ └── no_revision_available/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ └── bge-multilingual-gemma2/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── Salesforce/
│ └── SFR-Embedding-2_R/
│ └── Salesforce__SFR-Embedding-2_R/
│ └── 91762139d94ed4371a9fa31db5551272e0b83818/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── Snowflake/
│ └── snowflake-arctic-embed-l-v2.0/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── dragonkue/
│ └── BGE-m3-ko/
│ └── dragonkue__BGE-m3-ko/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── intfloat/
│ ├── e5-mistral-7b-instruct/
│ │ └── intfloat__e5-mistral-7b-instruct/
│ │ └── 07163b72af1488142a360786df853f237b1a3ca1/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ ├── multilingual-e5-base/
│ │ └── intfloat__multilingual-e5-base/
│ │ └── d13f1b27baf31030b7fd040960d60d909913633f/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ ├── multilingual-e5-large/
│ │ └── intfloat__multilingual-e5-large/
│ │ └── ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ └── multilingual-e5-large-instruct/
│ └── intfloat__multilingual-e5-large-instruct/
│ └── baa7be480a7de1539afce709c8f13f833a510e0a/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── jhgan/
│ └── ko-sroberta-multitask/
│ └── jhgan__ko-sroberta-multitask/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── jinaai/
│ └── jina-embeddings-v3/
│ └── jinaai__jina-embeddings-v3/
│ └── 215a6e121fa0183376388ac6b1ae230326bfeaed/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── nlpai-lab/
│ ├── KURE-v1/
│ │ └── nlpai-lab__KURE-v1/
│ │ └── no_revision_available/
│ │ ├── AutoRAGRetrieval.json
│ │ ├── BelebeleRetrieval.json
│ │ ├── Ko-StrategyQA.json
│ │ ├── MIRACLRetrieval.json
│ │ ├── MrTidyRetrieval.json
│ │ ├── MultiLongDocRetrieval.json
│ │ ├── PublicHealthQA.json
│ │ ├── XPQARetrieval.json
│ │ └── model_meta.json
│ └── KoE5/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── nomic-ai/
│ └── nomic-embed-text-v2-moe/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
├── openai/
│ └── text-embedding-3-large/
│ └── no_model_name_available/
│ └── no_revision_available/
│ ├── AutoRAGRetrieval.json
│ ├── BelebeleRetrieval.json
│ ├── Ko-StrategyQA.json
│ ├── MIRACLRetrieval.json
│ ├── MrTidyRetrieval.json
│ ├── MultiLongDocRetrieval.json
│ ├── PublicHealthQA.json
│ ├── XPQARetrieval.json
│ └── model_meta.json
└── upskyy/
└── bge-m3-korean/
└── upskyy__bge-m3-korean/
└── no_revision_available/
├── AutoRAGRetrieval.json
├── BelebeleRetrieval.json
├── Ko-StrategyQA.json
├── MIRACLRetrieval.json
├── MrTidyRetrieval.json
├── MultiLongDocRetrieval.json
├── PublicHealthQA.json
├── XPQARetrieval.json
└── model_meta.json
SYMBOL INDEX (2 symbols across 2 files) FILE: eval/evaluate.py function evaluate_model (line 103) | def evaluate_model(model_name, gpu_id, tasks): FILE: eval/leaderboard.py function app (line 9) | def app():
Condensed preview — 169 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,791K chars).
[
{
"path": ".gitignore",
"chars": 11,
"preview": "__pycache__"
},
{
"path": "LICENSE",
"chars": 1069,
"preview": "MIT License\n\nCopyright (c) [year] [fullname]\n\nPermission is hereby granted, free of charge, to any person obtaining a co"
},
{
"path": "README.md",
"chars": 15369,
"preview": "# 🔎 KURE: Korea University Retrieval Embedding model\n\n## Update Logs\n- 2024.12.21: [🤗 KURE-v1](https://huggingface.co/nl"
},
{
"path": "README_EN.md",
"chars": 15744,
"preview": "# 🔎 KURE: Korea University Retrieval Embedding model\n\n## Update Logs\n- 2024.12.21: [🤗 KURE-v1](https://huggingface.co/nl"
},
{
"path": "eval/evaluate.py",
"chars": 7912,
"preview": "\"\"\"Benchmarking all datasets constituting the MTEB Korean leaderboard & average scores\"\"\"\nfrom __future__ import annotat"
},
{
"path": "eval/leaderboard.py",
"chars": 5484,
"preview": "import streamlit as st\nimport os\nimport json\nimport pandas as pd\n\nst.set_page_config(layout=\"wide\")\n\n\ndef app():\n dat"
},
{
"path": "eval/requirements.txt",
"chars": 41,
"preview": "mteb\npython-dotenv\nstreamlit\nsetproctitle"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/AutoRAGRetrieval.json",
"chars": 7117,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 846.0815389156342,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/BelebeleRetrieval.json",
"chars": 20543,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 537.1708543300629,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/Ko-StrategyQA.json",
"chars": 7118,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 9451.070049762726,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MIRACLRetrieval.json",
"chars": 7128,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 529443.3452823162,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MrTidyRetrieval.json",
"chars": 7140,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 523189.4379520416,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/MultiLongDocRetrieval.json",
"chars": 14033,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 6735.936784029007,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/PublicHealthQA.json",
"chars": 6727,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 46.12740516662598,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/XPQARetrieval.json",
"chars": 21117,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 373.6595883369446,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-Qwen2-7B-instruct/Alibaba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/model_meta.json",
"chars": 588,
"preview": "{\"name\": \"Alibaba-NLP/gte-Qwen2-7B-instruct\", \"revision\": \"e26182b2122f4435e8b3ebecbf363990f409b45b\", \"release_date\": \"2"
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/AutoRAGRetrieval.json",
"chars": 6952,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 9.638917446136475,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/BelebeleRetrieval.json",
"chars": 20703,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 9.514682531356812,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/Ko-StrategyQA.json",
"chars": 7118,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 47.95084285736084,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MIRACLRetrieval.json",
"chars": 7123,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 4725.387804031372,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MrTidyRetrieval.json",
"chars": 7175,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 4789.847511768341,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/MultiLongDocRetrieval.json",
"chars": 13887,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 4641.1542456150055,\n \"kg_co2_e"
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/PublicHealthQA.json",
"chars": 6902,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 0.8287994861602783,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/XPQARetrieval.json",
"chars": 21082,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 8.458330631256104,\n \"kg_co2_em"
},
{
"path": "eval/results/Alibaba-NLP/gte-multilingual-base/Alibaba-NLP__gte-multilingual-base/no_revision_available/model_meta.json",
"chars": 462,
"preview": "{\"name\": \"Alibaba-NLP/gte-multilingual-base\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": []"
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/AutoRAGRetrieval.json",
"chars": 6847,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 50.766931772232056,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/BelebeleRetrieval.json",
"chars": 20666,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 42.1163535118103,\n \"kg_co2_emi"
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/Ko-StrategyQA.json",
"chars": 7112,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 185.89697933197021,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/MIRACLRetrieval.json",
"chars": 7145,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 25134.011864423752,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/MrTidyRetrieval.json",
"chars": 7159,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 15034.357434511185,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/MultiLongDocRetrieval.json",
"chars": 13917,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 11767.261229991913,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/PublicHealthQA.json",
"chars": 6915,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 3.1353759765625,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1.1"
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/XPQARetrieval.json",
"chars": 21097,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 14.820614337921143,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-m3/BAAI__bge-m3/no_revision_available/model_meta.json",
"chars": 440,
"preview": "{\"name\": \"BAAI/bge-m3\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": [], \"n_parameters\": null"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/AutoRAGRetrieval.json",
"chars": 7076,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 119.05822992324829,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/BelebeleRetrieval.json",
"chars": 20532,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 195.57231187820435,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/Ko-StrategyQA.json",
"chars": 7108,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 1484.8957543373108,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/MIRACLRetrieval.json",
"chars": 7140,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 226490.0695669651,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/MrTidyRetrieval.json",
"chars": 7178,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 225582.5098335743,\n \"kg_co2_em"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json",
"chars": 14021,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 1957.7613937854767,\n \"kg_co2_e"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/PublicHealthQA.json",
"chars": 6818,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 13.14871883392334,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/XPQARetrieval.json",
"chars": 21174,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 214.9201488494873,\n \"kg_co2_em"
},
{
"path": "eval/results/BAAI/bge-multilingual-gemma2/no_model_name_available/no_revision_available/model_meta.json",
"chars": 431,
"preview": "{\"name\": \"no_model_name_available\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": null, \"n_par"
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/AutoRAGRetrieval.json",
"chars": 7086,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 1549.6882767677307,\n \"kg_co2_e"
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/BelebeleRetrieval.json",
"chars": 20670,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 1465.829463481903,\n \"kg_co2_em"
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/Ko-StrategyQA.json",
"chars": 7112,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 26252.303451776505,\n \"kg_co2_e"
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MIRACLRetrieval.json",
"chars": 7118,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 1186423.9800457954,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MrTidyRetrieval.json",
"chars": 7152,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 1455631.1653485298,\n \"kg_co2_e"
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/MultiLongDocRetrieval.json",
"chars": 13983,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 12127.463791370392,\n \"kg_co2_e"
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/PublicHealthQA.json",
"chars": 6843,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 88.03694748878479,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/XPQARetrieval.json",
"chars": 21072,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 1152.1036405563354,\n \"kg_co2_e"
},
{
"path": "eval/results/Salesforce/SFR-Embedding-2_R/Salesforce__SFR-Embedding-2_R/91762139d94ed4371a9fa31db5551272e0b83818/model_meta.json",
"chars": 589,
"preview": "{\"name\": \"Salesforce/SFR-Embedding-2_R\", \"revision\": \"91762139d94ed4371a9fa31db5551272e0b83818\", \"release_date\": \"2024-0"
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/AutoRAGRetrieval.json",
"chars": 5889,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"task_name\": \"AutoRAGRetrieval\",\n \"mteb_version\":"
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/BelebeleRetrieval.json",
"chars": 17413,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"task_name\": \"BelebeleRetrieval\",\n \"mteb_version\""
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/Ko-StrategyQA.json",
"chars": 5963,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"task_name\": \"Ko-StrategyQA\",\n \"mteb_version\": \"1"
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/MIRACLRetrieval.json",
"chars": 5936,
"preview": "{\n \"dataset_revision\": \"main\",\n \"task_name\": \"MIRACLRetrieval\",\n \"mteb_version\": \"1.21.6\",\n \"scores\": {\n \"dev\": ["
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/MrTidyRetrieval.json",
"chars": 5984,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"task_name\": \"MrTidyRetrieval\",\n \"mteb_version\": "
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json",
"chars": 11628,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"task_name\": \"MultiLongDocRetrieval\",\n \"mteb_vers"
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/PublicHealthQA.json",
"chars": 5886,
"preview": "{\n \"dataset_revision\": \"main\",\n \"task_name\": \"PublicHealthQA\",\n \"mteb_version\": \"1.21.6\",\n \"scores\": {\n \"test\": ["
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/XPQARetrieval.json",
"chars": 17541,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"task_name\": \"XPQARetrieval\",\n \"mteb_version\": \"1"
},
{
"path": "eval/results/Snowflake/snowflake-arctic-embed-l-v2.0/no_model_name_available/no_revision_available/model_meta.json",
"chars": 476,
"preview": "{\"name\": \"no_model_name_available\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": null, \"n_par"
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/AutoRAGRetrieval.json",
"chars": 6834,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 22.999385833740234,\n \"kg_co2_e"
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/BelebeleRetrieval.json",
"chars": 20716,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 20.49944758415222,\n \"kg_co2_em"
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/Ko-StrategyQA.json",
"chars": 7118,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 115.5916862487793,\n \"kg_co2_em"
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/MIRACLRetrieval.json",
"chars": 7082,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 11312.0097489357,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1."
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/MrTidyRetrieval.json",
"chars": 7175,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 11472.845931529999,\n \"kg_co2_e"
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/MultiLongDocRetrieval.json",
"chars": 13975,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 11403.819789886475,\n \"kg_co2_e"
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/PublicHealthQA.json",
"chars": 6944,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 1.4347784519195557,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/XPQARetrieval.json",
"chars": 21086,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 15.096574306488037,\n \"kg_co2_e"
},
{
"path": "eval/results/dragonkue/BGE-m3-ko/dragonkue__BGE-m3-ko/no_revision_available/model_meta.json",
"chars": 448,
"preview": "{\"name\": \"dragonkue/BGE-m3-ko\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": [], \"n_parameter"
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/AutoRAGRetrieval.json",
"chars": 7070,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 151.6671278476715,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/BelebeleRetrieval.json",
"chars": 20616,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 151.1522355079651,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/Ko-StrategyQA.json",
"chars": 7105,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 1837.136996269226,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/MIRACLRetrieval.json",
"chars": 7114,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 133762.75083470345,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/MrTidyRetrieval.json",
"chars": 7168,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 132131.55453968048,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/MultiLongDocRetrieval.json",
"chars": 13979,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 1299.6901452541351,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/PublicHealthQA.json",
"chars": 6782,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 9.609801292419434,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/XPQARetrieval.json",
"chars": 21072,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 94.5943055152893,\n \"kg_co2_emi"
},
{
"path": "eval/results/intfloat/e5-mistral-7b-instruct/intfloat__e5-mistral-7b-instruct/07163b72af1488142a360786df853f237b1a3ca1/model_meta.json",
"chars": 619,
"preview": "{\"name\": \"intfloat/e5-mistral-7b-instruct\", \"revision\": \"07163b72af1488142a360786df853f237b1a3ca1\", \"release_date\": \"202"
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/AutoRAGRetrieval.json",
"chars": 7048,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 12.844457387924194,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/BelebeleRetrieval.json",
"chars": 20667,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 22.750174522399902,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/Ko-StrategyQA.json",
"chars": 7117,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 69.76440715789795,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/MIRACLRetrieval.json",
"chars": 7124,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 10329.316873311996,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/MrTidyRetrieval.json",
"chars": 7177,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 10726.250535488129,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/MultiLongDocRetrieval.json",
"chars": 13981,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 365.5563635826111,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/PublicHealthQA.json",
"chars": 6939,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 2.6252050399780273,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/XPQARetrieval.json",
"chars": 21101,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 16.628807067871094,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-base/intfloat__multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/model_meta.json",
"chars": 1766,
"preview": "{\"name\": \"intfloat/multilingual-e5-base\", \"revision\": \"d13f1b27baf31030b7fd040960d60d909913633f\", \"release_date\": \"2024-"
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/AutoRAGRetrieval.json",
"chars": 6948,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 18.56531572341919,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/BelebeleRetrieval.json",
"chars": 20671,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 45.867305755615234,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/Ko-StrategyQA.json",
"chars": 7119,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 103.04216527938843,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/MIRACLRetrieval.json",
"chars": 7134,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 22242.406965255737,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/MrTidyRetrieval.json",
"chars": 7173,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 23272.43692088127,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/MultiLongDocRetrieval.json",
"chars": 13951,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 785.9183006286621,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/PublicHealthQA.json",
"chars": 6908,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 5.381413459777832,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/XPQARetrieval.json",
"chars": 21102,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 32.079678773880005,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-large/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/model_meta.json",
"chars": 1769,
"preview": "{\"name\": \"intfloat/multilingual-e5-large\", \"revision\": \"ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb\", \"release_date\": \"2024"
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/AutoRAGRetrieval.json",
"chars": 6972,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 9.748866319656372,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/BelebeleRetrieval.json",
"chars": 20644,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 16.148106575012207,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/Ko-StrategyQA.json",
"chars": 7112,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 115.93965744972229,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MIRACLRetrieval.json",
"chars": 7123,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 15485.132284641266,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MrTidyRetrieval.json",
"chars": 7159,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 15758.073652267456,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/MultiLongDocRetrieval.json",
"chars": 14015,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 156.01362895965576,\n \"kg_co2_e"
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/PublicHealthQA.json",
"chars": 6934,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 1.2370071411132812,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/XPQARetrieval.json",
"chars": 21050,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 15.38088846206665,\n \"kg_co2_em"
},
{
"path": "eval/results/intfloat/multilingual-e5-large-instruct/intfloat__multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/model_meta.json",
"chars": 1760,
"preview": "{\"name\": \"intfloat/multilingual-e5-large-instruct\", \"revision\": \"baa7be480a7de1539afce709c8f13f833a510e0a\", \"release_dat"
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/AutoRAGRetrieval.json",
"chars": 7086,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 1.926253318786621,\n \"kg_co2_em"
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/BelebeleRetrieval.json",
"chars": 20723,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 14.881865978240967,\n \"kg_co2_e"
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/Ko-StrategyQA.json",
"chars": 7151,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 20.74926519393921,\n \"kg_co2_em"
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/MIRACLRetrieval.json",
"chars": 7152,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 5007.700176239014,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/MrTidyRetrieval.json",
"chars": 7182,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 5739.107554197311,\n \"kg_co2_em"
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/MultiLongDocRetrieval.json",
"chars": 14039,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 85.13093566894531,\n \"kg_co2_em"
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/PublicHealthQA.json",
"chars": 6944,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 0.5960137844085693,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/XPQARetrieval.json",
"chars": 21103,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 13.350108623504639,\n \"kg_co2_e"
},
{
"path": "eval/results/jhgan/ko-sroberta-multitask/jhgan__ko-sroberta-multitask/no_revision_available/model_meta.json",
"chars": 456,
"preview": "{\"name\": \"jhgan/ko-sroberta-multitask\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": [], \"n_p"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/AutoRAGRetrieval.json",
"chars": 6925,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 41.10066056251526,\n \"kg_co2_em"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/BelebeleRetrieval.json",
"chars": 20720,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 57.27869248390198,\n \"kg_co2_em"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/Ko-StrategyQA.json",
"chars": 7130,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 223.86375641822815,\n \"kg_co2_e"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MIRACLRetrieval.json",
"chars": 7134,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 25491.57020497322,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MrTidyRetrieval.json",
"chars": 7167,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 25669.12554335594,\n \"kg_co2_em"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/MultiLongDocRetrieval.json",
"chars": 14070,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 18277.223182439804,\n \"kg_co2_e"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/PublicHealthQA.json",
"chars": 6891,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 2.920142412185669,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/XPQARetrieval.json",
"chars": 21045,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 56.67747497558594,\n \"kg_co2_em"
},
{
"path": "eval/results/jinaai/jina-embeddings-v3/jinaai__jina-embeddings-v3/215a6e121fa0183376388ac6b1ae230326bfeaed/model_meta.json",
"chars": 1753,
"preview": "{\"name\": \"jinaai/jina-embeddings-v3\", \"revision\": \"215a6e121fa0183376388ac6b1ae230326bfeaed\", \"release_date\": \"2024-09-1"
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/AutoRAGRetrieval.json",
"chars": 5887,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"task_name\": \"AutoRAGRetrieval\",\n \"mteb_version\":"
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/BelebeleRetrieval.json",
"chars": 17437,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"task_name\": \"BelebeleRetrieval\",\n \"mteb_version\""
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/Ko-StrategyQA.json",
"chars": 5960,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"task_name\": \"Ko-StrategyQA\",\n \"mteb_version\": \"1"
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/MIRACLRetrieval.json",
"chars": 5923,
"preview": "{\n \"dataset_revision\": \"main\",\n \"task_name\": \"MIRACLRetrieval\",\n \"mteb_version\": \"1.21.6\",\n \"scores\": {\n \"dev\": ["
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/MrTidyRetrieval.json",
"chars": 5958,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"task_name\": \"MrTidyRetrieval\",\n \"mteb_version\": "
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/MultiLongDocRetrieval.json",
"chars": 11635,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"task_name\": \"MultiLongDocRetrieval\",\n \"mteb_vers"
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/PublicHealthQA.json",
"chars": 5889,
"preview": "{\n \"dataset_revision\": \"main\",\n \"task_name\": \"PublicHealthQA\",\n \"mteb_version\": \"1.21.6\",\n \"scores\": {\n \"test\": ["
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/XPQARetrieval.json",
"chars": 17533,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"task_name\": \"XPQARetrieval\",\n \"mteb_version\": \"1"
},
{
"path": "eval/results/nlpai-lab/KURE-v1/nlpai-lab__KURE-v1/no_revision_available/model_meta.json",
"chars": 505,
"preview": "{\"name\": \"nlpai-lab/KURE-v1\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": null, \"n_parameter"
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/AutoRAGRetrieval.json",
"chars": 6969,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 19.922383546829224,\n \"kg_co2_e"
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/BelebeleRetrieval.json",
"chars": 20673,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 26.43672013282776,\n \"kg_co2_em"
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/Ko-StrategyQA.json",
"chars": 7123,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 106.8914806842804,\n \"kg_co2_em"
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/MIRACLRetrieval.json",
"chars": 7131,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 10921.967575788498,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/MrTidyRetrieval.json",
"chars": 7158,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 11006.327818870544,\n \"kg_co2_e"
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json",
"chars": 14061,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 374.64185643196106,\n \"kg_co2_e"
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/PublicHealthQA.json",
"chars": 6923,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 2.5225961208343506,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/XPQARetrieval.json",
"chars": 21064,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 21.183428287506104,\n \"kg_co2_e"
},
{
"path": "eval/results/nlpai-lab/KoE5/no_model_name_available/no_revision_available/model_meta.json",
"chars": 431,
"preview": "{\"name\": \"no_model_name_available\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": null, \"n_par"
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/AutoRAGRetrieval.json",
"chars": 5924,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"task_name\": \"AutoRAGRetrieval\",\n \"mteb_version\":"
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/BelebeleRetrieval.json",
"chars": 17396,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"task_name\": \"BelebeleRetrieval\",\n \"mteb_version\""
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/Ko-StrategyQA.json",
"chars": 5969,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"task_name\": \"Ko-StrategyQA\",\n \"mteb_version\": \"1"
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/MIRACLRetrieval.json",
"chars": 5938,
"preview": "{\n \"dataset_revision\": \"main\",\n \"task_name\": \"MIRACLRetrieval\",\n \"mteb_version\": \"1.29.16\",\n \"scores\": {\n \"dev\": "
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/MrTidyRetrieval.json",
"chars": 5965,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"task_name\": \"MrTidyRetrieval\",\n \"mteb_version\": "
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json",
"chars": 11646,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"task_name\": \"MultiLongDocRetrieval\",\n \"mteb_vers"
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/PublicHealthQA.json",
"chars": 5854,
"preview": "{\n \"dataset_revision\": \"main\",\n \"task_name\": \"PublicHealthQA\",\n \"mteb_version\": \"1.29.16\",\n \"scores\": {\n \"test\": "
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/XPQARetrieval.json",
"chars": 17528,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"task_name\": \"XPQARetrieval\",\n \"mteb_version\": \"1"
},
{
"path": "eval/results/nomic-ai/nomic-embed-text-v2-moe/no_model_name_available/no_revision_available/model_meta.json",
"chars": 451,
"preview": "{\"name\": \"no_model_name_available\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": null, \"n_par"
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/AutoRAGRetrieval.json",
"chars": 7073,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 26.646844148635864,\n \"kg_co2_e"
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/BelebeleRetrieval.json",
"chars": 20582,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 104.64979100227356,\n \"kg_co2_e"
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/Ko-StrategyQA.json",
"chars": 7106,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 237.30947017669678,\n \"kg_co2_e"
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/MIRACLRetrieval.json",
"chars": 7122,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 33137.345321416855,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/MrTidyRetrieval.json",
"chars": 7144,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 33973.33324599266,\n \"kg_co2_em"
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/MultiLongDocRetrieval.json",
"chars": 14032,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 791.2342638969421,\n \"kg_co2_em"
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/PublicHealthQA.json",
"chars": 6830,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 5.753293514251709,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/XPQARetrieval.json",
"chars": 21052,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 127.63971376419067,\n \"kg_co2_e"
},
{
"path": "eval/results/openai/text-embedding-3-large/no_model_name_available/no_revision_available/model_meta.json",
"chars": 431,
"preview": "{\"name\": \"no_model_name_available\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": null, \"n_par"
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/AutoRAGRetrieval.json",
"chars": 6962,
"preview": "{\n \"dataset_revision\": \"fd7df84ac089bbec763b1c6bb1b56e985df5cc5c\",\n \"evaluation_time\": 19.277517318725586,\n \"kg_co2_e"
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/BelebeleRetrieval.json",
"chars": 20790,
"preview": "{\n \"dataset_revision\": \"75b399394a9803252cfec289d103de462763db7c\",\n \"evaluation_time\": 20.426774740219116,\n \"kg_co2_e"
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/Ko-StrategyQA.json",
"chars": 7143,
"preview": "{\n \"dataset_revision\": \"d243889a3eb6654029dbd7e7f9319ae31d58f97c\",\n \"evaluation_time\": 112.89276218414307,\n \"kg_co2_e"
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/MIRACLRetrieval.json",
"chars": 7133,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 11371.988452672958,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \""
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/MrTidyRetrieval.json",
"chars": 7174,
"preview": "{\n \"dataset_revision\": \"fc24a3ce8f09746410daee3d5cd823ff7a0675b7\",\n \"evaluation_time\": 11457.999158143997,\n \"kg_co2_e"
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/MultiLongDocRetrieval.json",
"chars": 13993,
"preview": "{\n \"dataset_revision\": \"d67138e705d963e346253a80e59676ddb418810a\",\n \"evaluation_time\": 411.78907680511475,\n \"kg_co2_e"
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/PublicHealthQA.json",
"chars": 6896,
"preview": "{\n \"dataset_revision\": \"main\",\n \"evaluation_time\": 1.244837760925293,\n \"kg_co2_emissions\": null,\n \"mteb_version\": \"1"
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/XPQARetrieval.json",
"chars": 21075,
"preview": "{\n \"dataset_revision\": \"c99d599f0a6ab9b85b065da6f9d94f9cf731679f\",\n \"evaluation_time\": 13.89358639717102,\n \"kg_co2_em"
},
{
"path": "eval/results/upskyy/bge-m3-korean/upskyy__bge-m3-korean/no_revision_available/model_meta.json",
"chars": 449,
"preview": "{\"name\": \"upskyy/bge-m3-korean\", \"revision\": \"no_revision_available\", \"release_date\": null, \"languages\": [], \"n_paramete"
}
]
About this extraction
This page contains the full source code of the nlpai-lab/KURE GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 169 files (1.6 MB), approximately 663.6k tokens, and a symbol index with 2 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.