Showing preview only (5,494K chars total). Download the full file or copy to clipboard to get everything.
Repository: nlpxucan/WizardLM
Branch: main
Commit: cd4b4cc022b6
Files: 55
Total size: 26.9 MB
Directory structure:
gitextract_mddl1sk4/
├── Evol_Instruct/
│ ├── breadth.py
│ ├── depth.py
│ ├── main.py
│ └── openai_access.py
├── README.md
├── WizardCoder/
│ ├── CODE_LICENSE
│ ├── DATA_LICENSE
│ ├── MODEL_WEIGHTS_LICENSE
│ ├── README.md
│ ├── data/
│ │ └── mbppplus.json
│ └── src/
│ ├── humaneval_gen.py
│ ├── humaneval_gen_vllm.py
│ ├── inference_wizardcoder.py
│ ├── mbpp_gen.py
│ ├── mbppplus_gen.py
│ ├── mbppplus_gen_vllm.py
│ ├── mbppplus_process_preds.py
│ ├── process_humaneval.py
│ ├── process_mbpp.py
│ └── train_wizardcoder.py
├── WizardLM/
│ ├── CODE_LICENSE
│ ├── DATA_LICENSE
│ ├── MODEL_DIFF_LICENSE
│ ├── README.md
│ ├── data/
│ │ └── WizardLM_testset.jsonl
│ ├── doc/
│ │ └── distributed_finetune.md
│ └── src/
│ ├── case_show.md
│ ├── infer_wizardlm13b.py
│ ├── inference_wizardlm.py
│ ├── train_freeform.py
│ └── weight_diff_wizard.py
├── WizardMath/
│ ├── LICENSE
│ ├── README.md
│ ├── config/
│ │ └── deepspeed_config.json
│ ├── data/
│ │ ├── MATH_test.jsonl
│ │ └── gsm8k_test.jsonl
│ ├── inference/
│ │ ├── MATH_inference.py
│ │ ├── grader.py
│ │ ├── gsm8k_inference.py
│ │ └── util.py
│ └── train/
│ └── train_wizardmath.py
├── demo/
│ ├── README.md
│ ├── wizardLM_demo.py
│ ├── wizardcoder_demo.py
│ └── wizardmath_demo.py
└── training/
├── data/
│ └── alpaca_data.json
├── requirements.txt
└── src/
├── configs/
│ ├── deepspeed_config.json
│ └── hostfile
├── conversation.py
├── environment.yml
├── generate.py
├── train.py
├── train_freeform_multiturn.py
└── utils.py
================================================
FILE CONTENTS
================================================
================================================
FILE: Evol_Instruct/breadth.py
================================================
base_instruction = "I want you act as a Prompt Creator.\r\n\
Your goal is to draw inspiration from the #Given Prompt# to create a brand new prompt.\r\n\
This new prompt should belong to the same domain as the #Given Prompt# but be even more rare.\r\n\
The LENGTH and complexity of the #Created Prompt# should be similar to that of the #Given Prompt#.\r\n\
The #Created Prompt# must be reasonable and must be understood and responded by humans.\r\n\
'#Given Prompt#', '#Created Prompt#', 'given prompt' and 'created prompt' are not allowed to appear in #Created Prompt#\r\n"
def createBreadthPrompt(instruction):
prompt = base_instruction
prompt += "#Given Prompt#: \r\n {} \r\n".format(instruction)
prompt += "#Created Prompt#:\r\n"
return prompt
================================================
FILE: Evol_Instruct/depth.py
================================================
base_instruction = "I want you act as a Prompt Rewriter.\r\n \
Your objective is to rewrite a given prompt into a more complex version to make those famous AI systems (e.g., chatgpt and GPT4) a bit harder to handle.\r\n \
But the rewritten prompt must be reasonable and must be understood and responded by humans.\r\n \
Your rewriting cannot omit the non-text parts such as the table and code in #The Given Prompt#:. Also, please do not omit the input in #The Given Prompt#. \r\n \
You SHOULD complicate the given prompt using the following method: \r\n\
{} \r\n\
You should try your best not to make the #Rewritten Prompt# become verbose, #Rewritten Prompt# can only add 10 to 20 words into #The Given Prompt#. \r\n\
'#The Given Prompt#', '#Rewritten Prompt#', 'given prompt' and 'rewritten prompt' are not allowed to appear in #Rewritten Prompt#\r\n"
def createConstraintsPrompt(instruction):
prompt = base_instruction.format("Please add one more constraints/requirements into #The Given Prompt#'")
prompt += "#The Given Prompt#: \r\n {} \r\n".format(instruction)
prompt += "#Rewritten Prompt#:\r\n"
return prompt
def createDeepenPrompt(instruction):
prompt = base_instruction.format("If #The Given Prompt# contains inquiries about certain issues, the depth and breadth of the inquiry can be increased.")
prompt += "#The Given Prompt#: \r\n {} \r\n".format(instruction)
prompt += "#Rewritten Prompt#:\r\n"
return prompt
def createConcretizingPrompt(instruction):
prompt = base_instruction.format("Please replace general concepts with more specific concepts.")
prompt += "#The Given Prompt#: \r\n {} \r\n".format(instruction)
prompt += "#Rewritten Prompt#:\r\n"
return prompt
def createReasoningPrompt(instruction):
prompt = base_instruction.format("If #The Given Prompt# can be solved with just a few simple thinking processes, you can rewrite it to explicitly request multiple-step reasoning.")
prompt += "#The Given Prompt#: \r\n {} \r\n".format(instruction)
prompt += "#Rewritten Prompt#:\r\n"
return prompt
================================================
FILE: Evol_Instruct/main.py
================================================
import json
import random
from openai_access import call_chatgpt
from depth import createConstraintsPrompt, createDeepenPrompt, createConcretizingPrompt, createReasoningPrompt
from breadth import createBreadthPrompt
fr = open('alpaca_data_cleaned.json','r')
all_objs = json.load(fr)
evol_objs = []
for cur_obj in all_objs:
instruction = cur_obj['instruction'].strip() + '\r\n'+ cur_obj['input'].strip()
evol_prompts = []
evol_prompts.append(createConstraintsPrompt(instruction))
evol_prompts.append(createDeepenPrompt(instruction))
evol_prompts.append(createConcretizingPrompt(instruction))
evol_prompts.append(createReasoningPrompt(instruction))
evol_prompts.append(createBreadthPrompt(instruction))
selected_evol_prompt = random.choice(evol_prompts)
evol_instruction = call_chatgpt(selected_evol_prompt)
answer = call_chatgpt(evol_instruction)
evol_objs.append({"instruction":evol_instruction,"output":answer})
with open('alpaca_data_evol.json', 'w') as f:
json.dump(evol_objs, f, indent=4)
================================================
FILE: Evol_Instruct/openai_access.py
================================================
import openai
import time
openai.api_key = 'your api key'
def get_oai_completion(prompt):
try:
response = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": prompt},
],
temperature=1,
max_tokens=2048,
top_p=0.95,
frequency_penalty=0,
presence_penalty=0,
stop=None
)
res = response["choices"][0]["message"]["content"]
gpt_output = res
return gpt_output
except requests.exceptions.Timeout:
# Handle the timeout error here
print("The OpenAI API request timed out. Please try again later.")
return None
except openai.error.InvalidRequestError as e:
# Handle the invalid request error here
print(f"The OpenAI API request was invalid: {e}")
return None
except openai.error.APIError as e:
if "The operation was timeout" in str(e):
# Handle the timeout error here
print("The OpenAI API request timed out. Please try again later.")
# time.sleep(3)
return get_oai_completion(prompt)
else:
# Handle other API errors here
print(f"The OpenAI API returned an error: {e}")
return None
except openai.error.RateLimitError as e:
return get_oai_completion(prompt)
def call_chatgpt(ins):
success = False
re_try_count = 15
ans = ''
while not success and re_try_count >= 0:
re_try_count -= 1
try:
ans = get_oai_completion(ins)
success = True
except:
time.sleep(5)
print('retry for sample:', ins)
return ans
================================================
FILE: README.md
================================================
## WizardLM: Empowering Large Pre-Trained Language Models to Follow Complex Instructions
<p style="font-size:50px;" align="center">
🏠 <a href="https://wizardlm.github.io/" target="_blank">Home Page</a> </p>
<p align="center">
<p align="center">
🤗 <a href="https://huggingface.co/WizardLMTeam" target="_blank">HF Repo</a> • 🐦 <a href="https://twitter.com/WizardLM_AI" target="_blank">Twitter</a> • 📃 <a href="https://arxiv.org/abs/2304.12244" target="_blank">[WizardLM] @ICLR2024</a> • 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder] @ICLR2024</a> • 📃 <a href="https://arxiv.org/abs/2308.09583" target="_blank">[WizardMath]</a> <br>
</p>
<p align="center">
👋 Join our <a href="https://discord.gg/VZjjHtWrKs" target="_blank">Discord</a>
</p>
<p align="center" width="100%">
<a ><img src="imgs/WizardLM.png" alt="WizardLM" style="width: 20%; min-width: 300px; display: block; margin: auto;"></a>
</p>
[](https://github.com/tatsu-lab/stanford_alpaca/blob/main/LICENSE)
[](https://github.com/tatsu-lab/stanford_alpaca/blob/main/DATA_LICENSE)
[](https://www.python.org/downloads/release/python-390/)
**Unofficial Video Introductions**
Thanks to the enthusiastic friends, their video introductions are more lively and interesting.
1. [NEW WizardLM 70b 🔥 Giant Model...Insane Performance](https://www.youtube.com/watch?v=WdpiIXrO4_o)
2. [GET WizardLM NOW! 7B LLM KING That Can Beat ChatGPT! I'm IMPRESSED!](https://www.youtube.com/watch?v=SaJ8wyKMBds)
3. [WizardLM: Enhancing Large Language Models to Follow Complex Instructions](https://www.youtube.com/watch?v=I6sER-qivYk)
4. [WizardCoder AI Is The NEW ChatGPT's Coding TWIN!](https://www.youtube.com/watch?v=XjsyHrmd3Xo)
## News
- 🔥🔥🔥[2024/01/04] We released **WizardCoder-33B-V1.1** trained from deepseek-coder-33b-base, the **SOTA OSS Code LLM** on [EvalPlus Leaderboard](https://evalplus.github.io/leaderboard.html), achieves **79.9 pass@1** on HumanEval, **73.2 pass@1** on HumanEval-Plus, **78.9 pass@1** on MBPP, and **66.9 pass@1** on MBPP-Plus. **WizardCoder-33B-V1.1** outperforms **ChatGPT 3.5**, **Gemini Pro**, and **DeepSeek-Coder-33B-instruct** on HumanEval and HumanEval-Plus pass@1. **WizardCoder-33B-V1.1** is comparable with **ChatGPT 3.5**, and surpasses **Gemini Pro** on MBPP and MBPP-Plus pass@1.
- [2023/08/26] We released **WizardCoder-Python-34B-V1.0** , which achieves the **73.2 pass@1** and surpasses **GPT4 (2023/03/15)**, **ChatGPT-3.5**, and **Claude2** on the [HumanEval Benchmarks](https://github.com/openai/human-eval). For more details, please refer to [WizardCoder](https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder).
- [2023/06/16] We released **WizardCoder-15B-V1.0** , which surpasses **Claude-Plus (+6.8)**, **Bard (+15.3)** and **InstructCodeT5+ (+22.3)** on the [HumanEval Benchmarks](https://github.com/openai/human-eval). For more details, please refer to [WizardCoder](https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder).
| Model | Checkpoint | Paper | HumanEval | HumanEval+ | MBPP | MBPP+ |
| ----- |------| ---- |------|-------| ----- | ----- |
| GPT-4-Turbo (Nov 2023) | - | - | 85.4 | 81.7 | 83.0 | 70.7 |
| GPT-4 (May 2023) | - | - | 88.4 | 76.8 | - | - |
| GPT-3.5-Turbo (Nov 2023) | - | - | 72.6 | 65.9 | 81.7 | 69.4 |
| Gemini Pro | - | - | 63.4 | 55.5 | 72.9 | 57.9 |
| DeepSeek-Coder-33B-instruct | - | - | 78.7 | 72.6 | 78.7 | 66.7 |
| WizardCoder-33B-V1.1 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-33B-V1.1" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 79.9 | 73.2 | 78.9 | 66.9 |
| WizardCoder-Python-34B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 73.2 | 64.6 | 73.2 | 59.9 |
| WizardCoder-15B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-15B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 59.8 | 52.4 | -- | -- |
| WizardCoder-Python-13B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-Python-13B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 64.0 | -- | -- | -- |
| WizardCoder-Python-7B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-Python-7B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 55.5 | -- | -- | -- |
| WizardCoder-3B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-3B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 34.8 | -- | -- | -- |
| WizardCoder-1B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-1B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 23.8 | -- | -- | -- |
- [12/19/2023] 🔥 We released **WizardMath-7B-V1.1** trained from Mistral-7B, the **SOTA 7B math LLM**, achieves **83.2 pass@1** on GSM8k, and **33.0 pass@1** on MATH.
- [12/19/2023] 🔥 **WizardMath-7B-V1.1** outperforms **ChatGPT 3.5**, **Gemini Pro**, **Mixtral MOE**, and **Claude Instant** on GSM8K pass@1.
- [12/19/2023] 🔥 **WizardMath-7B-V1.1** is comparable with **ChatGPT 3.5**, **Gemini Pro**, and surpasses **Mixtral MOE** on MATH pass@1.
- 🔥 Our **WizardMath-70B-V1.0** model slightly outperforms some closed-source LLMs on the GSM8K, including **ChatGPT 3.5**, **Claude Instant 1** and **PaLM 2 540B**.
- 🔥 Our **WizardMath-70B-V1.0** model achieves **81.6 pass@1** on the [GSM8k Benchmarks](https://github.com/openai/grade-school-math), which is **24.8** points higher than the SOTA open-source LLM.
- 🔥 Our **WizardMath-70B-V1.0** model achieves **22.7 pass@1** on the [MATH Benchmarks](https://github.com/hendrycks/math), which is **9.2** points higher than the SOTA open-source LLM.
| Model | Checkpoint | Paper | GSM8k | MATH |
| ----- |------| ---- |------|-------|
| **WizardMath-7B-V1.1** | 🤗 <a href="https://huggingface.co/WizardLM/WizardMath-7B-V1.1" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2308.09583" target="_blank">[WizardMath]</a>| **83.2** | **33.0** |
| WizardMath-70B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardMath-70B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2308.09583" target="_blank">[WizardMath]</a>| **81.6** | **22.7** |
| WizardMath-13B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardMath-13B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2308.09583" target="_blank">[WizardMath]</a>| **63.9** | **14.0** |
| WizardMath-7B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardMath-7B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2308.09583" target="_blank">[WizardMath]</a>| **54.9** | **10.7** |
- [08/09/2023] We released **WizardLM-70B-V1.0** model. Here is [Full Model Weight](https://huggingface.co/WizardLM/WizardLM-70B-V1.0).
<font size=0.5>
| <sup>Model</sup> | <sup>Checkpoint</sup> | <sup>Paper</sup> |<sup>MT-Bench</sup> | <sup>AlpacaEval</sup> | <sup>GSM8k</sup> | <sup>HumanEval</sup> | <sup>Demo</sup> | <sup>License</sup>|
| ----- |------| ---- |------|-------| ----- | ----- | ----- | ----- |
| <sup>**WizardLM-70B-V1.0**</sup> | <sup>🤗 <a href="https://huggingface.co/WizardLM/WizardLM-70B-V1.0" target="_blank">HF Link</a> </sup>|<sup>📃**Coming Soon**</sup>| <sup>**7.78**</sup> | <sup>**92.91%**</sup> |<sup>**77.6%**</sup> | <sup> **50.6**</sup>| |<sup> <a href="https://ai.meta.com/resources/models-and-libraries/llama-downloads/" target="_blank">Llama 2 License </a></sup> |
| <sup>WizardLM-13B-V1.2</sup> | <sup>🤗 <a href="https://huggingface.co/WizardLM/WizardLM-13B-V1.2" target="_blank">HF Link</a> </sup>| | <sup>7.06</sup> | <sup>89.17%</sup> |<sup>55.3%</sup> | <sup>36.6 </sup>| [Demo](http://47.103.63.15:50087/) |<sup> <a href="https://ai.meta.com/resources/models-and-libraries/llama-downloads/" target="_blank">Llama 2 License </a></sup> |
| <sup>WizardLM-13B-V1.1</sup> |<sup> 🤗 <a href="https://huggingface.co/WizardLM/WizardLM-13B-V1.1" target="_blank">HF Link</a> </sup> | | <sup>6.76</sup> |<sup>86.32%</sup> | | <sup>25.0 </sup>| | <sup>Non-commercial</sup>|
| <sup>WizardLM-30B-V1.0</sup> | <sup>🤗 <a href="https://huggingface.co/WizardLM/WizardLM-30B-V1.0" target="_blank">HF Link</a></sup> | | <sup>7.01</sup> | | | <sup>37.8 </sup>| | <sup>Non-commercial</sup> |
| <sup>WizardLM-13B-V1.0</sup> | <sup>🤗 <a href="https://huggingface.co/WizardLM/WizardLM-13B-V1.0" target="_blank">HF Link</a> </sup> | | <sup>6.35</sup> | <sup>75.31%</sup> | | <sup> 24.0 </sup> | | <sup>Non-commercial</sup>|
| <sup>WizardLM-7B-V1.0 </sup>| <sup>🤗 <a href="https://huggingface.co/WizardLM/WizardLM-7B-V1.0" target="_blank">HF Link</a> </sup> |<sup> 📃 <a href="https://arxiv.org/abs/2304.12244" target="_blank">[WizardLM]</a> </sup>| | | |<sup>19.1 </sup>| | <sup> Non-commercial</sup>|
</font>
### Citation
Please cite the paper if you use the data or code from WizardLM.
```
@inproceedings{
xu2024wizardlm,
title={Wizard{LM}: Empowering Large Pre-Trained Language Models to Follow Complex Instructions},
author={Can Xu and Qingfeng Sun and Kai Zheng and Xiubo Geng and Pu Zhao and Jiazhan Feng and Chongyang Tao and Qingwei Lin and Daxin Jiang},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
url={https://openreview.net/forum?id=CfXh93NDgH}
}
```
Please cite the paper if you use the data or code from WizardCoder.
```
@inproceedings{
luo2024wizardcoder,
title={WizardCoder: Empowering Code Large Language Models with Evol-Instruct},
author={Ziyang Luo and Can Xu and Pu Zhao and Qingfeng Sun and Xiubo Geng and Wenxiang Hu and Chongyang Tao and Jing Ma and Qingwei Lin and Daxin Jiang},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
url={https://openreview.net/forum?id=UnUwSIgK5W}
}
```
Please cite the paper if you refer to our model or code or data or paper from WizardMath.
```
@article{luo2023wizardmath,
title={Wizardmath: Empowering mathematical reasoning for large language models via reinforced evol-instruct},
author={Luo, Haipeng and Sun, Qingfeng and Xu, Can and Zhao, Pu and Lou, Jianguang and Tao, Chongyang and Geng, Xiubo and Lin, Qingwei and Chen, Shifeng and Zhang, Dongmei},
journal={arXiv preprint arXiv:2308.09583},
year={2023}
}
```
❗To commen concern about dataset:
Recently, there have been clear changes in the open-source policy and regulations of our overall organization's code, data, and models.
Despite this, we have still worked hard to obtain opening the weights of the model first, but the data involves stricter auditing and is in review with our legal team .
Our researchers have no authority to publicly release them without authorization.
Thank you for your understanding.
## Hiring
- 📣 We are looking for highly motivated students to join us as interns to create more intelligent AI together. Please contact caxu@microsoft.com
<!-- Although on our **complexity-balanced test set**, **WizardLM-7B has more cases that are preferred by human labelers than ChatGPT** in the high-complexity instructions (difficulty level >= 8), it still lags behind ChatGPT on the entire test set, and we also consider WizardLM to still be in a **baby state**. This repository will **continue to improve WizardLM**, train on larger scales, add more training data, and innovate more advanced large-model training methods. -->
<b>Note for model system prompts usage:</b>
To obtain results **identical to our demo**, please strictly follow the prompts and invocation methods provided in the **"src/infer_wizardlm13b.py"** to use our model for inference. Our model adopts the prompt format from <b>Vicuna</b> and supports **multi-turn** conversation.
<b>For WizardLM</b>, the Prompt should be as following:
```
A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: Hi ASSISTANT: Hello.</s>USER: Who are you? ASSISTANT: I am WizardLM.</s>......
```
<b>For WizardCoder </b>, the Prompt should be as following:
```
"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:"
```
<b>For WizardMath</b>, the Prompts should be as following:
**Default version:**
```
"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:"
```
**CoT Version:** （❗For the **simple** math questions, we do NOT recommend to use the CoT prompt.）
```
"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response: Let's think step by step."
```
### GPT-4 automatic evaluation
We adopt the automatic evaluation framework based on GPT-4 proposed by FastChat to assess the performance of chatbot models. As shown in the following figure, WizardLM-30B achieved better results than Guanaco-65B.
<p align="center" width="100%">
<a ><img src="imgs/WizarLM30b-GPT4.png" alt="WizardLM" style="width: 100%; min-width: 300px; display: block; margin: auto;"></a>
</p>
### WizardLM-30B performance on different skills.
The following figure compares WizardLM-30B and ChatGPT’s skill on Evol-Instruct testset. The result indicates that WizardLM-30B achieves 97.8% of ChatGPT’s performance on average, with almost 100% (or more than) capacity on 18 skills, and more than 90% capacity on 24 skills.
<p align="center" width="100%">
<a ><img src="imgs/evol-testset_skills-30b.png" alt="WizardLM" style="width: 100%; min-width: 300px; display: block; margin: auto;"></a>
</p>
### WizardLM performance on NLP foundation tasks.
The following table provides a comparison of WizardLMs and other LLMs on NLP foundation tasks. The results indicate that WizardLMs consistently exhibit superior performance in comparison to the LLaMa models of the same size. Furthermore, our WizardLM-30B model showcases comparable performance to OpenAI's Text-davinci-003 on the MMLU and HellaSwag benchmarks.
| Model | MMLU 5-shot | ARC 25-shot | TruthfulQA 0-shot | HellaSwag 10-shot | Average |
|------------------|-------------|-------------|-------------------|-------------------|------------|
| Text-davinci-003 | <u>56.9<u/> | **85.2** | **59.3** | <u>82.2<u/> | **70.9** |
|Vicuna-13b 1.1 | 51.3 | 53.0 | 51.8 | 80.1 | 59.1 |
|Guanaco 30B | 57.6 | 63.7 | 50.7 | **85.1** | 64.3 |
| WizardLM-7B 1.0 | 42.7 | 51.6 | 44.7 | 77.7 | 54.2 |
| WizardLM-13B 1.0 | 52.3 | 57.2 | 50.5 | 81.0 | 60.2 |
| WizardLM-30B 1.0 | **58.8** | <u>62.5<u/> | <u>52.4<u/> | 83.3 | <u>64.2<u/>|
### WizardLM performance on code generation.
The following table provides a comprehensive comparison of WizardLMs and several other LLMs on the code generation task, namely HumanEval. The evaluation metric is pass@1. The results indicate that WizardLMs consistently exhibit superior performance in comparison to the LLaMa models of the same size. Furthermore, our WizardLM-30B model surpasses StarCoder and OpenAI's code-cushman-001. Moreover, our Code LLM, WizardCoder, demonstrates exceptional performance, achieving a pass@1 score of 57.3, surpassing the open-source SOTA by approximately 20 points.
| Model | HumanEval Pass@1 |
|------------------|------------------|
| LLaMA-7B | 10.5 |
| LLaMA-13B | 15.8 |
| CodeGen-16B-Multi| 18.3 |
| CodeGeeX | 22.9 |
| LLaMA-33B | 21.7 |
| LLaMA-65B | 23.7 |
| PaLM-540B | 26.2 |
| CodeGen-16B-Mono | 29.3 |
| code-cushman-001 | 33.5 |
| StarCoder | <u>33.6<u/> |
| WizardLM-7B 1.0 | 19.1 |
| WizardLM-13B 1.0 | 24.0 |
| WizardLM-30B 1.0 | **37.8** |
| WizardCoder-15B 1.0 | **57.3** |
## Call for Feedbacks
We welcome everyone to use your professional and difficult instructions to evaluate WizardLM, and show us examples of poor performance and your suggestions in the [issue discussion](https://github.com/nlpxucan/WizardLM/issues) area. We are focusing on improving the Evol-Instruct now and hope to relieve existing weaknesses and issues in the the next version of WizardLM. After that, we will open the code and pipeline of up-to-date Evol-Instruct algorithm and work with you together to improve it.
## Overview of Evol-Instruct
[Evol-Instruct](https://github.com/nlpxucan/evol-instruct) is a novel method using LLMs instead of humans to automatically mass-produce open-domain instructions of various difficulty levels and skills range, to improve the performance of LLMs. You can easily embark on your own evolutionary journey with the [Evol Script](https://github.com/nlpxucan/WizardLM/tree/main/Evol-Instruct) we provide.
<p align="center" width="100%">
<a ><img src="imgs/git_overall.png" alt="WizardLM" style="width: 86%; min-width: 300px; display: block; margin: auto;"></a>
</p>
<p align="center" width="100%">
<a ><img src="imgs/git_running.png" alt="WizardLM" style="width: 86%; min-width: 300px; display: block; margin: auto;"></a>
</p>
## Disclaimer
The resources, including code, data, and model weights, associated with this project are restricted for academic research purposes only and cannot be used for commercial purposes. The content produced by any version of WizardLM is influenced by uncontrollable variables such as randomness, and therefore, the accuracy of the output cannot be guaranteed by this project. This project does not accept any legal liability for the content of the model output, nor does it assume responsibility for any losses incurred due to the use of associated resources and output results.
## Star History
[](https://star-history.com/#nlpxucan/WizardLM&Timeline)
================================================
FILE: WizardCoder/CODE_LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: WizardCoder/DATA_LICENSE
================================================
Attribution-NonCommercial 4.0 International
=======================================================================
Creative Commons Corporation ("Creative Commons") is not a law firm and
does not provide legal services or legal advice. Distribution of
Creative Commons public licenses does not create a lawyer-client or
other relationship. Creative Commons makes its licenses and related
information available on an "as-is" basis. Creative Commons gives no
warranties regarding its licenses, any material licensed under their
terms and conditions, or any related information. Creative Commons
disclaims all liability for damages resulting from their use to the
fullest extent possible.
Using Creative Commons Public Licenses
Creative Commons public licenses provide a standard set of terms and
conditions that creators and other rights holders may use to share
original works of authorship and other material subject to copyright
and certain other rights specified in the public license below. The
following considerations are for informational purposes only, are not
exhaustive, and do not form part of our licenses.
Considerations for licensors: Our public licenses are
intended for use by those authorized to give the public
permission to use material in ways otherwise restricted by
copyright and certain other rights. Our licenses are
irrevocable. Licensors should read and understand the terms
and conditions of the license they choose before applying it.
Licensors should also secure all rights necessary before
applying our licenses so that the public can reuse the
material as expected. Licensors should clearly mark any
material not subject to the license. This includes other CC-
licensed material, or material used under an exception or
limitation to copyright. More considerations for licensors:
wiki.creativecommons.org/Considerations_for_licensors
Considerations for the public: By using one of our public
licenses, a licensor grants the public permission to use the
licensed material under specified terms and conditions. If
the licensor's permission is not necessary for any reason--for
example, because of any applicable exception or limitation to
copyright--then that use is not regulated by the license. Our
licenses grant only permissions under copyright and certain
other rights that a licensor has authority to grant. Use of
the licensed material may still be restricted for other
reasons, including because others have copyright or other
rights in the material. A licensor may make special requests,
such as asking that all changes be marked or described.
Although not required by our licenses, you are encouraged to
respect those requests where reasonable. More considerations
for the public:
wiki.creativecommons.org/Considerations_for_licensees
=======================================================================
Creative Commons Attribution-NonCommercial 4.0 International Public
License
By exercising the Licensed Rights (defined below), You accept and agree
to be bound by the terms and conditions of this Creative Commons
Attribution-NonCommercial 4.0 International Public License ("Public
License"). To the extent this Public License may be interpreted as a
contract, You are granted the Licensed Rights in consideration of Your
acceptance of these terms and conditions, and the Licensor grants You
such rights in consideration of benefits the Licensor receives from
making the Licensed Material available under these terms and
conditions.
Section 1 -- Definitions.
a. Adapted Material means material subject to Copyright and Similar
Rights that is derived from or based upon the Licensed Material
and in which the Licensed Material is translated, altered,
arranged, transformed, or otherwise modified in a manner requiring
permission under the Copyright and Similar Rights held by the
Licensor. For purposes of this Public License, where the Licensed
Material is a musical work, performance, or sound recording,
Adapted Material is always produced where the Licensed Material is
synched in timed relation with a moving image.
b. Adapter's License means the license You apply to Your Copyright
and Similar Rights in Your contributions to Adapted Material in
accordance with the terms and conditions of this Public License.
c. Copyright and Similar Rights means copyright and/or similar rights
closely related to copyright including, without limitation,
performance, broadcast, sound recording, and Sui Generis Database
Rights, without regard to how the rights are labeled or
categorized. For purposes of this Public License, the rights
specified in Section 2(b)(1)-(2) are not Copyright and Similar
Rights.
d. Effective Technological Measures means those measures that, in the
absence of proper authority, may not be circumvented under laws
fulfilling obligations under Article 11 of the WIPO Copyright
Treaty adopted on December 20, 1996, and/or similar international
agreements.
e. Exceptions and Limitations means fair use, fair dealing, and/or
any other exception or limitation to Copyright and Similar Rights
that applies to Your use of the Licensed Material.
f. Licensed Material means the artistic or literary work, database,
or other material to which the Licensor applied this Public
License.
g. Licensed Rights means the rights granted to You subject to the
terms and conditions of this Public License, which are limited to
all Copyright and Similar Rights that apply to Your use of the
Licensed Material and that the Licensor has authority to license.
h. Licensor means the individual(s) or entity(ies) granting rights
under this Public License.
i. NonCommercial means not primarily intended for or directed towards
commercial advantage or monetary compensation. For purposes of
this Public License, the exchange of the Licensed Material for
other material subject to Copyright and Similar Rights by digital
file-sharing or similar means is NonCommercial provided there is
no payment of monetary compensation in connection with the
exchange.
j. Share means to provide material to the public by any means or
process that requires permission under the Licensed Rights, such
as reproduction, public display, public performance, distribution,
dissemination, communication, or importation, and to make material
available to the public including in ways that members of the
public may access the material from a place and at a time
individually chosen by them.
k. Sui Generis Database Rights means rights other than copyright
resulting from Directive 96/9/EC of the European Parliament and of
the Council of 11 March 1996 on the legal protection of databases,
as amended and/or succeeded, as well as other essentially
equivalent rights anywhere in the world.
l. You means the individual or entity exercising the Licensed Rights
under this Public License. Your has a corresponding meaning.
Section 2 -- Scope.
a. License grant.
1. Subject to the terms and conditions of this Public License,
the Licensor hereby grants You a worldwide, royalty-free,
non-sublicensable, non-exclusive, irrevocable license to
exercise the Licensed Rights in the Licensed Material to:
a. reproduce and Share the Licensed Material, in whole or
in part, for NonCommercial purposes only; and
b. produce, reproduce, and Share Adapted Material for
NonCommercial purposes only.
2. Exceptions and Limitations. For the avoidance of doubt, where
Exceptions and Limitations apply to Your use, this Public
License does not apply, and You do not need to comply with
its terms and conditions.
3. Term. The term of this Public License is specified in Section
6(a).
4. Media and formats; technical modifications allowed. The
Licensor authorizes You to exercise the Licensed Rights in
all media and formats whether now known or hereafter created,
and to make technical modifications necessary to do so. The
Licensor waives and/or agrees not to assert any right or
authority to forbid You from making technical modifications
necessary to exercise the Licensed Rights, including
technical modifications necessary to circumvent Effective
Technological Measures. For purposes of this Public License,
simply making modifications authorized by this Section 2(a)
(4) never produces Adapted Material.
5. Downstream recipients.
a. Offer from the Licensor -- Licensed Material. Every
recipient of the Licensed Material automatically
receives an offer from the Licensor to exercise the
Licensed Rights under the terms and conditions of this
Public License.
b. No downstream restrictions. You may not offer or impose
any additional or different terms or conditions on, or
apply any Effective Technological Measures to, the
Licensed Material if doing so restricts exercise of the
Licensed Rights by any recipient of the Licensed
Material.
6. No endorsement. Nothing in this Public License constitutes or
may be construed as permission to assert or imply that You
are, or that Your use of the Licensed Material is, connected
with, or sponsored, endorsed, or granted official status by,
the Licensor or others designated to receive attribution as
provided in Section 3(a)(1)(A)(i).
b. Other rights.
1. Moral rights, such as the right of integrity, are not
licensed under this Public License, nor are publicity,
privacy, and/or other similar personality rights; however, to
the extent possible, the Licensor waives and/or agrees not to
assert any such rights held by the Licensor to the limited
extent necessary to allow You to exercise the Licensed
Rights, but not otherwise.
2. Patent and trademark rights are not licensed under this
Public License.
3. To the extent possible, the Licensor waives any right to
collect royalties from You for the exercise of the Licensed
Rights, whether directly or through a collecting society
under any voluntary or waivable statutory or compulsory
licensing scheme. In all other cases the Licensor expressly
reserves any right to collect such royalties, including when
the Licensed Material is used other than for NonCommercial
purposes.
Section 3 -- License Conditions.
Your exercise of the Licensed Rights is expressly made subject to the
following conditions.
a. Attribution.
1. If You Share the Licensed Material (including in modified
form), You must:
a. retain the following if it is supplied by the Licensor
with the Licensed Material:
i. identification of the creator(s) of the Licensed
Material and any others designated to receive
attribution, in any reasonable manner requested by
the Licensor (including by pseudonym if
designated);
ii. a copyright notice;
iii. a notice that refers to this Public License;
iv. a notice that refers to the disclaimer of
warranties;
v. a URI or hyperlink to the Licensed Material to the
extent reasonably practicable;
b. indicate if You modified the Licensed Material and
retain an indication of any previous modifications; and
c. indicate the Licensed Material is licensed under this
Public License, and include the text of, or the URI or
hyperlink to, this Public License.
2. You may satisfy the conditions in Section 3(a)(1) in any
reasonable manner based on the medium, means, and context in
which You Share the Licensed Material. For example, it may be
reasonable to satisfy the conditions by providing a URI or
hyperlink to a resource that includes the required
information.
3. If requested by the Licensor, You must remove any of the
information required by Section 3(a)(1)(A) to the extent
reasonably practicable.
4. If You Share Adapted Material You produce, the Adapter's
License You apply must not prevent recipients of the Adapted
Material from complying with this Public License.
Section 4 -- Sui Generis Database Rights.
Where the Licensed Rights include Sui Generis Database Rights that
apply to Your use of the Licensed Material:
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
to extract, reuse, reproduce, and Share all or a substantial
portion of the contents of the database for NonCommercial purposes
only;
b. if You include all or a substantial portion of the database
contents in a database in which You have Sui Generis Database
Rights, then the database in which You have Sui Generis Database
Rights (but not its individual contents) is Adapted Material; and
c. You must comply with the conditions in Section 3(a) if You Share
all or a substantial portion of the contents of the database.
For the avoidance of doubt, this Section 4 supplements and does not
replace Your obligations under this Public License where the Licensed
Rights include other Copyright and Similar Rights.
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
c. The disclaimer of warranties and limitation of liability provided
above shall be interpreted in a manner that, to the extent
possible, most closely approximates an absolute disclaimer and
waiver of all liability.
Section 6 -- Term and Termination.
a. This Public License applies for the term of the Copyright and
Similar Rights licensed here. However, if You fail to comply with
this Public License, then Your rights under this Public License
terminate automatically.
b. Where Your right to use the Licensed Material has terminated under
Section 6(a), it reinstates:
1. automatically as of the date the violation is cured, provided
it is cured within 30 days of Your discovery of the
violation; or
2. upon express reinstatement by the Licensor.
For the avoidance of doubt, this Section 6(b) does not affect any
right the Licensor may have to seek remedies for Your violations
of this Public License.
c. For the avoidance of doubt, the Licensor may also offer the
Licensed Material under separate terms or conditions or stop
distributing the Licensed Material at any time; however, doing so
will not terminate this Public License.
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
License.
Section 7 -- Other Terms and Conditions.
a. The Licensor shall not be bound by any additional or different
terms or conditions communicated by You unless expressly agreed.
b. Any arrangements, understandings, or agreements regarding the
Licensed Material not stated herein are separate from and
independent of the terms and conditions of this Public License.
Section 8 -- Interpretation.
a. For the avoidance of doubt, this Public License does not, and
shall not be interpreted to, reduce, limit, restrict, or impose
conditions on any use of the Licensed Material that could lawfully
be made without permission under this Public License.
b. To the extent possible, if any provision of this Public License is
deemed unenforceable, it shall be automatically reformed to the
minimum extent necessary to make it enforceable. If the provision
cannot be reformed, it shall be severed from this Public License
without affecting the enforceability of the remaining terms and
conditions.
c. No term or condition of this Public License will be waived and no
failure to comply consented to unless expressly agreed to by the
Licensor.
d. Nothing in this Public License constitutes or may be interpreted
as a limitation upon, or waiver of, any privileges and immunities
that apply to the Licensor or You, including from the legal
processes of any jurisdiction or authority.
=======================================================================
Creative Commons is not a party to its public
licenses. Notwithstanding, Creative Commons may elect to apply one of
its public licenses to material it publishes and in those instances
will be considered the “Licensor.” The text of the Creative Commons
public licenses is dedicated to the public domain under the CC0 Public
Domain Dedication. Except for the limited purpose of indicating that
material is shared under a Creative Commons public license or as
otherwise permitted by the Creative Commons policies published at
creativecommons.org/policies, Creative Commons does not authorize the
use of the trademark "Creative Commons" or any other trademark or logo
of Creative Commons without its prior written consent including,
without limitation, in connection with any unauthorized modifications
to any of its public licenses or any other arrangements,
understandings, or agreements concerning use of licensed material. For
the avoidance of doubt, this paragraph does not form part of the
public licenses.
Creative Commons may be contacted at creativecommons.org.
================================================
FILE: WizardCoder/MODEL_WEIGHTS_LICENSE
================================================
BigCode Open RAIL-M v1 License Agreement
Section I: Preamble
This OpenRAIL-M License Agreement was created under BigCode, an open and collaborative research project aimed at the responsible development and Use of Large Language Models (“LLMs”) for code generation. This license is generally applicable to any machine-learning Model.
This License Agreement strives for both the open and responsible Use of the accompanying Model. Openness here is understood as enabling users of the Model on a royalty free basis to Use it, modify it, and even share commercial versions of it. Use restrictions are included to prevent misuse of the Model.
This License Agreement governs the Use of the Model and Modifications of the Model. You and Licensor agree as follows:
1.Definitions
a. “Contribution” means any work of authorship, including the original version of the Model and any Modifications of the Model that is intentionally submitted to Licensor for inclusion in the Model by the copyright owner or by an individual or entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Model, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as “Not a Contribution.”
b. “Contributor” means Licensor and any individual or entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Model.
c. “Data” means a collection of information extracted from the dataset used with the Model, including to train, pretrain, or otherwise evaluate the Model. The Data is not licensed under this License Agreement.
d. “Explanatory Documentation” means model cards, data cards, or any other similar documentation or related information dedicated to inform the public about the characteristics of the model. Explanatory documentation is not licensed under this license.
e. “Harm” includes but is not limited to physical, mental, psychological, financial and reputational damage, pain, or loss.
f. “License Agreement” means this document.
g. “Licensor” means the rights owners or entity authorized by the rights owners that are granting the terms and conditions of this License Agreement.
h. “Model” means machine-learning based assemblies (including checkpoints), consisting of learnt weights and parameters (including optimizer states), corresponding to a model architecture as embodied in source code. Source code is not licensed under this License Agreement.
i. “Modifications of the Model” means all changes to the Model or any other model which is created or initialized by transfer of patterns of the weights, parameters, activations or Output of the Model.
j. “Output” means the results of operating the Model.
k. “Share” means any transmission, reproduction, publication or other sharing of the Model or Modifications of the Model to a third party, including providing the Model as a hosted service made available by electronic or other remote means, including - but not limited to - API-based or web access.
l. “Third Parties” means individuals or legal entities that are not under common control with Licensor or You.
m. “Use” includes - but is not limited to - generating any Output, fine tuning, updating, running, training, evaluating and/or reparametrizing the Model.
n. “You” (or “Your”) means an individual or Legal Entity exercising permissions granted by this License Agreement and/or making Use of the Model for whichever purpose and in any field of Use.
Section II: INTELLECTUAL PROPERTY RIGHTS
The Model and Modifications of the Model are subject to additional terms as described in Section III, which shall govern the Use of the Model and Modifications of the Model.
Grant of Copyright license. Subject to the terms and conditions of this License Agreement and where and as applicable, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, copyright license to reproduce, prepare, publicly display, publicly perform, sublicense under the terms herein, and distribute the Model and Modifications of the Model.
Grant of Patent license. Subject to the terms and conditions of this License Agreement and where and as applicable, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free patent license to make, have made, Use, offer to sell, sell, import, and otherwise transfer the Model, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Model to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Model or a Contribution incorporated within the Model constitutes direct or contributory patent infringement, then any rights granted to You under this License Agreement for the Model shall terminate as of the date such litigation is filed.
Section III: CONDITIONS OF USE
4. Use conditions. Compliance with the restrictions in Attachment A is a condition to the grants in this License Agreement. If You Use the Model, You agree not to Use it for the specified restricted uses set forth in Attachment A.
5. Sharing of the Model
5.1. You may Share the Model or Modifications of the Model under any license of your choice that does not contradict the restrictions in Attachment A of this License Agreement and includes:
a. Paragraph 4 and the restrictions in Attachment A of this License Agreement, or,
b. Use conditions similar to Paragraph 4 that must accomplish the same purpose as the use conditions in Paragraph 4 and a similar set of restrictions to those in Attachment A that must accomplish the same purpose as the restrictions in Attachment A.
5.2. When You Share the Model or Modifications of the Model, You agree to:
a. Give any recipients a copy of this License Agreement;
b. Retain all Explanatory Documentation; and if sharing Modifications of the Model, add Explanatory Documentation of the same or better quality documenting the changes made to create the Modifications of the Model; and
c. Retain all copyright, patent, trademark, and attribution notices.
6. The Output You Generate. Licensor claims no rights in the Output. You agree not to contravene any provision as stated in the License Agreement with your Use of the Output.
Section IV: OTHER PROVISIONS
7. Updates and Runtime Restrictions. Licensor reserves the right to restrict (remotely or otherwise) usage of the Model in violation of this License Agreement.
8. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Model by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
9. Trademarks and related. Nothing in this License Agreement permits You to make Use of Licensors’ trademarks, trade names, logos or to otherwise suggest endorsement or misrepresent the relationship between the parties; and any rights not expressly granted herein are reserved by the Licensors.
10. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Model (and each Contributor provides its Contributions) on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or sharing the Model and Modifications of the Model, and assume any risks associated with Your exercise of permissions under this License Agreement.
11. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License Agreement or out of the Use or inability to Use the Model (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, model failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
12. Accepting Warranty or Additional Liability. While sharing the Model or Modifications of the Model thereof, You may choose to offer and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License Agreement. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
13. This License Agreement is a license of copyright and patent rights and an agreement in contract between You and the Licensor. If any provision of this License Agreement is held to be invalid, illegal or unenforceable, the remaining provisions shall be unaffected thereby and remain valid as if such provision had not been set forth herein.
END OF TERMS AND CONDITIONS
Attachment A - USE RESTRICTIONS
You agree not to Use the Model or Modifications of the Model:
(a) In any way that violates any applicable national, federal, state, local or international law or regulation;
(b) For the purpose of exploiting, Harming or attempting to exploit or harm minors in any way;
(c) To generate and/or disseminate malware (including - but not limited to - ransomware) or any other content to be used for the purpose of Harming electronic systems;
(d) To generate or disseminate verifiably false information and/or content with the purpose of Harming others;
(e) To generate or disseminate personal identifiable information with the purpose of Harming others;
(f) To generate or disseminate information (including - but not limited to - images, code, posts, articles), and place the information in any public context (including - but not limited to - bot generating tweets) without expressly and intelligibly disclaiming that the information and/or content is machine generated;
(g) To intentionally defame, disparage or otherwise harass others;
(h) To impersonate or attempt to impersonate human beings for purposes of deception;
(i) For fully automated decision making that adversely impacts an individual’s legal rights or otherwise creates or modifies a binding, enforceable obligation without expressly and intelligibly disclaiming that the creation or modification of the obligation is machine generated;
(j) For any Use intended to discriminate against or Harm individuals or groups based on online or offline social behavior or known or predicted personal or personality characteristics;
(k) To intentionally exploit any of the vulnerabilities of a specific group of persons based on their age, social, physical or mental characteristics, in order to materially distort the behavior of a person pertaining to that group in a manner that causes or is likely to cause that person or another person physical or psychological harm;
(l) For any Use intended to discriminate against individuals or groups based on legally protected characteristics or categories;
(m) To provide medical advice or medical results interpretation that is intended to be a substitute for professional medical advice, diagnosis, or treatment;
(n) For fully automated decision making in administration of justice, law enforcement, immigration or asylum processes.
================================================
FILE: WizardCoder/README.md
================================================
# WizardCoder: Empowering Code Large Language Models with Evol-Instruct
[](CODE_LICENSE)
[](DATA_LICENSE)
<!-- [](MODEL_WEIGHTS_LICENSE) -->
[](https://www.python.org/downloads/release/python-390/)
To develop our WizardCoder model, we begin by adapting the Evol-Instruct method specifically for coding tasks. This involves tailoring the prompt to the domain of code-related instructions. Subsequently, we fine-tune the Code LLMs, StarCoder or Code LLama, utilizing the newly created instruction-following training set.
## News
- 🔥🔥🔥[2024/01/04] We released **WizardCoder-33B-V1.1** trained from deepseek-coder-33b-base, the **SOTA OSS Code LLM** on [EvalPlus Leaderboard](https://evalplus.github.io/leaderboard.html), achieves **79.9 pass@1** on HumanEval, **73.2 pass@1** on HumanEval-Plus, **78.9 pass@1** on MBPP, and **66.9 pass@1** on MBPP-Plus. **WizardCoder-33B-V1.1** outperforms **ChatGPT 3.5**, **Gemini Pro**, and **DeepSeek-Coder-33B-instruct** on HumanEval and HumanEval-Plus pass@1. **WizardCoder-33B-V1.1** is comparable with **ChatGPT 3.5**, and surpasses **Gemini Pro** on MBPP and MBPP-Plus pass@1.
- [2023/08/26] We released **WizardCoder-Python-34B-V1.0** , which achieves the **73.2 pass@1** and surpasses **GPT4 (2023/03/15)**, **ChatGPT-3.5**, and **Claude2** on the [HumanEval Benchmarks](https://github.com/openai/human-eval).
- [2023/06/16] We released **WizardCoder-15B-V1.0** , which achieves the **57.3 pass@1** and surpasses **Claude-Plus (+6.8)**, **Bard (+15.3)** and **InstructCodeT5+ (+22.3)** on the [HumanEval Benchmarks](https://github.com/openai/human-eval).
❗❗❗**This performance is 100% reproducible!**
❗Note: There are two HumanEval results of GPT4 and ChatGPT-3.5. The 67.0 and 48.1 are reported by the official GPT4 Report (2023/03/15) of [OpenAI](https://arxiv.org/abs/2303.08774). The 82.0 and 72.5 are tested by ourselves with the latest API (2023/08/26).
| Model | Checkpoint | Paper | HumanEval | HumanEval+ | MBPP | MBPP+ |
| ----- |------| ---- |------|-------| ----- | ----- |
| GPT-4-Turbo (Nov 2023) | - | - | 85.4 | 81.7 | 83.0 | 70.7 |
| GPT-4 (May 2023) | - | - | 88.4 | 76.8 | - | - |
| GPT-3.5-Turbo (Nov 2023) | - | - | 72.6 | 65.9 | 81.7 | 69.4 |
| Gemini Pro | - | - | 63.4 | 55.5 | 72.9 | 57.9 |
| DeepSeek-Coder-33B-instruct | - | - | 78.7 | 72.6 | 78.7 | 66.7 |
| WizardCoder-33B-V1.1 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-33B-V1.1" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 79.9 | 73.2 | 78.9 | 66.9 |
| WizardCoder-Python-34B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 73.2 | 64.6 | 73.2 | 59.9 |
| WizardCoder-15B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-15B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 59.8 | 52.4 | -- | -- |
| WizardCoder-Python-13B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-Python-13B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 64.0 | -- | -- | -- |
| WizardCoder-Python-7B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-Python-7B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 55.5 | -- | -- | -- |
| WizardCoder-3B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-3B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 34.8 | -- | -- | -- |
| WizardCoder-1B-V1.0 | 🤗 <a href="https://huggingface.co/WizardLM/WizardCoder-1B-V1.0" target="_blank">HF Link</a> | 📃 <a href="https://arxiv.org/abs/2306.08568" target="_blank">[WizardCoder]</a> | 23.8 | -- | -- | -- |
- 📣 Please refer to our Twitter account https://twitter.com/WizardLM_AI and HuggingFace Repo https://huggingface.co/WizardLM . We will use them to announce any new release at the 1st time.
## Comparing WizardCoder-Python-34B-V1.0 with Other LLMs.
🔥 The following figure shows that our **WizardCoder-Python-34B-V1.0 attains the second position in this benchmark**, surpassing GPT4 (2023/03/15, 73.2 vs. 67.0), ChatGPT-3.5 (73.2 vs. 72.5) and Claude2 (73.2 vs. 71.2).
<p align="center" width="100%">
<a ><img src="imgs/compare_sota.png" alt="WizardCoder" style="width: 96%; min-width: 300px; display: block; margin: auto;"></a>
</p>
❗❗❗**Note: This performance is 100% reproducible! If you cannot reproduce it, please follow the steps in [Evaluation](#evaluation).**
❗Note: There are two HumanEval results of GPT4 and ChatGPT-3.5. The 67.0 and 48.1 are reported by the official GPT4 Report (2023/03/15) of [OpenAI](https://arxiv.org/abs/2303.08774). The 82.0 and 72.5 are tested by ourselves with the latest API (2023/08/26).
## Comparing WizardCoder-15B-V1.0 with the Closed-Source Models.
🔥 The following figure shows that our **WizardCoder attains the third position in this benchmark**, surpassing Claude-Plus (59.8 vs. 53.0) and Bard (59.8 vs. 44.5). Notably, our model exhibits a substantially smaller size compared to these models.
<p align="center" width="100%">
<a ><img src="imgs/pass1.png" alt="WizardCoder" style="width: 86%; min-width: 300px; display: block; margin: auto;"></a>
</p>
❗❗❗**Note: This performance is 100% reproducible! If you cannot reproduce it, please follow the steps in [Evaluation](#evaluation).**
❗**Note: In this study, we copy the scores for HumanEval and HumanEval+ from the [LLM-Humaneval-Benchmarks](https://github.com/my-other-github-account/llm-humaneval-benchmarks). Notably, all the mentioned models generate code solutions for each problem utilizing a **single attempt**, and the resulting pass rate percentage is reported. Our **WizardCoder** generates answers using greedy decoding and tests with the same [code](https://github.com/evalplus/evalplus).**
## Comparing WizardCoder-15B-V1.0 with the Open-Source Models.
The following table clearly demonstrates that our **WizardCoder** exhibits a substantial performance advantage over all the open-source models. ❗**If you are confused with the different scores of our model (57.3 and 59.8), please check the Notes.**
| Model | HumanEval Pass@1 | MBPP Pass@1 |
|------------------|------------------|-------------|
| CodeGen-16B-Multi| 18.3 |20.9 |
| CodeGeeX | 22.9 |24.4 |
| LLaMA-33B | 21.7 |30.2 |
| LLaMA-65B | 23.7 |37.7 |
| PaLM-540B | 26.2 |36.8 |
| PaLM-Coder-540B | 36.0 |47.0 |
| PaLM 2-S | 37.6 |50.0 |
| CodeGen-16B-Mono | 29.3 |35.3 |
| Code-Cushman-001 | 33.5 |45.9 |
| StarCoder-15B | 33.6 |43.6* |
| InstructCodeT5+ | 35.0 |-- |
| WizardLM-30B 1.0| 37.8 |-- |
| WizardCoder-15B 1.0 | **57.3** |**51.8** |
❗**Note: The reproduced result of StarCoder on MBPP.**
❗**Note: The above table conducts a comprehensive comparison of our **WizardCoder** with other models on the HumanEval and MBPP benchmarks. We adhere to the approach outlined in previous studies by generating **20 samples** for each problem to estimate the pass@1 score and evaluate with the same [code](https://github.com/openai/human-eval/tree/master). The scores of GPT4 and GPT3.5 reported by [OpenAI](https://openai.com/research/gpt-4) are 67.0 and 48.1 (maybe these are the early version GPT4&3.5).**
## Call for Feedbacks
We welcome everyone to use your professional and difficult instructions to evaluate WizardCoder, and show us examples of poor performance and your suggestions in the [issue discussion](https://github.com/nlpxucan/WizardLM/issues) area. We are focusing on improving the Evol-Instruct now and hope to relieve existing weaknesses and issues in the the next version of WizardCoder. After that, we will open the code and pipeline of up-to-date Evol-Instruct algorithm and work with you together to improve it.
## Unofficial Video Introductions
Thanks to the enthusiastic friends, their video introductions are more lively and interesting.
1. [WizardCoder AI Is The NEW ChatGPT's Coding TWIN!](https://www.youtube.com/watch?v=XjsyHrmd3Xo)
## Contents
1. [Online Demo](#online-demo)
2. [Fine-tuning](#fine-tuning)
3. [Inference](#inference)
4. [Evaluation](#evaluation)
5. [Citation](#citation)
6. [Disclaimer](#disclaimer)
## Online Demo
We will provide our latest models for you to try for as long as possible. If you find a link is not working, please try another one. At the same time, please try as many **real-world** and **challenging** code-related problems that you encounter in your work and life as possible. We will continue to evolve our models with your feedbacks.
[Demo Link](https://e5eaf7d09cc1521c.gradio.app/) (We adopt the greedy decoding now.)
## Fine-tuning
We fine-tune WizardCoder using the modified code `train.py` from [Llama-X](https://github.com/AetherCortex/Llama-X).
We fine-tune StarCoder-15B with the following hyperparameters:
| Hyperparameter | StarCoder-15B |
|----------------|---------------|
| Batch size | 512 |
| Learning rate | 2e-5 |
| Epochs | 3 |
| Max length | 2048 |
| Warmup step | 30 |
| LR scheduler | cosine |
To reproduce our fine-tuning of WizardCoder, please follow the following steps:
1. According to the instructions of [Llama-X](https://github.com/AetherCortex/Llama-X), install the environment, download the training code, and deploy. (Note: `deepspeed==0.9.2` and `transformers==4.29.2`)
2. Replace the `train.py` with the `train_wizardcoder.py` in our repo (`src/train_wizardcoder.py`)
3. Login Huggingface:
```bash
huggingface-cli login
```
4. Execute the following training command:
```bash
deepspeed train_wizardcoder.py \
--model_name_or_path "bigcode/starcoder" \
--data_path "/your/path/to/code_instruction_data.json" \
--output_dir "/your/path/to/ckpt" \
--num_train_epochs 3 \
--model_max_length 2048 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 1 \
--gradient_accumulation_steps 4 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50 \
--save_total_limit 2 \
--learning_rate 2e-5 \
--warmup_steps 30 \
--logging_steps 2 \
--lr_scheduler_type "cosine" \
--report_to "tensorboard" \
--gradient_checkpointing True \
--deepspeed configs/deepspeed_config.json \
--fp16 True
```
## Inference
We provide the decoding script for WizardCoder, which reads a input file and generates corresponding responses for each sample, and finally consolidates them into an output file.
You can specify `base_model`, `input_data_path` and `output_data_path` in `src\inference_wizardcoder.py` to set the decoding model, path of input file and path of output file.
```bash
pip install jsonlines
```
The decoding command is:
```
python src\inference_wizardcoder.py \
--base_model "/your/path/to/ckpt" \
--input_data_path "/your/path/to/input/data.jsonl" \
--output_data_path "/your/path/to/output/result.jsonl"
```
The format of `data.jsonl` should be:
```
{"idx": 11, "Instruction": "Write a Python code to count 1 to 10."}
{"idx": 12, "Instruction": "Write a Java code to sum 1 to 10."}
```
The prompt for our WizardCoder in `src\inference_wizardcoder.py` is:
```
Below is an instruction that describes a task. Write a response that appropriately completes the request.
### Instruction:
{instruction}
### Response:
```
## Evaluation
### HumanEval
1. According to the instructions of [HumanEval](https://github.com/openai/human-eval), install the environment.
2. Run the following scripts to generate the answer.
- (1) For WizardCoder-15B-V1.0 (base on StarCoder)
```bash
model="/path/to/your/model"
temp=0.2
max_len=2048
pred_num=200
num_seqs_per_iter=2
output_path=preds/T${temp}_N${pred_num}
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
# 164 problems, 21 per GPU if GPU=8
index=0
gpu_num=8
for ((i = 0; i < $gpu_num; i++)); do
start_index=$((i * 21))
end_index=$(((i + 1) * 21))
gpu=$((i))
echo 'Running process #' ${i} 'from' $start_index 'to' $end_index 'on GPU' ${gpu}
((index++))
(
CUDA_VISIBLE_DEVICES=$gpu python humaneval_gen.py --model ${model} \
--start_index ${start_index} --end_index ${end_index} --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path}
) &
if (($index % $gpu_num == 0)); then wait; fi
done
```
- (2) For WizardCoder-Python-34B-V1.0 (base on CodeLLama)
```bash
pip install vllm # This can acclerate the inference process a lot.
pip install transformers==4.31.0
model="/path/to/your/model"
temp=0.2
max_len=2048
pred_num=200
num_seqs_per_iter=2
output_path=preds/T${temp}_N${pred_num}
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
CUDA_VISIBLE_DEVICES=0,1,2,3 python humaneval_gen_vllm.py --model ${model} \
--start_index 0 --end_index 164 --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} --num_gpus 4
```
3. Run the post processing code `src/process_humaneval.py` to collect the code completions from all answer files.
```bash
output_path=preds/T${temp}_N${pred_num}
echo 'Output path: '$output_path
python process_humaneval.py --path ${output_path} --out_path ${output_path}.jsonl --add_prompt
evaluate_functional_correctness ${output_path}.jsonl
```
### How to Reproduce the Humaneval(Plus)/MBPP(Plus) Performance of WizardCoder-33B-v1.1?
❗❗❗**This performance is 100% reproducible!**
We also provide all generated results in `WizardLM/WizardCoder/data/humaneval_mbpp_wizardcoder33b_v1.1_results.zip`
```
transformers==4.36.2
vllm==0.2.5
```
(1) HumanEval and HumanEval-Plus
- Step 1
Code Generation (w/o accelerate)
```bash
model="WizardLM/WizardCoder-33B-V1.1"
temp=0.0
max_len=2048
pred_num=1
num_seqs_per_iter=1
output_path=preds/T${temp}_N${pred_num}_WizardCoder-33B-V1.1_Greedy_Decode
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
# 164 problems, 21 per GPU if GPU=8
index=0
gpu_num=8
for ((i = 0; i < $gpu_num; i++)); do
start_index=$((i * 21))
end_index=$(((i + 1) * 21))
gpu=$((i))
echo 'Running process #' ${i} 'from' $start_index 'to' $end_index 'on GPU' ${gpu}
((index++))
(
CUDA_VISIBLE_DEVICES=$gpu python humaneval_gen.py --model ${model} \
--start_index ${start_index} --end_index ${end_index} --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} --greedy_decode
) &
if (($index % $gpu_num == 0)); then wait; fi
done
```
Code Generation (w/ vllm accelerate)
```bash
model="WizardLM/WizardCoder-33B-V1.1"
temp=0.0
max_len=2048
pred_num=1
num_seqs_per_iter=1
output_path=preds/T${temp}_N${pred_num}_WizardCoder-33B-V1.1_Greedy_Decode_vllm
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
CUDA_VISIBLE_DEVICES=0,1,2,3 python humaneval_gen_vllm.py --model ${model} \
--start_index 0 --end_index 164 --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} --num_gpus 4 --overwrite
```
- Step 2: Get the score
Install [Eval-Plus](https://github.com/evalplus/evalplus) benchmark.
```bash
git clone https://github.com/evalplus/evalplus.git
cd evalplus
export PYTHONPATH=$PYTHONPATH:$(pwd)
pip install -r requirements.txt
```
Get HumanEval and HumanEval-Plus scores.
```bash
output_path=preds/T0.0_N1_WizardCoder-33B-V1.1_Greedy_Decode
echo 'Output path: '$output_path
python process_humaneval.py --path ${output_path} --out_path ${output_path}.jsonl --add_prompt
evalplus.evaluate --dataset humaneval --samples ${output_path}.jsonl
```
(2) MBPP and MBPP-Plus
The preprocessed questions are provided in `WizardLM/WizardCoder/data/mbppplus.json`
- Step 1
Code Generation (w/o accelerate)
```bash
model="WizardLM/WizardCoder-33B-V1.1"
temp=0.0
max_len=2048
pred_num=1
num_seqs_per_iter=1
output_path=preds/MBPP_T${temp}_N${pred_num}_WizardCoder-33B-V1.1_Greedy_Decode
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
# 399 problems, 50 per GPU if GPU=8
index=0
gpu_num=8
for ((i = 0; i < $gpu_num; i++)); do
start_index=$((i * 50))
end_index=$(((i + 1) * 50))
gpu=$((i))
echo 'Running process #' ${i} 'from' $start_index 'to' $end_index 'on GPU' ${gpu}
((index++))
(
CUDA_VISIBLE_DEVICES=$gpu python mbppplus_gen.py --model ${model} \
--start_index ${start_index} --end_index ${end_index} --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} --mbpp_path "mbppplus.json" --greedy_decode
) &
if (($index % $gpu_num == 0)); then wait; fi
done
```
Code Generation (w/ vllm accelerate)
```bash
model="WizardLM/WizardCoder-33B-V1.1"
temp=0.0
max_len=2048
pred_num=1
num_seqs_per_iter=1
output_path=preds/MBPP_T${temp}_N${pred_num}_WizardCoder-33B-V1.1_Greedy_Decode_vllm
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
CUDA_VISIBLE_DEVICES=0,1,2,3 python mbppplus_gen_vllm.py --model ${model} \
--start_index ${start_index} --end_index ${end_index} --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} --mbpp_path "mbppplus.json" --num_gpus 4
```
- Step 2: Get the score
Install [Eval-Plus](https://github.com/evalplus/evalplus) benchmark.
```bash
git clone https://github.com/evalplus/evalplus.git
cd evalplus
export PYTHONPATH=$PYTHONPATH:$(pwd)
pip install -r requirements.txt
```
Get HumanEval and HumanEval-Plus scores.
```bash
output_path=preds/MBPP_T0.0_N1_WizardCoder-33B-V1.1_Greedy_Decode
echo 'Output path: '$output_path
python mbppplus_process_preds.py --path ${output_path} --out_path ${output_path}.jsonl --add_prompt
evalplus.evaluate --dataset mbpp --samples ${output_path}.jsonl
```
### How to Reproduce the 73.2 Pass@1 on HumanEval with Greedy Decoding?
❗❗❗**This performance is 100% reproducible!**
- Step 1: Setup the environment
```bash
conda create -n eval python=3.10
conda activate eval
conda install pytorch==1.12.0 torchvision==0.13.0 torchaudio==0.12.0 cudatoolkit=11.3 -c pytorch
```
- Step 2: Install the packages
```
transformers==4.31.0
numpy
fire
sentencepiece
deepspeed==0.10.0
accelerate
vllm==0.1.4
pandas
ray
pyarrow
```
- Step 3: Install Human-Eval from OpenAI
```bash
git clone https://github.com/openai/human-eval.git
pip install -e human-eval
```
uncomment the execution call in `human-eval/human_eval/execution.py`
- Step 4: Generate Answer
use the code `WizardLM/blob/main/WizardCoder/src/humaneval_gen_vllm.py` to generate answer.
```bash
model="WizardLM/WizardCoder-Python-34B-V1.0"
temp=0.0
max_len=2048
pred_num=1
num_seqs_per_iter=1
output_path=preds/T${temp}_N${pred_num}
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
CUDA_VISIBLE_DEVICES=0,1,2,3 python humaneval_gen_vllm.py --model ${model} \
--start_index 0 --end_index 164 --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} --num_gpus 4
```
- Step 5: Get the score
use the code `WizardLM/blob/main/WizardCoder/src/process_humaneval.py` to get the score.
```bash
output_path=preds/T0.0_N1
echo 'Output path: '$output_path
python process_humaneval.py --path ${output_path} --out_path ${output_path}.jsonl --add_prompt
evaluate_functional_correctness ${output_path}.jsonl
```
### How to Reproduce the 59.8 Pass@1 on HumanEval with Greedy Decoding?
❗❗❗**This performance is 100% reproducible!**
Run the following script to generate the answer with greedy decoding. Then follow the above steps 2 and 3 to get the evaluation result.
❗We also provide the generated codes in `data/humaneval.59.8.gen.zip`
```bash
model="WizardLM/WizardCoder-15B-V1.0"
temp=0.0
max_len=2048
pred_num=1
num_seqs_per_iter=1
output_path=preds/T${temp}_N${pred_num}_WizardCoder_Greedy_Decode
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
# 164 problems, 21 per GPU if GPU=8
index=0
gpu_num=8
for ((i = 0; i < $gpu_num; i++)); do
start_index=$((i * 21))
end_index=$(((i + 1) * 21))
gpu=$((i))
echo 'Running process #' ${i} 'from' $start_index 'to' $end_index 'on GPU' ${gpu}
((index++))
(
CUDA_VISIBLE_DEVICES=$gpu python humaneval_gen.py --model ${model} \
--start_index ${start_index} --end_index ${end_index} --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} --greedy_decode
) &
if (($index % $gpu_num == 0)); then wait; fi
done
```
### MBPP
1. Run the following script to generate the answer.
```bash
model="/path/to/your/model"
temp=0.2
max_len=2048
pred_num=200
num_seqs_per_iter=2
output_path=preds/MBPP_T${temp}_N${pred_num}
mbpp_path=data/mbpp.test.jsonl # we provide this file in data/mbpp.test.zip
mkdir -p ${output_path}
echo 'Output path: '$output_path
echo 'Model to eval: '$model
# 500 problems, 63 per GPU if GPU=8
index=0
gpu_num=8
for ((i = 0; i < $gpu_num; i++)); do
start_index=$((i * 50))
end_index=$(((i + 1) * 50))
gpu=$((i))
echo 'Running process #' ${i} 'from' $start_index 'to' $end_index 'on GPU' ${gpu}
((index++))
(
CUDA_VISIBLE_DEVICES=$gpu python mbpp_gen.py --model ${model} \
--start_index ${start_index} --end_index ${end_index} --temperature ${temp} \
--num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} --mbpp_path ${mbpp_path}
) &
if (($index % $gpu_num == 0)); then wait; fi
done
```
3. Run the post processing code `src/process_mbpp.py` to collect the code completions from all answer files.
```bash
output_path=preds/MBPP_T${temp}_N${pred_num}
mbpp_path=data/mbpp.test.jsonl # we provide this file in data/mbpp.test.zip
echo 'Output path: '$output_path
python process_mbpp.py --path ${output_path} --out_path ${output_path}.jsonl --mbpp_path ${mbpp_path} --add_prompt
```
4. Evaluate the `MBPP_T${temp}_N${pred_num}.jsonl` with [bigcode-evaluation-harness](https://github.com/bigcode-project/bigcode-evaluation-harness).
Acknowledgement: The evaluation code `humaneval_gen.py`, `mbpp_gen.py` and bash scripts are modified from the great works of [CodeT5](https://github.com/salesforce/CodeT5).
## Citation
Please cite the repo if you use the data or code in this repo.
```
@inproceedings{
luo2024wizardcoder,
title={WizardCoder: Empowering Code Large Language Models with Evol-Instruct},
author={Ziyang Luo and Can Xu and Pu Zhao and Qingfeng Sun and Xiubo Geng and Wenxiang Hu and Chongyang Tao and Jing Ma and Qingwei Lin and Daxin Jiang},
booktitle={The Twelfth International Conference on Learning Representations},
year={2024},
url={https://openreview.net/forum?id=UnUwSIgK5W}
}
```
## Disclaimer
WizardCoder model follows the same license as StarCoder. The content produced by any version of WizardCoder is influenced by uncontrollable variables such as randomness, and therefore, the accuracy of the output cannot be guaranteed by this project. This project does not accept any legal liability for the content of the model output, nor does it assume responsibility for any losses incurred due to the use of associated resources and output results.
## Star History
[](https://star-history.com/#nlpxucan/WizardLM&Timeline)
================================================
FILE: WizardCoder/data/mbppplus.json
================================================
{
"Mbpp/2": "Write a function to find the shared elements from the given two lists. Your code should satisfy the following assertion:\n```python\nassert set(similar_elements((3, 4, 5, 6),(5, 7, 4, 10))) == set((4, 5))\n```",
"Mbpp/3": "Write a python function to identify non-prime numbers. Your code should satisfy the following assertion:\n```python\nassert is_not_prime(2) == False\n```",
"Mbpp/4": "Write a function to find the n largest integers from a given list of numbers, returned in descending order. Your code should satisfy the following assertion:\n```python\nassert heap_queue_largest( [25, 35, 22, 85, 14, 65, 75, 22, 58],3)==[85, 75, 65]\n```",
"Mbpp/6": "Write a python function to check whether the two numbers differ at one bit position only or not. Your code should satisfy the following assertion:\n```python\nassert differ_At_One_Bit_Pos(13,9) == True\n```",
"Mbpp/7": "Write a function to find all words which are at least 4 characters long in a string. Your code should satisfy the following assertion:\n```python\nassert set(find_char_long('Please move back to stream')) == set(['Please', 'move', 'back', 'stream'])\n```",
"Mbpp/8": "Write a function to find squares of individual elements in a list. Your code should satisfy the following assertion:\n```python\nassert square_nums([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])==[1, 4, 9, 16, 25, 36, 49, 64, 81, 100]\n```",
"Mbpp/9": "Write a python function to find the minimum number of rotations (greater than 0) required to get the same string. Your code should satisfy the following assertion:\n```python\nassert find_Rotations(\"aaaa\") == 1\n```",
"Mbpp/11": "Write a python function to remove first and last occurrence of a given character from the string. Your code should satisfy the following assertion:\n```python\nassert remove_Occ(\"hello\",\"l\") == \"heo\"\n```",
"Mbpp/12": "Write a function to sort a given matrix in ascending order according to the sum of its rows. Your code should satisfy the following assertion:\n```python\nassert sort_matrix([[1, 2, 3], [2, 4, 5], [1, 1, 1]])==[[1, 1, 1], [1, 2, 3], [2, 4, 5]]\n```",
"Mbpp/14": "Write a python function to find the volume of a triangular prism. Your code should satisfy the following assertion:\n```python\nassert find_Volume(10,8,6) == 240\n```",
"Mbpp/16": "Write a function to that returns true if the input string contains sequences of lowercase letters joined with an underscore and false otherwise. Your code should satisfy the following assertion:\n```python\nassert text_lowercase_underscore(\"aab_cbbbc\")==(True)\n```",
"Mbpp/17": "Write a function that returns the perimeter of a square given its side length as input. Your code should satisfy the following assertion:\n```python\nassert square_perimeter(10)==40\n```",
"Mbpp/18": "Write a function to remove characters from the first string which are present in the second string. Your code should satisfy the following assertion:\n```python\nassert remove_dirty_chars(\"probasscurve\", \"pros\") == 'bacuve'\n```",
"Mbpp/19": "Write a function to find whether a given array of integers contains any duplicate element. Your code should satisfy the following assertion:\n```python\nassert test_duplicate(([1,2,3,4,5]))==False\n```",
"Mbpp/20": "Write a function to check if the given number is woodball or not. Your code should satisfy the following assertion:\n```python\nassert is_woodall(383) == True\n```",
"Mbpp/56": "Write a python function to check if a given number is one less than twice its reverse. Your code should satisfy the following assertion:\n```python\nassert check(70) == False\n```",
"Mbpp/57": "Write a python function to find the largest number that can be formed with the given list of digits. Your code should satisfy the following assertion:\n```python\nassert find_Max_Num([1,2,3]) == 321\n```",
"Mbpp/58": "Write a python function to check whether the given two integers have opposite sign or not. Your code should satisfy the following assertion:\n```python\nassert opposite_Signs(1,-2) == True\n```",
"Mbpp/59": "Write a function to find the nth octagonal number. Your code should satisfy the following assertion:\n```python\nassert is_octagonal(5) == 65\n```",
"Mbpp/61": "Write a python function to count the number of substrings with the sum of digits equal to their length. Your code should satisfy the following assertion:\n```python\nassert count_Substrings('112112') == 6\n```",
"Mbpp/62": "Write a python function to find smallest number in a list. Your code should satisfy the following assertion:\n```python\nassert smallest_num([10, 20, 1, 45, 99]) == 1\n```",
"Mbpp/63": "Write a function to find the maximum difference between available pairs in the given tuple list. Your code should satisfy the following assertion:\n```python\nassert max_difference([(3, 5), (1, 7), (10, 3), (1, 2)]) == 7\n```",
"Mbpp/64": "Write a function to sort a list of tuples using the second value of each tuple. Your code should satisfy the following assertion:\n```python\nassert subject_marks([('English', 88), ('Science', 90), ('Maths', 97), ('Social sciences', 82)])==[('Social sciences', 82), ('English', 88), ('Science', 90), ('Maths', 97)]\n```",
"Mbpp/65": "Write a function to flatten a list and sum all of its elements. Your code should satisfy the following assertion:\n```python\nassert recursive_list_sum(([1, 2, [3,4],[5,6]]))==21\n```",
"Mbpp/66": "Write a python function to count the number of positive numbers in a list. Your code should satisfy the following assertion:\n```python\nassert pos_count([1,-2,3,-4]) == 2\n```",
"Mbpp/67": "Write a function to find the number of ways to partition a set of Bell numbers. Your code should satisfy the following assertion:\n```python\nassert bell_number(2)==2\n```",
"Mbpp/68": "Write a python function to check whether the given array is monotonic or not. Your code should satisfy the following assertion:\n```python\nassert is_Monotonic([6, 5, 4, 4]) == True\n```",
"Mbpp/69": "Write a function to check whether a list contains the given sublist or not. Your code should satisfy the following assertion:\n```python\nassert is_sublist([2,4,3,5,7],[3,7])==False\n```",
"Mbpp/70": "Write a function to find whether all the given tuples have equal length or not. Your code should satisfy the following assertion:\n```python\nassert get_equal([(11, 22, 33), (44, 55, 66)]) == True\n```",
"Mbpp/71": "Write a function to sort a list of elements. Your code should satisfy the following assertion:\n```python\nassert comb_sort([5, 15, 37, 25, 79]) == [5, 15, 25, 37, 79]\n```",
"Mbpp/72": "Write a python function to check whether the given number can be represented as the difference of two squares or not. Your code should satisfy the following assertion:\n```python\nassert dif_Square(5) == True\n```",
"Mbpp/74": "Write a function to check whether it follows the sequence given in the patterns array. Your code should satisfy the following assertion:\n```python\nassert is_samepatterns([\"red\",\"green\",\"green\"], [\"a\", \"b\", \"b\"])==True\n```",
"Mbpp/75": "Write a function to find tuples which have all elements divisible by k from the given list of tuples. Your code should satisfy the following assertion:\n```python\nassert find_tuples([(6, 24, 12), (7, 9, 6), (12, 18, 21)], 6) == [(6, 24, 12)]\n```",
"Mbpp/77": "Write a python function to find whether a number is divisible by 11. Your code should satisfy the following assertion:\n```python\nassert is_Diff (12345) == False\n```",
"Mbpp/79": "Write a python function to check whether the length of the word is odd or not. Your code should satisfy the following assertion:\n```python\nassert word_len(\"Hadoop\") == False\n```",
"Mbpp/80": "Write a function to find the nth tetrahedral number. Your code should satisfy the following assertion:\n```python\nassert tetrahedral_number(5) == 35\n```",
"Mbpp/82": "Write a function to find the volume of a sphere. Your code should satisfy the following assertion:\n```python\nassert math.isclose(volume_sphere(10), 4188.790204786391, rel_tol=0.001)\n```",
"Mbpp/83": "Write a python function to find the character made by adding the ASCII value of all the characters of the given string modulo 26. Your code should satisfy the following assertion:\n```python\nassert get_Char(\"abc\") == \"f\"\n```",
"Mbpp/84": "Write a function to find the nth number in the newman conway sequence. Your code should satisfy the following assertion:\n```python\nassert sequence(10) == 6\n```",
"Mbpp/85": "Write a function to find the surface area of a sphere. Your code should satisfy the following assertion:\n```python\nassert math.isclose(surfacearea_sphere(10), 1256.6370614359173, rel_tol=0.001)\n```",
"Mbpp/86": "Write a function to find nth centered hexagonal number. Your code should satisfy the following assertion:\n```python\nassert centered_hexagonal_number(10) == 271\n```",
"Mbpp/87": "Write a function to merge three dictionaries into a single dictionary. Your code should satisfy the following assertion:\n```python\nassert merge_dictionaries_three({ \"R\": \"Red\", \"B\": \"Black\", \"P\": \"Pink\" }, { \"G\": \"Green\", \"W\": \"White\" },{ \"O\": \"Orange\", \"W\": \"White\", \"B\": \"Black\" })=={'B': 'Black', 'R': 'Red', 'P': 'Pink', 'G': 'Green', 'W': 'White', 'O': 'Orange'}\n```",
"Mbpp/88": "Write a function to get the frequency of all the elements in a list, returned as a dictionary. Your code should satisfy the following assertion:\n```python\nassert freq_count([10,10,10,10,20,20,20,20,40,40,50,50,30])==({10: 4, 20: 4, 40: 2, 50: 2, 30: 1})\n```",
"Mbpp/89": "Write a function to find the closest smaller number than n. Your code should satisfy the following assertion:\n```python\nassert closest_num(11) == 10\n```",
"Mbpp/90": "Write a python function to find the length of the longest word. Your code should satisfy the following assertion:\n```python\nassert len_log([\"python\",\"PHP\",\"bigdata\"]) == 7\n```",
"Mbpp/91": "Write a function to check if a string is present as a substring in a given list of string values. Your code should satisfy the following assertion:\n```python\nassert find_substring([\"red\", \"black\", \"white\", \"green\", \"orange\"],\"ack\")==True\n```",
"Mbpp/92": "Write a function to check whether the given number is undulating or not. Your code should satisfy the following assertion:\n```python\nassert is_undulating(1212121) == True\n```",
"Mbpp/93": "Write a function to calculate the value of 'a' to the power 'b'. Your code should satisfy the following assertion:\n```python\nassert power(3,4) == 81\n```",
"Mbpp/94": "Given a list of tuples, write a function that returns the first value of the tuple with the smallest second value. Your code should satisfy the following assertion:\n```python\nassert index_minimum([('Rash', 143), ('Manjeet', 200), ('Varsha', 100)]) == 'Varsha'\n```",
"Mbpp/95": "Write a python function to find the length of the smallest list in a list of lists. Your code should satisfy the following assertion:\n```python\nassert Find_Min_Length([[1],[1,2]]) == 1\n```",
"Mbpp/96": "Write a python function to find the number of divisors of a given integer. Your code should satisfy the following assertion:\n```python\nassert divisor(15) == 4\n```",
"Mbpp/97": "Write a function to find frequency of each element in a flattened list of lists, returned in a dictionary. Your code should satisfy the following assertion:\n```python\nassert frequency_lists([[1, 2, 3, 2], [4, 5, 6, 2], [7, 8, 9, 5]])=={1: 1, 2: 3, 3: 1, 4: 1, 5: 2, 6: 1, 7: 1, 8: 1, 9: 1}\n```",
"Mbpp/98": "Write a function to multiply all the numbers in a list and divide with the length of the list. Your code should satisfy the following assertion:\n```python\nassert math.isclose(multiply_num((8, 2, 3, -1, 7)), -67.2, rel_tol=0.001)\n```",
"Mbpp/99": "Write a function to convert the given decimal number to its binary equivalent, represented as a string with no leading zeros. Your code should satisfy the following assertion:\n```python\nassert decimal_to_binary(8) == '1000'\n```",
"Mbpp/100": "Write a function to find the next smallest palindrome of a specified integer, returned as an integer. Your code should satisfy the following assertion:\n```python\nassert next_smallest_palindrome(99)==101\n```",
"Mbpp/101": "Write a function to find the kth element in the given array using 1-based indexing. Your code should satisfy the following assertion:\n```python\nassert kth_element([12,3,5,7,19], 2) == 3\n```",
"Mbpp/102": "Write a function to convert a snake case string to camel case string. Your code should satisfy the following assertion:\n```python\nassert snake_to_camel('python_program')=='PythonProgram'\n```",
"Mbpp/103": "Write a function to find the Eulerian number a(n, m). Your code should satisfy the following assertion:\n```python\nassert eulerian_num(3, 1) == 4\n```",
"Mbpp/104": "Write a function to sort each sublist of strings in a given list of lists. Your code should satisfy the following assertion:\n```python\nassert sort_sublists(([\"green\", \"orange\"], [\"black\", \"white\"], [\"white\", \"black\", \"orange\"]))==[['green', 'orange'], ['black', 'white'], ['black', 'orange', 'white']]\n```",
"Mbpp/105": "Write a python function to count true booleans in the given list. Your code should satisfy the following assertion:\n```python\nassert count([True,False,True]) == 2\n```",
"Mbpp/106": "Write a function to append the given list to the given tuples. Your code should satisfy the following assertion:\n```python\nassert add_lists([5, 6, 7], (9, 10)) == (9, 10, 5, 6, 7)\n```",
"Mbpp/108": "Write a function to merge three lists into a single sorted list. Your code should satisfy the following assertion:\n```python\nassert merge_sorted_list([25, 24, 15, 4, 5, 29, 110],[19, 20, 11, 56, 25, 233, 154],[24, 26, 54, 48])==[4, 5, 11, 15, 19, 20, 24, 24, 25, 25, 26, 29, 48, 54, 56, 110, 154, 233]\n```",
"Mbpp/109": "Write a python function to find the number of numbers with an odd value when rotating a binary string the given number of times. Your code should satisfy the following assertion:\n```python\nassert odd_Equivalent(\"011001\",6) == 3\n```",
"Mbpp/111": "Write a function to find the common elements in given nested lists. Your code should satisfy the following assertion:\n```python\nassert set(common_in_nested_lists([[12, 18, 23, 25, 45], [7, 12, 18, 24, 28], [1, 5, 8, 12, 15, 16, 18]]))==set([18, 12])\n```",
"Mbpp/113": "Write a function to check if a string represents an integer or not. Your code should satisfy the following assertion:\n```python\nassert check_integer(\"python\")==False\n```",
"Mbpp/115": "Write a function to check whether all dictionaries in a list are empty or not. Your code should satisfy the following assertion:\n```python\nassert empty_dit([{},{},{}])==True\n```",
"Mbpp/116": "Write a function to convert a given tuple of positive integers into a single integer. Your code should satisfy the following assertion:\n```python\nassert tuple_to_int((1,2,3))==123\n```",
"Mbpp/117": "Write a function to convert all possible convertible elements in a list of lists to floats. Your code should satisfy the following assertion:\n```python\nassert list_to_float( [(\"3\", \"4\"), (\"1\", \"26.45\"), (\"7.32\", \"8\"), (\"4\", \"8\")] ) == [(3.0, 4.0), (1.0, 26.45), (7.32, 8.0), (4.0, 8.0)]\n```",
"Mbpp/118": "Write a function to convert a string to a list of strings split on the space character. Your code should satisfy the following assertion:\n```python\nassert string_to_list(\"python programming\")==['python','programming']\n```",
"Mbpp/119": "Write a python function to find the element that appears only once in a sorted array. Your code should satisfy the following assertion:\n```python\nassert search([1,1,2,2,3]) == 3\n```",
"Mbpp/120": "Write a function to find the maximum absolute product between numbers in pairs of tuples within a given list. Your code should satisfy the following assertion:\n```python\nassert max_product_tuple([(2, 7), (2, 6), (1, 8), (4, 9)] )==36\n```",
"Mbpp/123": "Write a function to sum all amicable numbers from 1 to a specified number. Your code should satisfy the following assertion:\n```python\nassert amicable_numbers_sum(999)==504\n```",
"Mbpp/124": "Write a function to get the angle of a complex number. Your code should satisfy the following assertion:\n```python\nassert math.isclose(angle_complex(0,1j), 1.5707963267948966, rel_tol=0.001)\n```",
"Mbpp/125": "Write a function to find the maximum difference between the number of 0s and number of 1s in any sub-string of the given binary string. Your code should satisfy the following assertion:\n```python\nassert find_length(\"11000010001\") == 6\n```",
"Mbpp/126": "Write a python function to find the sum of common divisors of two given numbers. Your code should satisfy the following assertion:\n```python\nassert sum(10,15) == 6\n```",
"Mbpp/127": "Write a function to multiply two integers. Your code should satisfy the following assertion:\n```python\nassert multiply_int(10,20)==200\n```",
"Mbpp/128": "Write a function to find words that are longer than n characters from a given list of words. Your code should satisfy the following assertion:\n```python\nassert long_words(3,\"python is a programming language\")==['python','programming','language']\n```",
"Mbpp/129": "Write a function to calculate whether the matrix is a magic square. Your code should satisfy the following assertion:\n```python\nassert magic_square_test([[7, 12, 1, 14], [2, 13, 8, 11], [16, 3, 10, 5], [9, 6, 15, 4]])==True\n```",
"Mbpp/130": "Write a function to find the item with maximum frequency in a given list. Your code should satisfy the following assertion:\n```python\nassert max_occurrences([2,3,8,4,7,9,8,2,6,5,1,6,1,2,3,2,4,6,9,1,2])==2\n```",
"Mbpp/131": "Write a python function to reverse only the vowels of a given string (where y is not a vowel). Your code should satisfy the following assertion:\n```python\nassert reverse_vowels(\"Python\") == \"Python\"\n```",
"Mbpp/132": "Write a function to convert a tuple to a string. Your code should satisfy the following assertion:\n```python\nassert tup_string(('e', 'x', 'e', 'r', 'c', 'i', 's', 'e', 's'))==(\"exercises\")\n```",
"Mbpp/133": "Write a function to calculate the sum of the negative numbers of a given list of numbers. Your code should satisfy the following assertion:\n```python\nassert sum_negativenum([2, 4, -6, -9, 11, -12, 14, -5, 17])==-32\n```",
"Mbpp/135": "Write a function to find the nth hexagonal number. Your code should satisfy the following assertion:\n```python\nassert hexagonal_num(10) == 190\n```",
"Mbpp/137": "Write a function to find the ratio of zeroes to non-zeroes in an array of integers. Your code should satisfy the following assertion:\n```python\nassert math.isclose(zero_count([0, 1, 2, -1, -5, 6, 0, -3, -2, 3, 4, 6, 8]), 0.181818, rel_tol=0.001)\n```",
"Mbpp/138": "Write a python function to check whether the given number can be represented as sum of non-zero powers of 2 or not. Your code should satisfy the following assertion:\n```python\nassert is_Sum_Of_Powers_Of_Two(10) == True\n```",
"Mbpp/139": "Write a function to find the circumference of a circle. Your code should satisfy the following assertion:\n```python\nassert math.isclose(circle_circumference(10), 62.830000000000005, rel_tol=0.001)\n```",
"Mbpp/140": "Write a function to flatten the list of lists into a single set of numbers. Your code should satisfy the following assertion:\n```python\nassert set(extract_singly([(3, 4, 5), (4, 5, 7), (1, 4)])) == set([3, 4, 5, 7, 1])\n```",
"Mbpp/141": "Write a function to sort a list of elements. Your code should satisfy the following assertion:\n```python\nassert pancake_sort([15, 79, 25, 38, 69]) == [15, 25, 38, 69, 79]\n```",
"Mbpp/142": "Write a function to count number items that are identical in the same position of three given lists. Your code should satisfy the following assertion:\n```python\nassert count_samepair([1,2,3,4,5,6,7,8],[2,2,3,1,2,6,7,9],[2,1,3,1,2,6,7,9])==3\n```",
"Mbpp/143": "Write a function to find number of lists present in the given tuple. Your code should satisfy the following assertion:\n```python\nassert find_lists(([1, 2, 3, 4], [5, 6, 7, 8])) == 2\n```",
"Mbpp/145": "Write a python function to find the maximum difference between any two elements in a given array. Your code should satisfy the following assertion:\n```python\nassert max_Abs_Diff((2,1,5,3)) == 4\n```",
"Mbpp/160": "Write a function that returns integers x and y that satisfy ax + by = n as a tuple, or return None if no solution exists. Your code should satisfy the following assertion:\n```python\nassert find_solution(2, 3, 7) == (2, 1)\n```",
"Mbpp/161": "Write a function to remove all elements from a given list present in another list. Your code should satisfy the following assertion:\n```python\nassert remove_elements([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [2, 4, 6, 8]) == [1, 3, 5, 7, 9, 10]\n```",
"Mbpp/162": "Write a function to calculate the sum (n - 2*i) from i=0 to n // 2, for instance n + (n-2) + (n-4)... (until n-x =< 0). Your code should satisfy the following assertion:\n```python\nassert sum_series(6) == 12\n```",
"Mbpp/164": "Write a function to determine if the sum of the divisors of two integers are the same. Your code should satisfy the following assertion:\n```python\nassert are_equivalent(36, 57) == False\n```",
"Mbpp/165": "Write a function to count the number of characters in a string that occur at the same position in the string as in the English alphabet (case insensitive). Your code should satisfy the following assertion:\n```python\nassert count_char_position(\"xbcefg\") == 2\n```",
"Mbpp/166": "Write a function that counts the number of pairs of integers in a list that xor to an even number. Your code should satisfy the following assertion:\n```python\nassert find_even_pair([5, 4, 7, 2, 1]) == 4\n```",
"Mbpp/167": "Write a python function to find the smallest power of 2 greater than or equal to n. Your code should satisfy the following assertion:\n```python\nassert next_power_of_2(0) == 1\n```",
"Mbpp/168": "Write a function to count the number of occurrences of a number in a given list. Your code should satisfy the following assertion:\n```python\nassert frequency([1,2,3], 4) == 0\n```",
"Mbpp/170": "Write a function to find the sum of numbers in a list within a range specified by two indices. Your code should satisfy the following assertion:\n```python\nassert sum_range_list([2,1,5,6,8,3,4,9,10,11,8,12], 8, 10) == 29\n```",
"Mbpp/171": "Write a function to find the perimeter of a regular pentagon from the length of its sides. Your code should satisfy the following assertion:\n```python\nassert perimeter_pentagon(5) == 25\n```",
"Mbpp/172": "Write a function to count the number of occurence of the string 'std' in a given string. Your code should satisfy the following assertion:\n```python\nassert count_occurance(\"letstdlenstdporstd\") == 3\n```",
"Mbpp/222": "Write a function to check if all the elements in tuple have same data type or not. Your code should satisfy the following assertion:\n```python\nassert check_type((5, 6, 7, 3, 5, 6) ) == True\n```",
"Mbpp/223": "Write a function that takes in a sorted array, its length (n), and an element and returns whether the element is the majority element in the given sorted array. (The majority element is the element that occurs more than n/2 times.). Your code should satisfy the following assertion:\n```python\nassert is_majority([1, 2, 3, 3, 3, 3, 10], 7, 3) == True\n```",
"Mbpp/224": "Write a python function to count the number of set bits (binary digits with value 1) in a given number. Your code should satisfy the following assertion:\n```python\nassert count_Set_Bits(2) == 1\n```",
"Mbpp/226": "Write a python function to remove the characters which have odd index values of a given string. Your code should satisfy the following assertion:\n```python\nassert odd_values_string('abcdef') == 'ace'\n```",
"Mbpp/227": "Write a function to find minimum of three numbers. Your code should satisfy the following assertion:\n```python\nassert min_of_three(10,20,0)==0\n```",
"Mbpp/229": "Write a function that takes in an array and an integer n, and re-arranges the first n elements of the given array so that all negative elements appear before positive ones, and where the relative order among negative and positive elements is preserved. Your code should satisfy the following assertion:\n```python\nassert re_arrange_array([-1, 2, -3, 4, 5, 6, -7, 8, 9], 9) == [-1, -3, -7, 4, 5, 6, 2, 8, 9]\n```",
"Mbpp/230": "Write a function that takes in a string and character, replaces blank spaces in the string with the character, and returns the string. Your code should satisfy the following assertion:\n```python\nassert replace_blank(\"hello people\",'@')==(\"hello@people\")\n```",
"Mbpp/232": "Write a function that takes in a list and an integer n and returns a list containing the n largest items from the list. Your code should satisfy the following assertion:\n```python\nassert set(larg_nnum([10, 20, 50, 70, 90, 20, 50, 40, 60, 80, 100],2))==set([100,90])\n```",
"Mbpp/233": "Write a function to find the lateral surface area of a cylinder. Your code should satisfy the following assertion:\n```python\nassert math.isclose(lateralsuface_cylinder(10,5), 314.15000000000003, rel_tol=0.001)\n```",
"Mbpp/234": "Write a function to find the volume of a cube given its side length. Your code should satisfy the following assertion:\n```python\nassert volume_cube(3)==27\n```",
"Mbpp/235": "Write a python function to set all even bits of a given number. Your code should satisfy the following assertion:\n```python\nassert even_bit_set_number(10) == 10\n```",
"Mbpp/237": "Write a function that takes in a list of tuples and returns a dictionary mapping each unique tuple to the number of times it occurs in the list. Your code should satisfy the following assertion:\n```python\nassert check_occurences([(3, 1), (1, 3), (2, 5), (5, 2), (6, 3)] ) == {(1, 3): 2, (2, 5): 2, (3, 6): 1}\n```",
"Mbpp/238": "Write a python function to count the number of non-empty substrings of a given string. Your code should satisfy the following assertion:\n```python\nassert number_of_substrings(\"abc\") == 6\n```",
"Mbpp/239": "Write a function that takes in positive integers m and n and finds the number of possible sequences of length n, such that each element is a positive integer and is greater than or equal to twice the previous element but less than or equal to m. Your code should satisfy the following assertion:\n```python\nassert get_total_number_of_sequences(10, 4) == 4\n```",
"Mbpp/240": "Write a function that takes in two lists and replaces the last element of the first list with the elements of the second list. Your code should satisfy the following assertion:\n```python\nassert replace_list([1, 3, 5, 7, 9, 10],[2, 4, 6, 8])==[1, 3, 5, 7, 9, 2, 4, 6, 8]\n```",
"Mbpp/242": "Write a function to count the total number of characters in a string. Your code should satisfy the following assertion:\n```python\nassert count_charac(\"python programming\")==18\n```",
"Mbpp/244": "Write a python function to find the next perfect square greater than a given number. Your code should satisfy the following assertion:\n```python\nassert next_Perfect_Square(35) == 36\n```",
"Mbpp/245": "Write a function that takes an array and finds the maximum sum of a bitonic subsequence for the given array, where a sequence is bitonic if it is first increasing and then decreasing. Your code should satisfy the following assertion:\n```python\nassert max_sum([1, 15, 51, 45, 33, 100, 12, 18, 9]) == 194\n```",
"Mbpp/247": "Write a function to find the length of the longest palindromic subsequence in the given string. Your code should satisfy the following assertion:\n```python\nassert lps(\"TENS FOR TENS\") == 5\n```",
"Mbpp/249": "Write a function to find the intersection of two arrays. Your code should satisfy the following assertion:\n```python\nassert intersection_array([1, 2, 3, 5, 7, 8, 9, 10],[1, 2, 4, 8, 9])==[1, 2, 8, 9]\n```",
"Mbpp/250": "Write a python function that takes in a tuple and an element and counts the occcurences of the element in the tuple. Your code should satisfy the following assertion:\n```python\nassert count_X((10, 8, 5, 2, 10, 15, 10, 8, 5, 8, 8, 2),4) == 0\n```",
"Mbpp/251": "Write a function that takes in a list and an element and inserts the element before each element in the list, and returns the resulting list. Your code should satisfy the following assertion:\n```python\nassert insert_element(['Red', 'Green', 'Black'] ,'c')==['c', 'Red', 'c', 'Green', 'c', 'Black']\n```",
"Mbpp/252": "Write a python function to convert complex numbers to polar coordinates. Your code should satisfy the following assertion:\n```python\nassert convert(1) == (1.0, 0.0)\n```",
"Mbpp/253": "Write a python function that returns the number of integer elements in a given list. Your code should satisfy the following assertion:\n```python\nassert count_integer([1,2,'abc',1.2]) == 2\n```",
"Mbpp/255": "Write a function that takes in a list and length n, and generates all combinations (with repetition) of the elements of the list and returns a list with a tuple for each combination. Your code should satisfy the following assertion:\n```python\nassert combinations_colors( [\"Red\",\"Green\",\"Blue\"],1)==[('Red',), ('Green',), ('Blue',)]\n```",
"Mbpp/256": "Write a python function that takes in a non-negative number and returns the number of prime numbers less than the given non-negative number. Your code should satisfy the following assertion:\n```python\nassert count_Primes_nums(5) == 2\n```",
"Mbpp/257": "Write a function that takes in two numbers and returns a tuple with the second number and then the first number. Your code should satisfy the following assertion:\n```python\nassert swap_numbers(10,20)==(20,10)\n```",
"Mbpp/259": "Write a function to maximize the given two tuples. Your code should satisfy the following assertion:\n```python\nassert maximize_elements(((1, 3), (4, 5), (2, 9), (1, 10)), ((6, 7), (3, 9), (1, 1), (7, 3))) == ((6, 7), (4, 9), (2, 9), (7, 10))\n```",
"Mbpp/260": "Write a function to find the nth newman\u2013shanks\u2013williams prime number. Your code should satisfy the following assertion:\n```python\nassert newman_prime(3) == 7\n```",
"Mbpp/261": "Write a function that takes in two tuples and performs mathematical division operation element-wise across the given tuples. Your code should satisfy the following assertion:\n```python\nassert division_elements((10, 4, 6, 9),(5, 2, 3, 3)) == (2, 2, 2, 3)\n```",
"Mbpp/262": "Write a function that takes in a list and an integer L and splits the given list into two parts where the length of the first part of the list is L, and returns the resulting lists in a tuple. Your code should satisfy the following assertion:\n```python\nassert split_two_parts([1,1,2,3,4,4,5,1],3)==([1, 1, 2], [3, 4, 4, 5, 1])\n```",
"Mbpp/264": "Write a function to calculate a dog's age in dog's years. Your code should satisfy the following assertion:\n```python\nassert dog_age(12)==61\n```",
"Mbpp/265": "Write a function that takes in a list and an integer n and splits a list for every nth element, returning a list of the resulting lists. Your code should satisfy the following assertion:\n```python\nassert list_split(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n'],3)==[['a', 'd', 'g', 'j', 'm'], ['b', 'e', 'h', 'k', 'n'], ['c', 'f', 'i', 'l']]\n```",
"Mbpp/266": "Write a function to find the lateral surface area of a cube given its side length. Your code should satisfy the following assertion:\n```python\nassert lateralsurface_cube(5)==100\n```",
"Mbpp/267": "Write a python function that takes in an integer n and returns the sum of the squares of the first n odd natural numbers. Your code should satisfy the following assertion:\n```python\nassert square_Sum(2) == 10\n```",
"Mbpp/268": "Write a function to find the n'th star number. Your code should satisfy the following assertion:\n```python\nassert find_star_num(3) == 37\n```",
"Mbpp/269": "Write a function to find the ascii value of a character. Your code should satisfy the following assertion:\n```python\nassert ascii_value('A')==65\n```",
"Mbpp/270": "Write a python function to find the sum of even numbers at even positions of a list. Your code should satisfy the following assertion:\n```python\nassert sum_even_and_even_index([5, 6, 12, 1, 18, 8]) == 30\n```",
"Mbpp/271": "Write a python function that takes in an integer n and finds the sum of the first n even natural numbers that are raised to the fifth power. Your code should satisfy the following assertion:\n```python\nassert even_Power_Sum(2) == 1056\n```",
"Mbpp/272": "Write a function that takes in a list of tuples and returns a list containing the rear element of each tuple. Your code should satisfy the following assertion:\n```python\nassert rear_extract([(1, 'Rash', 21), (2, 'Varsha', 20), (3, 'Kil', 19)]) == [21, 20, 19]\n```",
"Mbpp/273": "Write a function that takes in two tuples and subtracts the elements of the first tuple by the elements of the second tuple with the same index. Your code should satisfy the following assertion:\n```python\nassert substract_elements((10, 4, 5), (2, 5, 18)) == (8, -1, -13)\n```",
"Mbpp/274": "Write a python function that takes in a positive integer n and finds the sum of even index binomial coefficients. Your code should satisfy the following assertion:\n```python\nassert even_binomial_Coeff_Sum(4) == 8\n```",
"Mbpp/276": "Write a function that takes in the radius and height of a cylinder and returns the the volume. Your code should satisfy the following assertion:\n```python\nassert math.isclose(volume_cylinder(10,5), 1570.7500000000002, rel_tol=0.001)\n```",
"Mbpp/277": "Write a function that takes in a dictionary and integer n and filters the dictionary to only include entries with values greater than or equal to n. Your code should satisfy the following assertion:\n```python\nassert dict_filter({'Cierra Vega': 175, 'Alden Cantrell': 180, 'Kierra Gentry': 165, 'Pierre Cox': 190},170)=={'Cierra Vega': 175, 'Alden Cantrell': 180, 'Pierre Cox': 190}\n```",
"Mbpp/278": "Write a function to find the number of elements that occurs before the tuple element in the given tuple. Your code should satisfy the following assertion:\n```python\nassert count_first_elements((1, 5, 7, (4, 6), 10) ) == 3\n```",
"Mbpp/279": "Write a function to find the nth decagonal number. Your code should satisfy the following assertion:\n```python\nassert is_num_decagonal(3) == 27\n```",
"Mbpp/280": "Write a function that takes in an array and element and returns a tuple containing a boolean that indicates if the element is in the array and the index position of the element (or -1 if the element is not found). Your code should satisfy the following assertion:\n```python\nassert sequential_search([11,23,58,31,56,77,43,12,65,19],31) == (True, 3)\n```",
"Mbpp/281": "Write a python function to check if the elements of a given list are unique or not. Your code should satisfy the following assertion:\n```python\nassert all_unique([1,2,3]) == True\n```",
"Mbpp/282": "Write a function to subtract two lists element-wise. Your code should satisfy the following assertion:\n```python\nassert sub_list([1, 2, 3],[4,5,6])==[-3,-3,-3]\n```",
"Mbpp/283": "Write a python function takes in an integer and check whether the frequency of each digit in the integer is less than or equal to the digit itself. Your code should satisfy the following assertion:\n```python\nassert validate(1234) == True\n```",
"Mbpp/284": "Write a function that takes in a list and element and checks whether all items in the list are equal to the given element. Your code should satisfy the following assertion:\n```python\nassert check_element([\"green\", \"orange\", \"black\", \"white\"],'blue')==False\n```",
"Mbpp/285": "Write a function that checks whether a string contains the 'a' character followed by two or three 'b' characters. Your code should satisfy the following assertion:\n```python\nassert text_match_two_three(\"ac\")==(False)\n```",
"Mbpp/286": "Write a function to find the largest sum of a contiguous array in the modified array which is formed by repeating the given array k times. Your code should satisfy the following assertion:\n```python\nassert max_sub_array_sum_repeated([10, 20, -30, -1], 4, 3) == 30\n```",
"Mbpp/287": "Write a python function takes in an integer n and returns the sum of squares of first n even natural numbers. Your code should satisfy the following assertion:\n```python\nassert square_Sum(2) == 20\n```",
"Mbpp/290": "Write a function to find the list of maximum length in a list of lists. Your code should satisfy the following assertion:\n```python\nassert max_length([[0], [1, 3], [5, 7], [9, 11], [13, 15, 17]])==(3, [13, 15, 17])\n```",
"Mbpp/292": "Write a python function to find quotient of two numbers (rounded down to the nearest integer). Your code should satisfy the following assertion:\n```python\nassert find(10,3) == 3\n```",
"Mbpp/293": "Write a function to find the third side of a right angled triangle. Your code should satisfy the following assertion:\n```python\nassert otherside_rightangle(7,8)==10.63014581273465\n```",
"Mbpp/294": "Write a function to find the maximum value in a given heterogeneous list. Your code should satisfy the following assertion:\n```python\nassert max_val(['Python', 3, 2, 4, 5, 'version'])==5\n```",
"Mbpp/295": "Write a function to return the sum of all divisors of a number. Your code should satisfy the following assertion:\n```python\nassert sum_div(8)==7\n```",
"Mbpp/296": "Write a python function to count inversions in an array. Your code should satisfy the following assertion:\n```python\nassert get_Inv_Count([1,20,6,4,5]) == 5\n```",
"Mbpp/297": "Write a function to flatten a given nested list structure. Your code should satisfy the following assertion:\n```python\nassert flatten_list([0, 10, [20, 30], 40, 50, [60, 70, 80], [90, 100, 110, 120]])==[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]\n```",
"Mbpp/299": "Write a function to calculate the maximum aggregate from the list of tuples. Your code should satisfy the following assertion:\n```python\nassert max_aggregate([('Juan Whelan',90),('Sabah Colley',88),('Peter Nichols',7),('Juan Whelan',122),('Sabah Colley',84)])==('Juan Whelan', 212)\n```",
"Mbpp/300": "Write a function to find the count of all binary sequences of length 2n such that sum of first n bits is same as sum of last n bits. Your code should satisfy the following assertion:\n```python\nassert math.isclose(count_binary_seq(1), 2.0, rel_tol=0.001)\n```",
"Mbpp/301": "Write a function to find the depth of a dictionary. Your code should satisfy the following assertion:\n```python\nassert dict_depth({'a':1, 'b': {'c': {'d': {}}}})==4\n```",
"Mbpp/305": "Write a function to return two words from a list of words starting with letter 'p'. Your code should satisfy the following assertion:\n```python\nassert start_withp([\"Python PHP\", \"Java JavaScript\", \"c c++\"])==('Python', 'PHP')\n```",
"Mbpp/306": "Write a function to find the maximum sum of increasing subsequence from prefix until ith index and also including a given kth element which is after i, i.e., k > i . Your code should satisfy the following assertion:\n```python\nassert max_sum_increasing_subseq([1, 101, 2, 3, 100, 4, 5 ], 7, 4, 6) == 11\n```",
"Mbpp/308": "Write a function to find the specified number of largest products from two given lists, selecting one factor from each list. Your code should satisfy the following assertion:\n```python\nassert large_product([1, 2, 3, 4, 5, 6],[3, 6, 8, 9, 10, 6],3)==[60, 54, 50]\n```",
"Mbpp/309": "Write a python function to find the maximum of two numbers. Your code should satisfy the following assertion:\n```python\nassert maximum(5,10) == 10\n```",
"Mbpp/310": "Write a function to convert a given string to a tuple of characters. Your code should satisfy the following assertion:\n```python\nassert string_to_tuple(\"python 3.0\")==('p', 'y', 't', 'h', 'o', 'n', '3', '.', '0')\n```",
"Mbpp/311": "Write a python function to set the left most unset bit. Your code should satisfy the following assertion:\n```python\nassert set_left_most_unset_bit(10) == 14\n```",
"Mbpp/312": "Write a function to find the volume of a cone. Your code should satisfy the following assertion:\n```python\nassert math.isclose(volume_cone(5,12), 314.15926535897927, rel_tol=0.001)\n```",
"Mbpp/388": "Write a python function to find the highest power of 2 that is less than or equal to n. Your code should satisfy the following assertion:\n```python\nassert highest_Power_of_2(10) == 8\n```",
"Mbpp/389": "Write a function to find the n'th lucas number. Your code should satisfy the following assertion:\n```python\nassert find_lucas(9) == 76\n```",
"Mbpp/390": "Write a function to apply a given format string to all of the elements in a list. Your code should satisfy the following assertion:\n```python\nassert add_string([1,2,3,4],'temp{0}')==['temp1', 'temp2', 'temp3', 'temp4']\n```",
"Mbpp/391": "Write a function to convert more than one list to nested dictionary. Your code should satisfy the following assertion:\n```python\nassert convert_list_dictionary([\"S001\", \"S002\", \"S003\", \"S004\"],[\"Adina Park\", \"Leyton Marsh\", \"Duncan Boyle\", \"Saim Richards\"] ,[85, 98, 89, 92])==[{'S001': {'Adina Park': 85}}, {'S002': {'Leyton Marsh': 98}}, {'S003': {'Duncan Boyle': 89}}, {'S004': {'Saim Richards': 92}}]\n```",
"Mbpp/392": "Write a function to find the maximum sum possible by using the given equation f(n) = max( (f(n/2) + f(n/3) + f(n/4) + f(n/5)), n). Your code should satisfy the following assertion:\n```python\nassert get_max_sum(60) == 106\n```",
"Mbpp/394": "Write a function to check if given tuple contains no duplicates. Your code should satisfy the following assertion:\n```python\nassert check_distinct((1, 4, 5, 6, 1, 4)) == False\n```",
"Mbpp/395": "Write a python function to find the first non-repeated character in a given string. Your code should satisfy the following assertion:\n```python\nassert first_non_repeating_character(\"abcabc\") == None\n```",
"Mbpp/396": "Write a function to check whether the given string starts and ends with the same character or not. Your code should satisfy the following assertion:\n```python\nassert check_char(\"abba\") == \"Valid\"\n```",
"Mbpp/397": "Write a function to find the median of three numbers. Your code should satisfy the following assertion:\n```python\nassert median_numbers(25,55,65)==55.0\n```",
"Mbpp/398": "Write a function to compute the sum of digits of each number of a given list. Your code should satisfy the following assertion:\n```python\nassert sum_of_digits([10,2,56])==14\n```",
"Mbpp/400": "Write a function to extract the number of unique tuples in the given list. Your code should satisfy the following assertion:\n```python\nassert extract_freq([(3, 4), (1, 2), (4, 3), (5, 6)] ) == 3\n```",
"Mbpp/404": "Write a python function to find the minimum of two numbers. Your code should satisfy the following assertion:\n```python\nassert minimum(1,2) == 1\n```",
"Mbpp/405": "Write a function to check whether an element exists within a tuple. Your code should satisfy the following assertion:\n```python\nassert check_tuplex((\"w\", 3, \"r\", \"e\", \"s\", \"o\", \"u\", \"r\", \"c\", \"e\"),'r')==True\n```",
"Mbpp/406": "Write a python function to find whether the parity of a given number is odd. Your code should satisfy the following assertion:\n```python\nassert find_Parity(12) == False\n```",
"Mbpp/407": "Write a function to create the next bigger number by rearranging the digits of a given number. Your code should satisfy the following assertion:\n```python\nassert rearrange_bigger(12)==21\n```",
"Mbpp/409": "Write a function to find the minimum product from the pairs of tuples within a given list. Your code should satisfy the following assertion:\n```python\nassert min_product_tuple([(2, 7), (2, 6), (1, 8), (4, 9)] )==8\n```",
"Mbpp/410": "Write a function to find the minimum value in a given heterogeneous list. Your code should satisfy the following assertion:\n```python\nassert min_val(['Python', 3, 2, 4, 5, 'version'])==2\n```",
"Mbpp/412": "Write a python function to remove odd numbers from a given list. Your code should satisfy the following assertion:\n```python\nassert remove_odd([1,2,3]) == [2]\n```",
"Mbpp/413": "Write a function to extract the nth element from a given list of tuples. Your code should satisfy the following assertion:\n```python\nassert extract_nth_element([('Greyson Fulton', 98, 99), ('Brady Kent', 97, 96), ('Wyatt Knott', 91, 94), ('Beau Turnbull', 94, 98)] ,0)==['Greyson Fulton', 'Brady Kent', 'Wyatt Knott', 'Beau Turnbull']\n```",
"Mbpp/414": "Write a python function to check whether any value in a sequence exists in a sequence or not. Your code should satisfy the following assertion:\n```python\nassert overlapping([1,2,3,4,5],[6,7,8,9]) == False\n```",
"Mbpp/415": "Write a python function to find a pair with highest product from a given array of integers. Your code should satisfy the following assertion:\n```python\nassert max_Product([1,2,3,4,7,0,8,4]) == (7,8)\n```",
"Mbpp/418": "Write a python function to find the element of a list having maximum length. Your code should satisfy the following assertion:\n```python\nassert Find_Max([['A'],['A','B'],['A','B','C']]) == ['A','B','C']\n```",
"Mbpp/419": "Write a function to round every number of a given list of numbers and print the total sum multiplied by the length of the list. Your code should satisfy the following assertion:\n```python\nassert round_and_sum([22.4, 4.0, -16.22, -9.10, 11.00, -12.22, 14.20, -5.20, 17.50])==243\n```",
"Mbpp/420": "Write a python function to find the cube sum of first n even natural numbers. Your code should satisfy the following assertion:\n```python\nassert cube_Sum(2) == 72\n```",
"Mbpp/421": "Write a function to concatenate each element of tuple by the delimiter. Your code should satisfy the following assertion:\n```python\nassert concatenate_tuple((\"ID\", \"is\", 4, \"UTS\") ) == 'ID-is-4-UTS'\n```",
"Mbpp/422": "Write a python function to find the average of cubes of first n natural numbers. Your code should satisfy the following assertion:\n```python\nassert find_Average_Of_Cube(2) == 4.5\n```",
"Mbpp/424": "Write a function to extract only the rear index element of each string in the given tuple. Your code should satisfy the following assertion:\n```python\nassert extract_rear(('Mers', 'for', 'Vers') ) == ['s', 'r', 's']\n```",
"Mbpp/425": "Write a function to count the number of sublists containing a particular element. Your code should satisfy the following assertion:\n```python\nassert count_element_in_list([[1, 3], [5, 7], [1, 11], [1, 15, 7]],1)==3\n```",
"Mbpp/426": "Write a function to filter odd numbers. Your code should satisfy the following assertion:\n```python\nassert filter_oddnumbers([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])==[1,3,5,7,9]\n```",
"Mbpp/427": "Write a function to convert a date of yyyy-mm-dd format to dd-mm-yyyy format. Your code should satisfy the following assertion:\n```python\nassert change_date_format(\"2026-01-02\") == '02-01-2026'\n```",
"Mbpp/428": "Write a function to sort the given array by using shell sort. Your code should satisfy the following assertion:\n```python\nassert shell_sort([12, 23, 4, 5, 3, 2, 12, 81, 56, 95]) == [2, 3, 4, 5, 12, 12, 23, 56, 81, 95]\n```",
"Mbpp/429": "Write a function to extract the elementwise and tuples from the given two tuples. Your code should satisfy the following assertion:\n```python\nassert and_tuples((10, 4, 6, 9), (5, 2, 3, 3)) == (0, 0, 2, 1)\n```",
"Mbpp/430": "Write a function to find the directrix of a parabola. Your code should satisfy the following assertion:\n```python\nassert parabola_directrix(5,3,2)==-198\n```",
"Mbpp/431": "Write a function that takes two lists and returns true if they have at least one common element. Your code should satisfy the following assertion:\n```python\nassert common_element([1,2,3,4,5], [5,6,7,8,9])==True\n```",
"Mbpp/432": "Write a function to find the median length of a trapezium. Your code should satisfy the following assertion:\n```python\nassert median_trapezium(15,25,35)==20\n```",
"Mbpp/433": "Write a function to check whether the entered number is greater than the elements of the given array. Your code should satisfy the following assertion:\n```python\nassert check_greater([1, 2, 3, 4, 5], 4) == False\n```",
"Mbpp/435": "Write a python function to find the last digit of a given number. Your code should satisfy the following assertion:\n```python\nassert last_Digit(123) == 3\n```",
"Mbpp/436": "Write a python function to return the negative numbers in a list. Your code should satisfy the following assertion:\n```python\nassert neg_nos([-1,4,5,-6]) == [-1,-6]\n```",
"Mbpp/437": "Write a function to remove odd characters in a string. Your code should satisfy the following assertion:\n```python\nassert remove_odd(\"python\")==(\"yhn\")\n```",
"Mbpp/438": "Write a function to count bidirectional tuple pairs. Your code should satisfy the following assertion:\n```python\nassert count_bidirectional([(5, 6), (1, 2), (6, 5), (9, 1), (6, 5), (2, 1)] ) == 3\n```",
"Mbpp/439": "Write a function to join a list of multiple integers into a single integer. Your code should satisfy the following assertion:\n```python\nassert multiple_to_single([11, 33, 50])==113350\n```",
"Mbpp/440": "Write a function to find the first adverb and their positions in a given sentence. Your code should satisfy the following assertion:\n```python\nassert find_adverb_position(\"clearly!! we can see the sky\")==(0, 7, 'clearly')\n```",
"Mbpp/441": "Write a function to find the surface area of a cube of a given size. Your code should satisfy the following assertion:\n```python\nassert surfacearea_cube(5)==150\n```",
"Mbpp/442": "Write a function to find the ration of positive numbers in an array of integers. Your code should satisfy the following assertion:\n```python\nassert positive_count([0, 1, 2, -1, -5, 6, 0, -3, -2, 3, 4, 6, 8])==0.54\n```",
"Mbpp/445": "Write a function to perform index wise multiplication of tuple elements in the given two tuples. Your code should satisfy the following assertion:\n```python\nassert index_multiplication(((1, 3), (4, 5), (2, 9), (1, 10)),((6, 7), (3, 9), (1, 1), (7, 3)) ) == ((6, 21), (12, 45), (2, 9), (7, 30))\n```",
"Mbpp/446": "Write a python function to count the occurence of all elements of list in a tuple. Your code should satisfy the following assertion:\n```python\nassert count_Occurrence(('a', 'a', 'c', 'b', 'd'),['a', 'b'] ) == 3\n```",
"Mbpp/447": "Write a function to find cubes of individual elements in a list. Your code should satisfy the following assertion:\n```python\nassert cube_nums([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])==[1, 8, 27, 64, 125, 216, 343, 512, 729, 1000]\n```",
"Mbpp/448": "Write a function to calculate the sum of perrin numbers. Your code should satisfy the following assertion:\n```python\nassert cal_sum(9) == 49\n```",
"Mbpp/450": "Write a function to extract specified size of strings from a given list of string values. Your code should satisfy the following assertion:\n```python\nassert extract_string(['Python', 'list', 'exercises', 'practice', 'solution'] ,8)==['practice', 'solution']\n```",
"Mbpp/451": "Write a function to remove all whitespaces from the given string. Your code should satisfy the following assertion:\n```python\nassert remove_whitespaces(' Google Flutter ') == 'GoogleFlutter'\n```",
"Mbpp/453": "Write a python function to find the sum of even factors of a number. Your code should satisfy the following assertion:\n```python\nassert sumofFactors(18) == 26\n```",
"Mbpp/454": "Write a function that matches a word containing 'z'. Your code should satisfy the following assertion:\n```python\nassert text_match_wordz(\"pythonz.\")==True\n```",
"Mbpp/455": "Write a function to check whether the given month number contains 31 days or not. Your code should satisfy the following assertion:\n```python\nassert check_monthnumb_number(5)==True\n```",
"Mbpp/456": "Write a function to reverse each string in a given list of string values. Your code should satisfy the following assertion:\n```python\nassert reverse_string_list(['Red', 'Green', 'Blue', 'White', 'Black'])==['deR', 'neerG', 'eulB', 'etihW', 'kcalB']\n```",
"Mbpp/457": "Write a python function to find the sublist having minimum length. Your code should satisfy the following assertion:\n```python\nassert Find_Min([[1],[1,2],[1,2,3]]) == [1]\n```",
"Mbpp/458": "Write a function to find the area of a rectangle. Your code should satisfy the following assertion:\n```python\nassert rectangle_area(10,20)==200\n```",
"Mbpp/459": "Write a function to remove uppercase substrings from a given string. Your code should satisfy the following assertion:\n```python\nassert remove_uppercase('cAstyoUrFavoRitETVshoWs') == 'cstyoravoitshos'\n```",
"Mbpp/460": "Write a python function to get the first element of each sublist. Your code should satisfy the following assertion:\n```python\nassert Extract([[1, 2], [3, 4, 5], [6, 7, 8, 9]]) == [1, 3, 6]\n```",
"Mbpp/461": "Write a python function to count the upper case characters in a given string. Your code should satisfy the following assertion:\n```python\nassert upper_ctr('PYthon') == 1\n```",
"Mbpp/462": "Write a function to find all possible combinations of the elements of a given list. Your code should satisfy the following assertion:\n```python\nassert combinations_list(['orange', 'red', 'green', 'blue'])==[[], ['orange'], ['red'], ['red', 'orange'], ['green'], ['green', 'orange'], ['green', 'red'], ['green', 'red', 'orange'], ['blue'], ['blue', 'orange'], ['blue', 'red'], ['blue', 'red', 'orange'], ['blue', 'green'], ['blue', 'green', 'orange'], ['blue', 'green', 'red'], ['blue', 'green', 'red', 'orange']]\n```",
"Mbpp/463": "Write a function to find the maximum product subarray of the given array. Your code should satisfy the following assertion:\n```python\nassert max_subarray_product([1, -2, -3, 0, 7, -8, -2]) == 112\n```",
"Mbpp/465": "Write a function to drop empty items from a given dictionary. Your code should satisfy the following assertion:\n```python\nassert drop_empty({'c1': 'Red', 'c2': 'Green', 'c3':None})=={'c1': 'Red', 'c2': 'Green'}\n```",
"Mbpp/468": "Write a function to find the maximum product formed by multiplying numbers of an increasing subsequence of that array. Your code should satisfy the following assertion:\n```python\nassert max_product([3, 100, 4, 5, 150, 6]) == 3000\n```",
"Mbpp/470": "Write a function to find the pairwise addition of the neighboring elements of the given tuple. Your code should satisfy the following assertion:\n```python\nassert add_pairwise((1, 5, 7, 8, 10)) == (6, 12, 15, 18)\n```",
"Mbpp/471": "Write a python function to find the product of the array multiplication modulo n. Your code should satisfy the following assertion:\n```python\nassert find_remainder([ 100, 10, 5, 25, 35, 14 ],11) ==9\n```",
"Mbpp/472": "Write a python function to check whether the given list contains consecutive numbers or not. Your code should satisfy the following assertion:\n```python\nassert check_Consecutive([1,2,3,4,5]) == True\n```",
"Mbpp/473": "Write a function to find the tuple intersection of elements in the given tuple list irrespective of their order. Your code should satisfy the following assertion:\n```python\nassert tuple_intersection([(3, 4), (5, 6), (9, 10), (4, 5)] , [(5, 4), (3, 4), (6, 5), (9, 11)]) == {(4, 5), (3, 4), (5, 6)}\n```",
"Mbpp/474": "Write a function to replace characters in a string. Your code should satisfy the following assertion:\n```python\nassert replace_char(\"polygon\",'y','l')==(\"pollgon\")\n```",
"Mbpp/475": "Write a function to sort a dictionary by value. Your code should satisfy the following assertion:\n```python\nassert sort_counter({'Math':81, 'Physics':83, 'Chemistry':87})==[('Chemistry', 87), ('Physics', 83), ('Math', 81)]\n```",
"Mbpp/476": "Write a python function to find the sum of the largest and smallest value in a given array. Your code should satisfy the following assertion:\n```python\nassert big_sum([1,2,3]) == 4\n```",
"Mbpp/477": "Write a python function to convert the given string to lower case. Your code should satisfy the following assertion:\n```python\nassert is_lower(\"InValid\") == \"invalid\"\n```",
"Mbpp/478": "Write a function to remove lowercase substrings from a given string. Your code should satisfy the following assertion:\n```python\nassert remove_lowercase(\"PYTHon\")==('PYTH')\n```",
"Mbpp/479": "Write a python function to find the first digit of a given number. Your code should satisfy the following assertion:\n```python\nassert first_Digit(123) == 1\n```",
"Mbpp/554": "Write a python function which takes a list of integers and only returns the odd ones. Your code should satisfy the following assertion:\n```python\nassert Split([1,2,3,4,5,6]) == [1,3,5]\n```",
"Mbpp/555": "Write a python function to find the difference between the sum of cubes of the first n natural numbers and the sum of the first n natural numbers. Your code should satisfy the following assertion:\n```python\nassert difference(3) == 30\n```",
"Mbpp/556": "Write a python function to count the number of pairs whose xor value is odd. Your code should satisfy the following assertion:\n```python\nassert find_Odd_Pair([5,4,7,2,1],5) == 6\n```",
"Mbpp/557": "Write a function to toggle the case of all characters in a string. Your code should satisfy the following assertion:\n```python\nassert toggle_string(\"Python\")==(\"pYTHON\")\n```",
"Mbpp/558": "Write a python function to find the sum of the per-digit difference between two integers. Your code should satisfy the following assertion:\n```python\nassert digit_distance_nums(1,2) == 1\n```",
"Mbpp/559": "Write a function to find the sum of the largest contiguous sublist in the given list. Your code should satisfy the following assertion:\n```python\nassert max_sub_array_sum([-2, -3, 4, -1, -2, 1, 5, -3], 8) == 7\n```",
"Mbpp/560": "Write a function to find the union of the elements of two given tuples and output them in sorted order. Your code should satisfy the following assertion:\n```python\nassert union_elements((3, 4, 5, 6),(5, 7, 4, 10) ) == (3, 4, 5, 6, 7, 10)\n```",
"Mbpp/562": "Write a python function to find the length of the longest sublists. Your code should satisfy the following assertion:\n```python\nassert Find_Max_Length([[1],[1,4],[5,6,7,8]]) == 4\n```",
"Mbpp/563": "Write a function to extract values between quotation marks from a string. Your code should satisfy the following assertion:\n```python\nassert extract_values('\"Python\", \"PHP\", \"Java\"')==['Python', 'PHP', 'Java']\n```",
"Mbpp/564": "Write a python function which takes a list of integers and counts the number of possible unordered pairs where both elements are unequal. Your code should satisfy the following assertion:\n```python\nassert count_Pairs([1,2,1],3) == 2\n```",
"Mbpp/565": "Write a python function to split a string into characters. Your code should satisfy the following assertion:\n```python\nassert split('python') == ['p','y','t','h','o','n']\n```",
"Mbpp/566": "Write a function to get the sum of the digits of a non-negative integer. Your code should satisfy the following assertion:\n```python\nassert sum_digits(345)==12\n```",
"Mbpp/567": "Write a function to check whether a specified list is sorted or not. Your code should satisfy the following assertion:\n```python\nassert issort_list([1,2,4,6,8,10,12,14,16,17])==True\n```",
"Mbpp/568": "Write a function to create a list of N empty dictionaries. Your code should satisfy the following assertion:\n```python\nassert empty_list(5)==[{},{},{},{},{}]\n```",
"Mbpp/569": "Write a function to sort each sublist of strings in a given list of lists. Your code should satisfy the following assertion:\n```python\nassert sort_sublists([['green', 'orange'], ['black', 'white'], ['white', 'black', 'orange']])==[['green', 'orange'], ['black', 'white'], ['black', 'orange', 'white']]\n```",
"Mbpp/572": "Write a python function to remove duplicate numbers from a given number of lists. Your code should satisfy the following assertion:\n```python\nassert two_unique_nums([1,2,3,2,3,4,5]) == [1, 4, 5]\n```",
"Mbpp/573": "Write a python function to calculate the product of the unique numbers in a given list. Your code should satisfy the following assertion:\n```python\nassert unique_product([10, 20, 30, 40, 20, 50, 60, 40]) == 720000000\n```",
"Mbpp/574": "Write a function to find the surface area of a cylinder. Your code should satisfy the following assertion:\n```python\nassert surfacearea_cylinder(10,5)==942.45\n```",
"Mbpp/576": "Write a python function to check whether a list is sublist of another or not. Your code should satisfy the following assertion:\n```python\nassert is_Sub_Array([1,4,3,5],[1,2]) == False\n```",
"Mbpp/577": "Write a python function to find the last digit in factorial of a given number. Your code should satisfy the following assertion:\n```python\nassert last_Digit_Factorial(4) == 4\n```",
"Mbpp/578": "Write a function to interleave 3 lists of the same length into a single flat list. Your code should satisfy the following assertion:\n```python\nassert interleave_lists([1,2,3,4,5,6,7],[10,20,30,40,50,60,70],[100,200,300,400,500,600,700])==[1, 10, 100, 2, 20, 200, 3, 30, 300, 4, 40, 400, 5, 50, 500, 6, 60, 600, 7, 70, 700]\n```",
"Mbpp/579": "Write a function to find the dissimilar elements in the given two tuples. Your code should satisfy the following assertion:\n```python\nassert find_dissimilar((3, 4, 5, 6), (5, 7, 4, 10)) == (3, 6, 7, 10)\n```",
"Mbpp/580": "Write a function to remove uneven elements in the nested mixed tuple. Your code should satisfy the following assertion:\n```python\nassert extract_even((4, 5, (7, 6, (2, 4)), 6, 8)) == (4, (6, (2, 4)), 6, 8)\n```",
"Mbpp/581": "Write a python function to find the surface area of a square pyramid with a given base edge and height. Your code should satisfy the following assertion:\n```python\nassert surface_Area(3,4) == 33\n```",
"Mbpp/582": "Write a function to check if a dictionary is empty. Your code should satisfy the following assertion:\n```python\nassert my_dict({10})==False\n```",
"Mbpp/583": "Write a function which returns nth catalan number. Your code should satisfy the following assertion:\n```python\nassert catalan_number(10)==16796\n```",
"Mbpp/585": "Write a function to find the n most expensive items in a given dataset. Your code should satisfy the following assertion:\n```python\nassert expensive_items([{'name': 'Item-1', 'price': 101.1},{'name': 'Item-2', 'price': 555.22}],1)==[{'name': 'Item-2', 'price': 555.22}]\n```",
"Mbpp/586": "Write a python function to split a list at the nth eelment and add the first part to the end. Your code should satisfy the following assertion:\n```python\nassert split_Arr([12,10,5,6,52,36],2) == [5,6,52,36,12,10]\n```",
"Mbpp/587": "Write a function to convert a list to a tuple. Your code should satisfy the following assertion:\n```python\nassert list_tuple([5, 10, 7, 4, 15, 3])==(5, 10, 7, 4, 15, 3)\n```",
"Mbpp/588": "Write a python function to find the difference between largest and smallest value in a given list. Your code should satisfy the following assertion:\n```python\nassert big_diff([1,2,3,4]) == 3\n```",
"Mbpp/589": "Write a function to find perfect squares between two given numbers. Your code should satisfy the following assertion:\n```python\nassert perfect_squares(1,30)==[1, 4, 9, 16, 25]\n```",
"Mbpp/590": "Write a function to convert polar coordinates to rectangular coordinates. Your code should satisfy the following assertion:\n```python\nassert polar_rect(3,4)==((5.0, 0.9272952180016122), (-2+2.4492935982947064e-16j))\n```",
"Mbpp/591": "Write a python function to interchange the first and last elements in a list. Your code should satisfy the following assertion:\n```python\nassert swap_List([12, 35, 9, 56, 24]) == [24, 35, 9, 56, 12]\n```",
"Mbpp/592": "Write a python function to find the sum of the product of consecutive binomial co-efficients. Your code should satisfy the following assertion:\n```python\nassert sum_Of_product(3) == 15\n```",
"Mbpp/593": "Write a function to remove leading zeroes from an ip address. Your code should satisfy the following assertion:\n```python\nassert removezero_ip(\"216.08.094.196\")==('216.8.94.196')\n```",
"Mbpp/594": "Write a function to find the difference of the first even and first odd number of a given list. Your code should satisfy the following assertion:\n```python\nassert diff_even_odd([1,3,5,7,4,1,6,8])==3\n```",
"Mbpp/595": "Write a python function to count minimum number of swaps required to convert one binary number represented as a string to another. Your code should satisfy the following assertion:\n```python\nassert min_Swaps(\"1101\",\"1110\") == 1\n```",
"Mbpp/596": "Write a function to find the size in bytes of the given tuple. Your code should satisfy the following assertion:\n```python\nassert tuple_size((\"A\", 1, \"B\", 2, \"C\", 3) ) == sys.getsizeof((\"A\", 1, \"B\", 2, \"C\", 3))\n```",
"Mbpp/597": "Write a function to find kth element from the given two sorted arrays. Your code should satisfy the following assertion:\n```python\nassert find_kth([2, 3, 6, 7, 9], [1, 4, 8, 10], 5) == 6\n```",
"Mbpp/598": "Write a function to check whether the given number is armstrong or not. Your code should satisfy the following assertion:\n```python\nassert armstrong_number(153)==True\n```",
"Mbpp/599": "Write a function to find sum and average of first n natural numbers. Your code should satisfy the following assertion:\n```python\nassert sum_average(10)==(55, 5.5)\n```",
"Mbpp/600": "Write a python function to check whether the given number is even or not. Your code should satisfy the following assertion:\n```python\nassert is_Even(1) == False\n```",
"Mbpp/602": "Write a python function to find the first repeated character in a given string. Your code should satisfy the following assertion:\n```python\nassert first_repeated_char(\"abcabc\") == \"a\"\n```",
"Mbpp/603": "Write a function to get all lucid numbers smaller than or equal to a given integer. Your code should satisfy the following assertion:\n```python\nassert get_ludic(10) == [1, 2, 3, 5, 7]\n```",
"Mbpp/604": "Write a function to reverse words seperated by spaces in a given string. Your code should satisfy the following assertion:\n```python\nassert reverse_words(\"python program\")==(\"program python\")\n```",
"Mbpp/605": "Write a function to check if the given integer is a prime number. Your code should satisfy the following assertion:\n```python\nassert prime_num(13)==True\n```",
"Mbpp/606": "Write a function to convert degrees to radians. Your code should satisfy the following assertion:\n```python\nassert radian_degree(90)==1.5707963267948966\n```",
"Mbpp/607": "Write a function to search a string for a regex pattern. The function should return the matching subtring, a start index and an end index. Your code should satisfy the following assertion:\n```python\nassert find_literals('The quick brown fox jumps over the lazy dog.', 'fox') == ('fox', 16, 19)\n```",
"Mbpp/608": "Write a python function to find nth bell number. Your code should satisfy the following assertion:\n```python\nassert bell_Number(2) == 2\n```",
"Mbpp/610": "Write a python function which takes a list and returns a list with the same elements, but the k'th element removed. Your code should satisfy the following assertion:\n```python\nassert remove_kth_element([1,1,2,3,4,4,5,1],3)==[1, 1, 3, 4, 4, 5, 1]\n```",
"Mbpp/611": "Write a function which given a matrix represented as a list of lists returns the max of the n'th column. Your code should satisfy the following assertion:\n```python\nassert max_of_nth([[5, 6, 7], [1, 3, 5], [8, 9, 19]], 2) == 19\n```",
"Mbpp/612": "Write a python function which takes a list of lists, where each sublist has two elements, and returns a list of two lists where the first list has the first element of each sublist and the second one has the second. Your code should satisfy the following assertion:\n```python\nassert merge([['x', 'y'], ['a', 'b'], ['m', 'n']]) == [['x', 'a', 'm'], ['y', 'b', 'n']]\n```",
"Mbpp/614": "Write a function to find the cumulative sum of all the values that are present in the given tuple list. Your code should satisfy the following assertion:\n```python\nassert cummulative_sum([(1, 3), (5, 6, 7), (2, 6)]) == 30\n```",
"Mbpp/615": "Write a function which takes a tuple of tuples and returns the average value for each tuple as a list. Your code should satisfy the following assertion:\n```python\nassert average_tuple(((10, 10, 10, 12), (30, 45, 56, 45), (81, 80, 39, 32), (1, 2, 3, 4)))==[30.5, 34.25, 27.0, 23.25]\n```",
"Mbpp/616": "Write a function which takes two tuples of the same length and performs the element wise modulo. Your code should satisfy the following assertion:\n```python\nassert tuple_modulo((10, 4, 5, 6), (5, 6, 7, 5)) == (0, 4, 5, 1)\n```",
"Mbpp/618": "Write a function to divide two lists element wise. Your code should satisfy the following assertion:\n```python\nassert div_list([4,5,6],[1, 2, 3])==[4.0,2.5,2.0]\n```",
"Mbpp/619": "Write a function to move all the numbers to the end of the given string. Your code should satisfy the following assertion:\n```python\nassert move_num('I1love143you55three3000thousand') == 'Iloveyouthreethousand1143553000'\n```",
"Mbpp/620": "Write a function to find the size of the largest subset of a list of numbers so that every pair is divisible. Your code should satisfy the following assertion:\n```python\nassert largest_subset([ 1, 3, 6, 13, 17, 18 ]) == 4\n```",
"Mbpp/622": "Write a function to find the median of two sorted lists of same size. Your code should satisfy the following assertion:\n```python\nassert get_median([1, 12, 15, 26, 38], [2, 13, 17, 30, 45], 5) == 16.0\n```",
"Mbpp/623": "Write a function to compute the n-th power of each number in a list. Your code should satisfy the following assertion:\n```python\nassert nth_nums([1, 2, 3, 4, 5, 6, 7, 8, 9, 10],2)==[1, 4, 9, 16, 25, 36, 49, 64, 81, 100]\n```",
"Mbpp/624": "Write a python function to convert a given string to uppercase. Your code should satisfy the following assertion:\n```python\nassert is_upper(\"person\") ==\"PERSON\"\n```",
"Mbpp/626": "Write a python function to find the area of the largest triangle that can be inscribed in a semicircle with a given radius. Your code should satisfy the following assertion:\n```python\nassert triangle_area(-1) == None\n```",
"Mbpp/628": "Write a function to replace all spaces in the given string with '%20'. Your code should satisfy the following assertion:\n```python\nassert replace_spaces(\"My Name is Dawood\") == 'My%20Name%20is%20Dawood'\n```",
"Mbpp/629": "Write a python function to find even numbers from a list of numbers. Your code should satisfy the following assertion:\n```python\nassert Split([1,2,3,4,5]) == [2,4]\n```",
"Mbpp/630": "Write a function to extract all the adjacent coordinates of the given coordinate tuple. Your code should satisfy the following assertion:\n```python\nassert get_coordinates((3, 4)) == [[2, 3], [2, 4], [2, 5], [3, 3], [3, 4], [3, 5], [4, 3], [4, 4], [4, 5]]\n```",
"Mbpp/631": "Write a function to replace whitespaces with an underscore and vice versa in a given string. Your code should satisfy the following assertion:\n```python\nassert replace_spaces('Jumanji The Jungle') == 'Jumanji_The_Jungle'\n```",
"Mbpp/632": "Write a python function to move all zeroes to the end of the given list. Your code should satisfy the following assertion:\n```python\nassert move_zero([1,0,2,0,3,4]) == [1,2,3,4,0,0]\n```",
"Mbpp/633": "Write a python function to find the sum of xor of all pairs of numbers in the given list. Your code should satisfy the following assertion:\n```python\nassert pair_xor_Sum([5,9,7,6],4) == 47\n```",
"Mbpp/635": "Write a function to sort the given list. Your code should satisfy the following assertion:\n```python\nassert heap_sort([1, 3, 5, 7, 9, 2, 4, 6, 8, 0])==[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n```",
"Mbpp/637": "Write a function to check whether the given amount has no profit and no loss. Your code should satisfy the following assertion:\n```python\nassert noprofit_noloss(1500,1200)==False\n```",
"Mbpp/638": "Write a function to calculate the wind chill index rounded to the next integer given the wind velocity in km/h and a temperature in celsius. Your code should satisfy the following assertion:\n```python\nassert wind_chill(120,35)==40\n```",
"Mbpp/639": "Write a function to sum the length of the names of a given list of names after removing the names that start with a lowercase letter. Your code should satisfy the following assertion:\n```python\nassert sample_nam(['sally', 'Dylan', 'rebecca', 'Diana', 'Joanne', 'keith'])==16\n```",
"Mbpp/640": "Write a function to remove the parenthesis and what is inbetween them from a string. Your code should satisfy the following assertion:\n```python\nassert remove_parenthesis([\"python (chrome)\"])==(\"python\")\n```",
"Mbpp/641": "Write a function to find the nth nonagonal number. Your code should satisfy the following assertion:\n```python\nassert is_nonagonal(10) == 325\n```",
"Mbpp/643": "Write a function that checks if a strings contains 'z', except at the start and end of the word. Your code should satisfy the following assertion:\n```python\nassert text_match_wordz_middle(\"pythonzabc.\")==True\n```",
"Mbpp/644": "Write a python function to reverse an array upto a given position. Your code should satisfy the following assertion:\n```python\nassert reverse_Array_Upto_K([1, 2, 3, 4, 5, 6],4) == [4, 3, 2, 1, 5, 6]\n```",
"Mbpp/720": "Write a function to add a dictionary to the tuple. The output should be a tuple. Your code should satisfy the following assertion:\n```python\nassert add_dict_to_tuple((4, 5, 6), {\"MSAM\" : 1, \"is\" : 2, \"best\" : 3} ) == (4, 5, 6, {'MSAM': 1, 'is': 2, 'best': 3})\n```",
"Mbpp/721": "Given a square matrix of size N*N given as a list of lists, where each cell is associated with a specific cost. A path is defined as a specific sequence of cells that starts from the top-left cell move only right or down and ends on bottom right cell. We want to find a path with the maximum average over all existing paths. Average is computed as total cost divided by the number of cells visited in the path. Your code should satisfy the following assertion:\n```python\nassert maxAverageOfPath([[1, 2, 3], [6, 5, 4], [7, 3, 9]]) == 5.2\n```",
"Mbpp/722": "The input is given as - a dictionary with a student name as a key and a tuple of float (student_height, student_weight) as a value, - minimal height, - minimal weight. Write a function to filter students that have height and weight above the minimum. Your code should satisfy the following assertion:\n```python\nassert filter_data({'Cierra Vega': (6.2, 70), 'Alden Cantrell': (5.9, 65), 'Kierra Gentry': (6.0, 68), 'Pierre Cox': (5.8, 66)},6.0,70)=={'Cierra Vega': (6.2, 70)}\n```",
"Mbpp/723": "The input is defined as two lists of the same length. Write a function to count indices where the lists have the same values. Your code should satisfy the following assertion:\n```python\nassert count_same_pair([1, 2, 3, 4, 5, 6, 7, 8],[2, 2, 3, 1, 2, 6, 7, 9])==4\n```",
"Mbpp/724": "Write a function that takes base and power as arguments and calculate the sum of all digits of the base to the specified power. Your code should satisfy the following assertion:\n```python\nassert power_base_sum(2,100)==115\n```",
"Mbpp/725": "Write a function to extract values between quotation marks \" \" of the given string. Your code should satisfy the following assertion:\n```python\nassert extract_quotation('Cortex \"A53\" Based \"multi\" tasking \"Processor\"') == ['A53', 'multi', 'Processor']\n```",
"Mbpp/726": "Write a function that takes as input a tuple of numbers (t_1,...,t_{N+1}) and returns a tuple of length N where the i-th element of the tuple is equal to t_i * t_{i+1}. Your code should satisfy the following assertion:\n```python\nassert multiply_elements((1, 5, 7, 8, 10)) == (5, 35, 56, 80)\n```",
"Mbpp/728": "Write a function takes as input two lists [a_1,...,a_n], [b_1,...,b_n] and returns [a_1+b_1,...,a_n+b_n]. Your code should satisfy the following assertion:\n```python\nassert sum_list([10,20,30],[15,25,35])==[25,45,65]\n```",
"Mbpp/730": "Write a function to remove consecutive duplicates of a given list. Your code should satisfy the following assertion:\n```python\nassert consecutive_duplicates([0, 0, 1, 2, 3, 4, 4, 5, 6, 6, 6, 7, 8, 9, 4, 4 ])==[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4]\n```",
"Mbpp/731": "Write a function to find the lateral surface area of a cone given radius r and the height h. Your code should satisfy the following assertion:\n```python\nassert lateralsurface_cone(5,12)==204.20352248333654\n```",
"Mbpp/732": "Write a function to replace all occurrences of spaces, commas, or dots with a colon. Your code should satisfy the following assertion:\n```python\nassert replace_specialchar('Python language, Programming language.')==('Python:language::Programming:language:')\n```",
"Mbpp/733": "Write a function to find the index of the first occurrence of a given number in a sorted array. Your code should satisfy the following assertion:\n```python\nassert find_first_occurrence([2, 5, 5, 5, 6, 6, 8, 9, 9, 9], 5) == 1\n```",
"Mbpp/734": "Write a python function to find sum of products of all possible sublists of a given list. https://www.geeksforgeeks.org/sum-of-products-of-all-possible-subarrays/. Your code should satisfy the following assertion:\n```python\nassert sum_Of_Subarray_Prod([1,2,3]) == 20\n```",
"Mbpp/735": "Write a python function to toggle bits of the number except the first and the last bit. https://www.geeksforgeeks.org/toggle-bits-number-expect-first-last-bits/. Your code should satisfy the following assertion:\n```python\nassert toggle_middle_bits(9) == 15\n```",
"Mbpp/736": "Write a function to locate the left insertion point for a specified value in sorted order. https://www.w3resource.com/python-exercises/data-structures-and-algorithms/python-data-structure-exercise-24.php. Your code should satisfy the following assertion:\n```python\nassert left_insertion([1,2,4,5],6)==4\n```",
"Mbpp/737": "Write a function to check whether the given string is starting with a vowel or not using regex. Your code should satisfy the following assertion:\n```python\nassert check_str(\"annie\")\n```",
"Mbpp/739": "Write a python function to find the index of smallest triangular number with n digits. https://www.geeksforgeeks.org/index-of-smallest-triangular-number-with-n-digits/. Your code should satisfy the following assertion:\n```python\nassert find_Index(2) == 4\n```",
"Mbpp/740": "Write a function to convert the given tuple to a key-value dictionary using adjacent elements. https://www.geeksforgeeks.org/python-convert-tuple-to-adjacent-pair-dictionary/. Your code should satisfy the following assertion:\n```python\nassert tuple_to_dict((1, 5, 7, 10, 13, 5)) == {1: 5, 7: 10, 13: 5}\n```",
"Mbpp/741": "Write a python function to check whether all the characters are same or not. Your code should satisfy the following assertion:\n```python\nassert all_Characters_Same(\"python\") == False\n```",
"Mbpp/742": "Write a function to caluclate the area of a tetrahedron. Your code should satisfy the following assertion:\n```python\nassert area_tetrahedron(3)==15.588457268119894\n```",
"Mbpp/743": "Write a function to rotate a given list by specified number of items to the right direction. https://www.geeksforgeeks.org/python-program-right-rotate-list-n/. Your code should satisfy the following assertion:\n```python\nassert rotate_right([1, 2, 3, 4, 5, 6, 7, 8, 9, 10],3)==[8, 9, 10, 1, 2, 3, 4, 5, 6, 7]\n```",
"Mbpp/744": "Write a function to check if the given tuple has any none value or not. Your code should satisfy the following assertion:\n```python\nassert check_none((10, 4, 5, 6, None)) == True\n```",
"Mbpp/745": "Write a function to find numbers within a given range from startnum ti endnum where every number is divisible by every digit it contains. https://www.w3resource.com/python-exercises/lambda/python-lambda-exercise-24.php. Your code should satisfy the following assertion:\n```python\nassert divisible_by_digits(1,22)==[1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 15, 22]\n```",
"Mbpp/746": "Write a function to find area of a sector. The function takes the radius and angle as inputs. Function should return None if the angle is larger than 360 degrees. Your code should satisfy the following assertion:\n```python\nassert sector_area(4,45)==6.283185307179586\n```",
"Mbpp/748": "Write a function to put spaces between words starting with capital letters in a given string. Your code should satisfy the following assertion:\n```python\nassert capital_words_spaces(\"Python\") == 'Python'\n```",
"Mbpp/749": "Write a function to sort a given list of strings of numbers numerically. https://www.geeksforgeeks.org/python-sort-numeric-strings-in-a-list/. Your code should satisfy the following assertion:\n```python\nassert sort_numeric_strings( ['4','12','45','7','0','100','200','-12','-500'])==[-500, -12, 0, 4, 7, 12, 45, 100, 200]\n```",
"Mbpp/750": "Write a function to add the given tuple to the given list. Your code should satisfy the following assertion:\n```python\nassert add_tuple([5, 6, 7], (9, 10)) == [5, 6, 7, 9, 10]\n```",
"Mbpp/751": "Write a function to check if the given array represents min heap or not. https://www.geeksforgeeks.org/how-to-check-if-a-given-array-represents-a-binary-heap/. Your code should satisfy the following assertion:\n```python\nassert check_min_heap([1, 2, 3, 4, 5, 6]) == True\n```",
"Mbpp/752": "Write a function to find the nth jacobsthal number. https://www.geeksforgeeks.org/jacobsthal-and-jacobsthal-lucas-numbers/ 0, 1, 1, 3, 5, 11, 21, 43, 85, 171, 341, 683, 1365, 2731, ... Your code should satisfy the following assertion:\n```python\nassert jacobsthal_num(5) == 11\n```",
"Mbpp/753": "Write a function to find minimum k records from tuple list. https://www.geeksforgeeks.org/python-find-minimum-k-records-from-tuple-list/ - in this case a verbatim copy of test cases. Your code should satisfy the following assertion:\n```python\nassert min_k([('Manjeet', 10), ('Akshat', 4), ('Akash', 2), ('Nikhil', 8)], 2) == [('Akash', 2), ('Akshat', 4)]\n```",
"Mbpp/754": "We say that an element is common for lists l1, l2, l3 if it appears in all three lists under the same index. Write a function to find common elements from three lists. The function should return a list. Your code should satisfy the following assertion:\n```python\nassert extract_index_list([1, 1, 3, 4, 5, 6, 7],[0, 1, 2, 3, 4, 5, 7],[0, 1, 2, 3, 4, 5, 7])==[1, 7]\n```",
"Mbpp/755": "Write a function to find the second smallest number in a list. Your code should satisfy the following assertion:\n```python\nassert second_smallest([1, 2, -8, -2, 0, -2])==-2\n```",
"Mbpp/757": "Write a function to count the pairs of reverse strings in the given string list. https://www.geeksforgeeks.org/python-program-to-count-the-pairs-of-reverse-strings/. Your code should satisfy the following assertion:\n```python\nassert count_reverse_pairs([\"julia\", \"best\", \"tseb\", \"for\", \"ailuj\"])== 2\n```",
"Mbpp/758": "Write a function to count lists within a list. The function should return a dictionary where every list is converted to a tuple and the value of such tuple is the number of its occurencies in the original list. Your code should satisfy the following assertion:\n```python\nassert unique_sublists([[1, 3], [5, 7], [1, 3], [13, 15, 17], [5, 7], [9, 11]] )=={(1, 3): 2, (5, 7): 2, (13, 15, 17): 1, (9, 11): 1}\n```",
"Mbpp/759": "Write a function to check whether a given string is a decimal number with a precision of 2. Your code should satisfy the following assertion:\n```python\nassert is_decimal('123.11')==True\n```",
"Mbpp/760": "Write a python function to check whether a list of numbers contains only one distinct element or not. Your code should satisfy the following assertion:\n```python\nassert unique_Element([1,1,1]) == True\n```",
"Mbpp/762": "Write a function to check whether the given month number contains 30 days or not. Months are given as number from 1 to 12. Your code should satisfy the following assertion:\n```python\nassert check_monthnumber_number(6)==True\n```",
"Mbpp/763": "Write a python function to find the minimum difference between any two elements in a given array. https://www.geeksforgeeks.org/find-minimum-difference-pair/. Your code should satisfy the following assertion:\n```python\nassert find_min_diff((1,5,3,19,18,25),6) == 1\n```",
"Mbpp/764": "Write a python function to count number of digits in a given string. Your code should satisfy the following assertion:\n```python\nassert number_ctr('program2bedone') == 1\n```",
"Mbpp/765": "Write a function to find nth polite number. geeksforgeeks.org/n-th-polite-number/. Your code should satisfy the following assertion:\n```python\nassert is_polite(7) == 11\n```",
"Mbpp/766": "Write a function to return a list of all pairs of consecutive items in a given list. Your code should satisfy the following assertion:\n```python\nassert pair_wise([1,1,2,3,3,4,4,5])==[(1, 1), (1, 2), (2, 3), (3, 3), (3, 4), (4, 4), (4, 5)]\n```",
"Mbpp/767": "Write a python function to count the number of pairs whose sum is equal to \u2018sum\u2019. The funtion gets as input a list of numbers and the sum,. Your code should satisfy the following assertion:\n```python\nassert get_pairs_count([1,1,1,1],2) == 6\n```",
"Mbpp/769": "Write a python function to get the difference between two lists. Your code should satisfy the following assertion:\n```python\nassert (Diff([10, 15, 20, 25, 30, 35, 40], [25, 40, 35])) == [10, 20, 30, 15]\n```",
"Mbpp/770": "Write a python function to find the sum of fourth power of first n odd natural numbers. Your code should satisfy the following assertion:\n```python\nassert odd_num_sum(2) == 82\n```",
"Mbpp/771": "Write a function to check if the given expression is balanced or not. https://www.geeksforgeeks.org/check-for-balanced-parentheses-in-an-expression/. Your code should satisfy the following assertion:\n```python\nassert check_expression(\"{()}[{}]\") == True\n```",
"Mbpp/772": "Write a function to remove all the words with k length in the given string. Your code should satisfy the following assertion:\n```python\nassert remove_length('The person is most value tet', 3) == 'person is most value'\n```",
"Mbpp/773": "Write a function to find the occurrence and position of the substrings within a string. Return None if there is no match. Your code should satisfy the following assertion:\n```python\nassert occurance_substring('python programming, python language','python')==('python', 0, 6)\n```",
"Mbpp/775": "Write a python function to check whether every odd index contains odd numbers of a given list. Your code should satisfy the following assertion:\n```python\nassert odd_position([2,1,4,3,6,7,6,3]) == True\n```",
"Mbpp/777": "Write a python function to find the sum of non-repeated elements in a given list. Your code should satisfy the following assertion:\n```python\nassert find_sum([1,2,3,1,1,4,5,6]) == 21\n```",
"Mbpp/778": "Write a function to pack consecutive duplicates of a given list elements into sublists. Your code should satisfy the following assertion:\n```python\nassert pack_consecutive_duplicates([0, 0, 1, 2, 3, 4, 4, 5, 6, 6, 6, 7, 8, 9, 4, 4])==[[0, 0], [1], [2], [3], [4, 4], [5], [6, 6, 6], [7], [8], [9], [4, 4]]\n```",
"Mbpp/780": "Write a function to find the combinations of sums with tuples in the given tuple list. https://www.geeksforgeeks.org/python-combinations-of-sum-with-tuples-in-tuple-list/. Your code should satisfy the following assertion:\n```python\nassert find_combinations([(2, 4), (6, 7), (5, 1), (6, 10)]) == [(8, 11), (7, 5), (8, 14), (11, 8), (12, 17), (11, 11)]\n```",
"Mbpp/781": "Write a python function to check whether the count of divisors is even. https://www.w3resource.com/python-exercises/basic/python-basic-1-exercise-24.php. Your code should satisfy the following assertion:\n```python\nassert count_divisors(10)\n```",
"Mbpp/782": "Write a python function to find the sum of all odd length subarrays. https://www.geeksforgeeks.org/sum-of-all-odd-length-subarrays/. Your code should satisfy the following assertion:\n```python\nassert odd_length_sum([1,2,4]) == 14\n```",
"Mbpp/783": "Write a function to convert rgb color to hsv color. https://www.geeksforgeeks.org/program-change-rgb-color-model-hsv-color-model/. Your code should satisfy the following assertion:\n```python\nassert rgb_to_hsv(255, 255, 255)==(0, 0.0, 100.0)\n```",
"Mbpp/784": "Write a function to find the product of first even and odd number of a given list. Your code should satisfy the following assertion:\n```python\nassert mul_even_odd([1,3,5,7,4,1,6,8])==4\n```",
"Mbpp/785": "Write a function to convert tuple string to integer tuple. Your code should satisfy the following assertion:\n```python\nassert tuple_str_int(\"(7, 8, 9)\") == (7, 8, 9)\n```",
"Mbpp/786": "Write a function to locate the right insertion point for a specified value in sorted order. Your code should satisfy the following assertion:\n```python\nassert right_insertion([1,2,4,5],6)==4\n```",
"Mbpp/787": "Write a function that matches a string that has an a followed by three 'b'. Your code should satisfy the following assertion:\n```python\nassert not text_match_three(\"ac\")\n```",
"Mbpp/788": "Write a function to create a new tuple from the given string and list. Your code should satisfy the following assertion:\n```python\nassert new_tuple([\"WEB\", \"is\"], \"best\") == ('WEB', 'is', 'best')\n```",
"Mbpp/790": "Write a python function to check whether every even index contains even numbers of a given list. Your code should satisfy the following assertion:\n```python\nassert even_position([3,2,1]) == False\n```",
"Mbpp/791": "Write a function to remove tuples from the given tuple. Your code should satisfy the following assertion:\n```python\nassert remove_nested((1, 5, 7, (4, 6), 10)) == (1, 5, 7, 10)\n```",
"Mbpp/792": "Write a python function to count the number of lists in a given number of lists. Your code should satisfy the following assertion:\n```python\nassert count_list([[1, 3], [5, 7], [9, 11], [13, 15, 17]]) == 4\n```",
"Mbpp/793": "Write a python function to find the last position of an element in a sorted array. Your code should satisfy the following assertion:\n```python\nassert last([1,2,3],1) == 0\n```",
"Mbpp/794": "Write a function that matches a string that has an 'a' followed by anything, ending in 'b'. Your code should satisfy the following assertion:\n```python\nassert text_starta_endb(\"aabbbb\")\n```",
"Mbpp/796": "Write function to find the sum of all items in the given dictionary. Your code should satisfy the following assertion:\n```python\nassert return_sum({'a': 100, 'b':200, 'c':300}) == 600\n```",
"Mbpp/797": "Write a python function to find the sum of all odd natural numbers within the range l and r. Your code should satisfy the following assertion:\n```python\nassert sum_in_range(2,5) == 8\n```",
"Mbpp/798": "Write a python function to find the sum of an array. Your code should satisfy the following assertion:\n```python\nassert _sum([1, 2, 3]) == 6\n```",
"Mbpp/799": "Write a function to that rotate left bits by d bits a given number. We assume that the number is 32 bit. Your code should satisfy the following assertion:\n```python\nassert left_rotate(16,2) == 64\n```",
"Mbpp/800": "Write a function to remove all whitespaces from a string. Your code should satisfy the following assertion:\n```python\nassert remove_all_spaces('python program')==('pythonprogram')\n```",
"Mbpp/801": "Write a python function to count the number of equal numbers from three given integers. Your code should satisfy the following assertion:\n```python\nassert test_three_equal(1,1,1) == 3\n```",
"Mbpp/803": "Write a function to check whether the given number is a perfect square or not. https://www.geeksforgeeks.org/check-if-given-number-is-perfect-square-in-cpp/. Your code should satisfy the following assertion:\n```python\nassert not is_perfect_square(10)\n```",
"Mbpp/804": "Write a function to check whether the product of numbers in a list is even or not. Your code should satisfy the following assertion:\n```python\nassert is_product_even([1,2,3])\n```",
"Mbpp/805": "Write a function that returns the list in a list of lists whose sum of elements is the highest. Your code should satisfy the following assertion:\n```python\nassert max_sum_list([[1,2,3], [4,5,6], [10,11,12], [7,8,9]])==[10, 11, 12]\n```",
"Mbpp/806": "Write a function to find maximum run of uppercase characters in the given string. Your code should satisfy the following assertion:\n```python\nassert max_run_uppercase('GeMKSForGERksISBESt') == 5\n```",
"Mbpp/807": "Write a python function to find the first odd number in a given list of numbers. Your code should satisfy the following assertion:\n```python\nassert first_odd([1,3,5]) == 1\n```",
"Mbpp/808": "Write a function to check if the given tuples contain the k or not. Your code should satisfy the following assertion:\n```python\nassert check_K((10, 4, 5, 6, 8), 6) == True\n```",
"Mbpp/809": "Write a function to check if each element of second tuple is smaller than its corresponding element in the first tuple. Your code should satisfy the following assertion:\n```python\nassert check_smaller((1, 2, 3), (2, 3, 4)) == False\n```"
}
================================================
FILE: WizardCoder/src/humaneval_gen.py
================================================
import argparse
import pprint
import sys
import os
import re
from tqdm import tqdm
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
from human_eval.data import write_jsonl, read_problems, stream_jsonl
if torch.cuda.is_available():
device = "cuda"
else:
device = "cpu"
try:
if torch.backends.mps.is_available():
device = "mps"
except:
pass
def generate_prompt(input):
INSTRUCTION = f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
### Instruction:
Create a Python script for this problem:
{input}
### Response:"""
return INSTRUCTION
def get_model(
load_8bit: bool = False,
base_model: str = "bigcode/starcoder",
):
assert base_model, (
"Please specify a --base_model, e.g. --base_model='bigcode/starcoder'"
)
tokenizer = AutoTokenizer.from_pretrained(base_model)
if device == "cuda":
model = AutoModelForCausalLM.from_pretrained(
base_model,
load_in_8bit=load_8bit,
torch_dtype=torch.float16,
device_map="auto",
)
elif device == "mps":
model = AutoModelForCausalLM.from_pretrained(
base_model,
device_map={"": device},
torch_dtype=torch.bfloat16,
)
model.config.pad_token_id = tokenizer.pad_token_id
if not load_8bit:
model.half() # seems to fix bugs for some users.
model.eval()
if torch.__version__ >= "2" and sys.platform != "win32":
model = torch.compile(model)
return tokenizer, model
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model', type=str, default='bigcode/starcoder', help="")
parser.add_argument('--output_path', type=str, help="")
parser.add_argument('--start_index', type=int, default=0, help="")
parser.add_argument('--end_index', type=int, default=164, help="")
parser.add_argument('--temperature', type=float, default=0.8, help="")
parser.add_argument('--N', type=int, default=200, help="")
parser.add_argument('--max_len', type=int, default=512, help="")
parser.add_argument('--decoding_style', type=str, default='sampling', help="")
parser.add_argument('--num_seqs_per_iter', type=int, default=50, help='')
parser.add_argument('--greedy_decode', action='store_true', help='')
parser.add_argument('--overwrite', action='store_true', help='')
args = parser.parse_args()
argsdict = vars(args)
print(pprint.pformat(argsdict))
problems = read_problems()
task_ids = sorted(problems.keys())[args.start_index: args.end_index]
prompts = [problems[task_id]['prompt'] for task_id in task_ids]
num_samples = len(prompts)
print("Number of samples: {}".format(num_samples))
tokenizer, model = get_model(base_model=args.model)
generation_config = GenerationConfig(
pad_token_id=tokenizer.pad_token_id,
do_sample=False if args.greedy_decode else True,
temperature=args.temperature,
max_length=args.max_len,
num_return_sequences=args.num_seqs_per_iter,
eos_token_id=tokenizer.eos_token_id,
top_p=0.95
)
print(f"Loaded {args.model}.")
for i in tqdm(range(num_samples), ncols=0, total=num_samples):
output_file = args.output_path + '/{}.jsonl'.format(args.start_index + i)
if os.path.exists(output_file) and not args.overwrite:
print(f'Skip {output_file} as it already exists')
continue
prompt = prompts[i].replace(' ', '\t')
prompt_batch = [generate_prompt(prompt)]
ids_batch = [task_ids[i]]
completion_seqs = []
encoding = tokenizer(prompt_batch, return_tensors="pt", truncation=True, max_length=args.max_len).to(device)
if args.decoding_style == 'sampling':
loops = int(args.N / args.num_seqs_per_iter)
else:
loops = 1
for _ in tqdm(range(loops), total=loops, leave=False, ncols=0):
with torch.no_grad():
gen_tokens = model.generate(
**encoding,
generation_config=generation_config
)
if gen_tokens is not None:
gen_seqs = tokenizer.batch_decode(gen_tokens, skip_special_tokens=True)
else:
gen_seqs = None
if gen_seqs is not None:
assert len(ids_batch) == 1
task_id = ids_batch[0]
for seq_idx, gen_seq in enumerate(gen_seqs):
completion_seq = gen_seq.split("### Response:")[1]
completion_seq = completion_seq.replace('\t', ' ')
all_code = gen_seq.replace('\t', ' ')
completion_seqs.append(
{'task_id': task_id,
'completion': completion_seq,
'all_code': all_code,
}
)
print("Saving results to {}".format(output_file))
write_jsonl(output_file, completion_seqs)
if __name__ == '__main__':
main()
================================================
FILE: WizardCoder/src/humaneval_gen_vllm.py
================================================
import argparse
import pprint
import sys
import os
import re
from tqdm import tqdm
import torch
from transformers import LlamaTokenizer, AutoModelForCausalLM, GenerationConfig, BitsAndBytesConfig
from human_eval.data import write_jsonl, read_problems, stream_jsonl
from vllm import LLM
from vllm import SamplingParams
if torch.cuda.is_available():
device = "cuda"
else:
device = "cpu"
try:
if torch.backends.mps.is_available():
device = "mps"
except:
pass
def generate_prompt(input):
INSTRUCTION = f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
### Instruction:
Create a Python script for this problem:
{input}
### Response:"""
return INSTRUCTION
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model', type=str, default='bigcode/starcoder', help="")
parser.add_argument('--lora', type=str, default='bigcode/starcoder', help="")
parser.add_argument('--output_path', type=str, help="")
parser.add_argument('--start_index', type=int, default=0, help="")
parser.add_argument('--end_index', type=int, default=164, help="")
parser.add_argument('--temperature', type=float, default=0.8, help="")
parser.add_argument('--N', type=int, default=200, help="")
parser.add_argument('--max_len', type=int, default=512, help="")
parser.add_argument('--num_gpus', type=int, default=4, help="")
parser.add_argument('--decoding_style', type=str, default='sampling', help="")
parser.add_argument('--num_seqs_per_iter', type=int, default=50, help='')
parser.add_argument('--overwrite', action='store_true', help='')
args = parser.parse_args()
argsdict = vars(args)
print(pprint.pformat(argsdict))
problems = read_problems()
task_ids = sorted(problems.keys())[args.start_index: args.end_index]
prompts = [problems[task_id]['prompt'] for task_id in task_ids]
num_samples = len(prompts)
print("Number of samples: {}".format(num_samples))
llm = LLM(model=args.model, tensor_parallel_size=args.num_gpus)
sampling_params = SamplingParams(temperature=args.temperature, top_p=1, max_tokens=args.max_len)
print(f"Loaded {args.model}.")
for i in tqdm(range(num_samples), ncols=0, total=num_samples):
output_file = args.output_path + '/{}.jsonl'.format(args.start_index + i)
if os.path.exists(output_file) and not args.overwrite:
print(f'Skip {output_file} as it already exists')
continue
prompt = prompts[i].replace(' ', '\t')
prompt_batch = [generate_prompt(prompt)]
ids_batch = [task_ids[i]]
completion_seqs = []
if args.decoding_style == 'sampling':
loops = int(args.N / args.num_seqs_per_iter)
else:
loops = 1
for _ in tqdm(range(loops), total=loops, leave=False, ncols=0):
with torch.no_grad():
completions = llm.generate(prompt_batch, sampling_params)
gen_seqs = [completions[0].outputs[0].text]
if gen_seqs is not None:
assert len(ids_batch) == 1
task_id = ids_batch[0]
for seq_idx, gen_seq in enumerate(gen_seqs):
completion_seq = gen_seq.split("### Response:")[-1]
completion_seq = completion_seq.repla
gitextract_mddl1sk4/
├── Evol_Instruct/
│ ├── breadth.py
│ ├── depth.py
│ ├── main.py
│ └── openai_access.py
├── README.md
├── WizardCoder/
│ ├── CODE_LICENSE
│ ├── DATA_LICENSE
│ ├── MODEL_WEIGHTS_LICENSE
│ ├── README.md
│ ├── data/
│ │ └── mbppplus.json
│ └── src/
│ ├── humaneval_gen.py
│ ├── humaneval_gen_vllm.py
│ ├── inference_wizardcoder.py
│ ├── mbpp_gen.py
│ ├── mbppplus_gen.py
│ ├── mbppplus_gen_vllm.py
│ ├── mbppplus_process_preds.py
│ ├── process_humaneval.py
│ ├── process_mbpp.py
│ └── train_wizardcoder.py
├── WizardLM/
│ ├── CODE_LICENSE
│ ├── DATA_LICENSE
│ ├── MODEL_DIFF_LICENSE
│ ├── README.md
│ ├── data/
│ │ └── WizardLM_testset.jsonl
│ ├── doc/
│ │ └── distributed_finetune.md
│ └── src/
│ ├── case_show.md
│ ├── infer_wizardlm13b.py
│ ├── inference_wizardlm.py
│ ├── train_freeform.py
│ └── weight_diff_wizard.py
├── WizardMath/
│ ├── LICENSE
│ ├── README.md
│ ├── config/
│ │ └── deepspeed_config.json
│ ├── data/
│ │ ├── MATH_test.jsonl
│ │ └── gsm8k_test.jsonl
│ ├── inference/
│ │ ├── MATH_inference.py
│ │ ├── grader.py
│ │ ├── gsm8k_inference.py
│ │ └── util.py
│ └── train/
│ └── train_wizardmath.py
├── demo/
│ ├── README.md
│ ├── wizardLM_demo.py
│ ├── wizardcoder_demo.py
│ └── wizardmath_demo.py
└── training/
├── data/
│ └── alpaca_data.json
├── requirements.txt
└── src/
├── configs/
│ ├── deepspeed_config.json
│ └── hostfile
├── conversation.py
├── environment.yml
├── generate.py
├── train.py
├── train_freeform_multiturn.py
└── utils.py
SYMBOL INDEX (151 symbols across 28 files)
FILE: Evol_Instruct/breadth.py
function createBreadthPrompt (line 10) | def createBreadthPrompt(instruction):
FILE: Evol_Instruct/depth.py
function createConstraintsPrompt (line 11) | def createConstraintsPrompt(instruction):
function createDeepenPrompt (line 17) | def createDeepenPrompt(instruction):
function createConcretizingPrompt (line 23) | def createConcretizingPrompt(instruction):
function createReasoningPrompt (line 30) | def createReasoningPrompt(instruction):
FILE: Evol_Instruct/openai_access.py
function get_oai_completion (line 7) | def get_oai_completion(prompt):
function call_chatgpt (line 49) | def call_chatgpt(ins):
FILE: WizardCoder/src/humaneval_gen.py
function generate_prompt (line 22) | def generate_prompt(input):
function get_model (line 33) | def get_model(
function main (line 67) | def main():
FILE: WizardCoder/src/humaneval_gen_vllm.py
function generate_prompt (line 25) | def generate_prompt(input):
function main (line 37) | def main():
FILE: WizardCoder/src/inference_wizardcoder.py
function evaluate (line 22) | def evaluate(
function generate_prompt (line 59) | def generate_prompt(instruction, input=None):
function main (line 68) | def main(
FILE: WizardCoder/src/mbpp_gen.py
function read_mbpp (line 23) | def read_mbpp(path):
function extract_text (line 30) | def extract_text(prompt, remove_lines=True):
function generate_prompt (line 45) | def generate_prompt(input):
function get_model (line 55) | def get_model(
function main (line 89) | def main():
FILE: WizardCoder/src/mbppplus_gen.py
function generate_prompt (line 24) | def generate_prompt(input):
function get_model (line 34) | def get_model(
function main (line 68) | def main():
FILE: WizardCoder/src/mbppplus_gen_vllm.py
function generate_prompt (line 27) | def generate_prompt(input):
function get_model (line 37) | def get_model(
function main (line 71) | def main():
FILE: WizardCoder/src/process_mbpp.py
function read_mbpp (line 8) | def read_mbpp(path):
FILE: WizardCoder/src/train_wizardcoder.py
class ModelArguments (line 49) | class ModelArguments:
class DataArguments (line 54) | class DataArguments:
class TrainingArguments (line 59) | class TrainingArguments(transformers.TrainingArguments):
function safe_save_model_for_hf_trainer (line 68) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, output...
function smart_tokenizer_and_embedding_resize (line 77) | def smart_tokenizer_and_embedding_resize(
function _tokenize_fn (line 100) | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrai...
function preprocess (line 124) | def preprocess(
class DataCollatorForSupervisedDataset (line 140) | class DataCollatorForSupervisedDataset(object):
method __call__ (line 145) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
function train_tokenize_function (line 159) | def train_tokenize_function(examples, tokenizer):
function train (line 177) | def train():
FILE: WizardLM/src/infer_wizardlm13b.py
function main (line 27) | def main(
FILE: WizardLM/src/inference_wizardlm.py
function evaluate (line 24) | def evaluate(
function generate_prompt (line 57) | def generate_prompt(instruction, input=None):
function main (line 64) | def main(
FILE: WizardLM/src/train_freeform.py
class ModelArguments (line 43) | class ModelArguments:
class DataArguments (line 48) | class DataArguments:
class TrainingArguments (line 54) | class TrainingArguments(transformers.TrainingArguments):
function safe_save_model_for_hf_trainer (line 63) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, output...
function smart_tokenizer_and_embedding_resize (line 72) | def smart_tokenizer_and_embedding_resize(
function _tokenize_fn (line 95) | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrai...
function preprocess (line 119) | def preprocess(
class SupervisedDataset (line 134) | class SupervisedDataset(Dataset):
method __init__ (line 137) | def __init__(self, data_path: str, tokenizer: transformers.PreTrainedT...
method __len__ (line 157) | def __len__(self):
method __getitem__ (line 160) | def __getitem__(self, i) -> Dict[str, torch.Tensor]:
class SupervisedComplexDataset (line 163) | class SupervisedComplexDataset(Dataset):
method __init__ (line 166) | def __init__(self, data_path: str, tokenizer: transformers.PreTrainedT...
method __len__ (line 193) | def __len__(self):
method __getitem__ (line 196) | def __getitem__(self, i) -> Dict[str, torch.Tensor]:
class DataCollatorForSupervisedDataset (line 200) | class DataCollatorForSupervisedDataset(object):
method __call__ (line 205) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
function make_supervised_data_module (line 218) | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokeni...
function train (line 228) | def train():
FILE: WizardLM/src/weight_diff_wizard.py
function make_diff (line 25) | def make_diff(
function recover (line 73) | def recover(
function main (line 158) | def main(task, **kwargs):
FILE: WizardMath/inference/MATH_inference.py
function remove_boxed (line 13) | def remove_boxed(s):
function process_results (line 22) | def process_results(doc, completion, answer):
function batch_data (line 41) | def batch_data(data_list, batch_size=1):
function test_hendrycks_math (line 54) | def test_hendrycks_math(model, data_path, start=0, end=MAX_INT, batch_si...
function parse_args (line 103) | def parse_args():
FILE: WizardMath/inference/grader.py
function is_digit (line 14) | def is_digit(s):
function math_equal (line 21) | def math_equal(prediction: Union[bool, float, str],
function math_equal_process (line 94) | def math_equal_process(param):
function symbolic_equal (line 98) | def symbolic_equal(a, b):
function symbolic_equal_process (line 123) | def symbolic_equal_process(a, b, output_queue):
function call_with_timeout (line 128) | def call_with_timeout(func, *args, timeout=1, **kwargs):
FILE: WizardMath/inference/gsm8k_inference.py
function is_number (line 11) | def is_number(s):
function extract_answer_number (line 25) | def extract_answer_number(completion):
function batch_data (line 53) | def batch_data(data_list, batch_size=1):
function gsm8k_test (line 67) | def gsm8k_test(model, data_path, start=0, end=MAX_INT, batch_size=1, ten...
function parse_args (line 124) | def parse_args():
FILE: WizardMath/inference/util.py
function last_boxed_only (line 4) | def last_boxed_only(sample):
function last_boxed_only_string (line 11) | def last_boxed_only_string(string):
function only_until_first_boxed_from_tokens (line 38) | def only_until_first_boxed_from_tokens(string, tokens):
function clean_numbers (line 55) | def clean_numbers(sample):
function _clean_numbers (line 64) | def _clean_numbers(string):
function fix_fracs (line 96) | def fix_fracs(string):
function fix_a_slash_b (line 127) | def fix_a_slash_b(string):
function remove_right_units (line 141) | def remove_right_units(string):
function fix_sqrt (line 150) | def fix_sqrt(string):
function strip_string (line 165) | def strip_string(string):
function is_equiv (line 230) | def is_equiv(str1, str2, verbose=False):
class NotEqual (line 251) | class NotEqual:
method __eq__ (line 252) | def __eq__(self, other):
FILE: WizardMath/train/train_wizardmath.py
class ModelArguments (line 49) | class ModelArguments:
class DataArguments (line 54) | class DataArguments:
class TrainingArguments (line 59) | class TrainingArguments(transformers.TrainingArguments):
function safe_save_model_for_hf_trainer (line 68) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, output...
function smart_tokenizer_and_embedding_resize (line 77) | def smart_tokenizer_and_embedding_resize(
function _tokenize_fn (line 100) | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrai...
function preprocess (line 124) | def preprocess(
class DataCollatorForSupervisedDataset (line 140) | class DataCollatorForSupervisedDataset(object):
method __call__ (line 145) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
function train_tokenize_function (line 159) | def train_tokenize_function(examples, tokenizer):
function train (line 176) | def train():
FILE: demo/wizardLM_demo.py
function parse_args (line 8) | def parse_args():
function predict (line 14) | def predict(message, history, system_prompt, temperature, max_tokens):
FILE: demo/wizardcoder_demo.py
function main (line 9) | def main(
FILE: demo/wizardmath_demo.py
function main (line 7) | def main(
FILE: training/src/conversation.py
class SeparatorStyle (line 10) | class SeparatorStyle(Enum):
class Conversation (line 25) | class Conversation:
method get_prompt (line 53) | def get_prompt(self) -> str:
method append_message (line 139) | def append_message(self, role: str, message: str):
method to_gradio_chatbot (line 143) | def to_gradio_chatbot(self):
method to_openai_api_messages (line 153) | def to_openai_api_messages(self):
method copy (line 165) | def copy(self):
method dict (line 181) | def dict(self):
function register_conv_template (line 197) | def register_conv_template(template: Conversation, override: bool = False):
function get_conv_template (line 204) | def get_conv_template(name: str) -> Conversation:
FILE: training/src/generate.py
function main (line 26) | def main(
function generate_prompt (line 122) | def generate_prompt(instruction, input=None):
FILE: training/src/train.py
class ModelArguments (line 49) | class ModelArguments:
class DataArguments (line 54) | class DataArguments:
class TrainingArguments (line 59) | class TrainingArguments(transformers.TrainingArguments):
function safe_save_model_for_hf_trainer (line 68) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, output...
function smart_tokenizer_and_embedding_resize (line 77) | def smart_tokenizer_and_embedding_resize(
function _tokenize_fn (line 100) | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrai...
function preprocess (line 124) | def preprocess(
class DataCollatorForSupervisedDataset (line 140) | class DataCollatorForSupervisedDataset(object):
method __call__ (line 145) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
function train_tokenize_function (line 159) | def train_tokenize_function(examples, tokenizer):
function train (line 176) | def train():
FILE: training/src/train_freeform_multiturn.py
class ModelArguments (line 50) | class ModelArguments:
class DataArguments (line 55) | class DataArguments:
class TrainingArguments (line 61) | class TrainingArguments(transformers.TrainingArguments):
function safe_save_model_for_hf_trainer (line 70) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, output...
function smart_tokenizer_and_embedding_resize (line 79) | def smart_tokenizer_and_embedding_resize(
function _tokenize_fn (line 102) | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrai...
function rank0_print (line 129) | def rank0_print(*args):
function preprocess (line 134) | def preprocess(
class SupervisedDataset (line 220) | class SupervisedDataset(Dataset):
method __init__ (line 223) | def __init__(self, data_path: str, tokenizer: transformers.PreTrainedT...
method __len__ (line 235) | def __len__(self):
method __getitem__ (line 238) | def __getitem__(self, i) -> Dict[str, torch.Tensor]:
function make_supervised_data_module (line 246) | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokeni...
function train (line 254) | def train():
FILE: training/src/utils.py
class OpenAIDecodingArguments (line 25) | class OpenAIDecodingArguments(object):
function openai_completion (line 39) | def openai_completion(
function _make_w_io_base (line 133) | def _make_w_io_base(f, mode: str):
function _make_r_io_base (line 142) | def _make_r_io_base(f, mode: str):
function jdump (line 148) | def jdump(obj, f, mode="w", indent=4, default=str):
function jload (line 168) | def jload(f, mode="r"):
Condensed preview — 55 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (5,880K chars).
[
{
"path": "Evol_Instruct/breadth.py",
"chars": 766,
"preview": "base_instruction = \"I want you act as a Prompt Creator.\\r\\n\\\r\nYour goal is to draw inspiration from the #Given Prompt# t"
},
{
"path": "Evol_Instruct/depth.py",
"chars": 2102,
"preview": "base_instruction = \"I want you act as a Prompt Rewriter.\\r\\n \\\r\n\t\t\t\t\tYour objective is to rewrite a given prompt into a "
},
{
"path": "Evol_Instruct/main.py",
"chars": 1069,
"preview": "import json\r\nimport random\r\n\r\nfrom openai_access import call_chatgpt\r\nfrom depth import createConstraintsPrompt, createD"
},
{
"path": "Evol_Instruct/openai_access.py",
"chars": 1824,
"preview": "import openai\r\nimport time\r\n\r\nopenai.api_key = 'your api key'\r\n\r\n\r\ndef get_oai_completion(prompt):\r\n\r\n try: \r\n "
},
{
"path": "README.md",
"chars": 18958,
"preview": "## WizardLM: Empowering Large Pre-Trained Language Models to Follow Complex Instructions\n\n<p style=\"font-size:50px;\" ali"
},
{
"path": "WizardCoder/CODE_LICENSE",
"chars": 11596,
"preview": " Apache License\r\n Version 2.0, January 2004\r\n "
},
{
"path": "WizardCoder/DATA_LICENSE",
"chars": 19747,
"preview": "Attribution-NonCommercial 4.0 International\r\n\r\n=======================================================================\r\n"
},
{
"path": "WizardCoder/MODEL_WEIGHTS_LICENSE",
"chars": 12647,
"preview": "BigCode Open RAIL-M v1 License Agreement\r\nSection I: Preamble\r\nThis OpenRAIL-M License Agreement was created under BigCo"
},
{
"path": "WizardCoder/README.md",
"chars": 25250,
"preview": "# WizardCoder: Empowering Code Large Language Models with Evol-Instruct\r\n\r\n[![Code License](https://img.shields.io/badge"
},
{
"path": "WizardCoder/data/mbppplus.json",
"chars": 95710,
"preview": "{\n \"Mbpp/2\": \"Write a function to find the shared elements from the given two lists. Your code should satisfy the fol"
},
{
"path": "WizardCoder/src/humaneval_gen.py",
"chars": 5357,
"preview": "import argparse\r\nimport pprint\r\nimport sys\r\nimport os\r\nimport re\r\nfrom tqdm import tqdm\r\nimport torch\r\nfrom transformers"
},
{
"path": "WizardCoder/src/humaneval_gen_vllm.py",
"chars": 3852,
"preview": "import argparse\nimport pprint\nimport sys\nimport os\nimport re\nfrom tqdm import tqdm\nimport torch\nfrom transformers import"
},
{
"path": "WizardCoder/src/inference_wizardcoder.py",
"chars": 3337,
"preview": "import sys\r\nimport os\r\nimport fire\r\nimport torch\r\nimport transformers\r\nimport json\r\nimport jsonlines\r\n\r\nfrom transformer"
},
{
"path": "WizardCoder/src/mbpp_gen.py",
"chars": 6509,
"preview": "import jsonlines\r\nimport argparse\r\nimport pprint\r\nimport sys\r\nimport os\r\nimport re\r\nfrom tqdm import tqdm\r\nimport torch\r"
},
{
"path": "WizardCoder/src/mbppplus_gen.py",
"chars": 5400,
"preview": "import argparse\r\nimport pprint\r\nimport sys\r\nimport os\r\nimport re\r\nimport json\r\nfrom tqdm import tqdm\r\nimport torch\r\nfrom"
},
{
"path": "WizardCoder/src/mbppplus_gen_vllm.py",
"chars": 4887,
"preview": "import argparse\r\nimport pprint\r\nimport sys\r\nimport os\r\nimport re\r\nimport json\r\nfrom tqdm import tqdm\r\nimport torch\r\nfrom"
},
{
"path": "WizardCoder/src/mbppplus_process_preds.py",
"chars": 1980,
"preview": "from human_eval.data import read_problems, write_jsonl, stream_jsonl\r\nimport glob \r\nfrom tqdm import tqdm\r\nimport argpar"
},
{
"path": "WizardCoder/src/process_humaneval.py",
"chars": 2165,
"preview": "from human_eval.data import read_problems, write_jsonl, stream_jsonl\r\nimport glob \r\nfrom tqdm import tqdm\r\nimport argpar"
},
{
"path": "WizardCoder/src/process_mbpp.py",
"chars": 2321,
"preview": "from human_eval.data import stream_jsonl\r\nimport glob \r\nfrom tqdm import tqdm\r\nimport argparse\r\nimport jsonlines\r\nimport"
},
{
"path": "WizardCoder/src/train_wizardcoder.py",
"chars": 9430,
"preview": "# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\r\n#\r\n# Licensed under the Apa"
},
{
"path": "WizardLM/CODE_LICENSE",
"chars": 11596,
"preview": " Apache License\r\n Version 2.0, January 2004\r\n "
},
{
"path": "WizardLM/DATA_LICENSE",
"chars": 19747,
"preview": "Attribution-NonCommercial 4.0 International\r\n\r\n=======================================================================\r\n"
},
{
"path": "WizardLM/MODEL_DIFF_LICENSE",
"chars": 19747,
"preview": "Attribution-NonCommercial 4.0 International\r\n\r\n=======================================================================\r\n"
},
{
"path": "WizardLM/README.md",
"chars": 14916,
"preview": "## WizardLM: An Instruction-following LLM Using Evol-Instruct \nEmpowering Large Pre-Trained Language Models to Follow Co"
},
{
"path": "WizardLM/data/WizardLM_testset.jsonl",
"chars": 80906,
"preview": "{\"idx\": 1, \"Skill\": \"Math\", \"Difficulty\": 1, \"Instruction\": \"If a car travels 120 miles in 2 hours, what is its average "
},
{
"path": "WizardLM/doc/distributed_finetune.md",
"chars": 3690,
"preview": "# 📢 Distributed Fine-tuning \nWe've conducted distributed fine tune experiment on our WizardLM utilizing original Llama"
},
{
"path": "WizardLM/src/case_show.md",
"chars": 2156,
"preview": "### Case Show with WizardLM\n\n\nWe just sample some cases to demonstrate the performance of WizardLM and ChatGPT on data o"
},
{
"path": "WizardLM/src/infer_wizardlm13b.py",
"chars": 3905,
"preview": "import sys\nimport os\n# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"7\"\nimport fire\nimport torch\n# from peft import PeftModel\nim"
},
{
"path": "WizardLM/src/inference_wizardlm.py",
"chars": 3409,
"preview": "import sys\r\nimport os\r\nimport fire\r\nimport torch\r\nimport transformers\r\nimport json\r\n\r\nassert (\r\n \"LlamaTokenizer\" in "
},
{
"path": "WizardLM/src/train_freeform.py",
"chars": 10215,
"preview": "# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\r\n#\r\n# Licensed under the Apa"
},
{
"path": "WizardLM/src/weight_diff_wizard.py",
"chars": 6551,
"preview": "# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\r\n#\r\n# Licensed under the Apa"
},
{
"path": "WizardMath/LICENSE",
"chars": 7037,
"preview": "LLAMA 2 COMMUNITY LICENSE AGREEMENT\r\nLlama 2 Version Release Date: July 18, 2023\r\n\r\n\"Agreement\" means the terms and cond"
},
{
"path": "WizardMath/README.md",
"chars": 16890,
"preview": "# WizardMath : Empowering Mathematical Reasoning for Large Language Models via Reinforced Evol-Instruct (RLEIF)\r\n\r\n[![Co"
},
{
"path": "WizardMath/config/deepspeed_config.json",
"chars": 1003,
"preview": "{\r\n \"zero_optimization\": {\r\n \"stage\": 3,\r\n \"offload_optimizer\": {\r\n \"device\": \"cpu\",\r\n "
},
{
"path": "WizardMath/data/MATH_test.jsonl",
"chars": 4202222,
"preview": "{\"idx\": \"hendrycks_math_1\", \"instruction\": \"Find the matrix $\\\\mathbf{M}$ such that\\n\\\\[\\\\mathbf{M} \\\\begin{pmatrix} 1 &"
},
{
"path": "WizardMath/data/gsm8k_test.jsonl",
"chars": 749738,
"preview": "{\"question\": \"Janet\\u2019s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for h"
},
{
"path": "WizardMath/inference/MATH_inference.py",
"chars": 4768,
"preview": "import argparse\r\nimport json\r\nimport pdb\r\nimport jsonlines\r\n\r\nimport util\r\nfrom vllm import LLM, SamplingParams\r\nimport "
},
{
"path": "WizardMath/inference/grader.py",
"chars": 4554,
"preview": "\"\"\"\r\nThis logic is largely copied from the Hendrycks' MATH release (math_equivalence), and borrowed from:\r\n- https://git"
},
{
"path": "WizardMath/inference/gsm8k_inference.py",
"chars": 5453,
"preview": "import argparse\r\nimport json\r\nimport re\r\nimport jsonlines\r\nfrom fraction import Fraction\r\nfrom vllm import LLM, Sampling"
},
{
"path": "WizardMath/inference/util.py",
"chars": 7310,
"preview": "import pprint\r\nfrom grader import math_equal\r\n\r\ndef last_boxed_only(sample):\r\n q, a = sample\r\n a = last_boxed_only"
},
{
"path": "WizardMath/train/train_wizardmath.py",
"chars": 9362,
"preview": "# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\r\n#\r\n# Licensed under the Apa"
},
{
"path": "demo/README.md",
"chars": 3779,
"preview": "\r\n## Choice-1: API Server\r\nThe OpenAI-compatible APIs are provided by vLLM. We advise you to use vLLM=0.2.1.post1 to bui"
},
{
"path": "demo/wizardLM_demo.py",
"chars": 2495,
"preview": "import gradio as gr\nimport argparse\nimport os\nimport json\nfrom vllm import LLM, SamplingParams\n\n\ndef parse_args():\n p"
},
{
"path": "demo/wizardcoder_demo.py",
"chars": 2267,
"preview": "import os\r\nimport sys\r\nimport fire\r\nimport torch\r\nimport transformers\r\nimport gradio as gr\r\nfrom vllm import LLM, Sampli"
},
{
"path": "demo/wizardmath_demo.py",
"chars": 2631,
"preview": "import json\r\nimport fire\r\nimport gradio as gr\r\nfrom vllm import LLM, SamplingParams\r\nimport os\r\n\r\ndef main(\r\n bas"
},
{
"path": "training/requirements.txt",
"chars": 104,
"preview": "numpy\nrouge_score\nfire\nopenai\nsentencepiece\nwandb\ngradio==3.9\ndeepspeed==0.9.2 \naccelerate\ntensorboardX\n"
},
{
"path": "training/src/configs/deepspeed_config.json",
"chars": 1190,
"preview": "{\n \"zero_optimization\": {\n \"stage\": 3,\n \"offload_optimizer\": {\n \"device\": \"cpu\",\n "
},
{
"path": "training/src/configs/hostfile",
"chars": 109,
"preview": "ip_address_of_main_node slots=num_of_gpus_in_each_node\nip_address_of_sub_node1 slots=num_of_gpus_in_each_node"
},
{
"path": "training/src/conversation.py",
"chars": 16671,
"preview": "\"\"\"\nConversation prompt templates.\n\"\"\"\n\nimport dataclasses\nfrom enum import auto, Enum\nfrom typing import List, Tuple, A"
},
{
"path": "training/src/environment.yml",
"chars": 5040,
"preview": "name: llamax\nchannels:\n - pytorch\n - defaults\ndependencies:\n - _libgcc_mutex=0.1=main\n - _openmp_mutex=5.1=1_gnu\n -"
},
{
"path": "training/src/generate.py",
"chars": 4290,
"preview": "import sys\n\nimport fire\nimport torch\n# from peft import PeftModel\nimport transformers\nimport gradio as gr\n\nassert (\n "
},
{
"path": "training/src/train.py",
"chars": 9118,
"preview": "# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\n#\n# Licensed under the Apach"
},
{
"path": "training/src/train_freeform_multiturn.py",
"chars": 9974,
"preview": "# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li\n#\n# Licensed under the Apach"
},
{
"path": "training/src/utils.py",
"chars": 6481,
"preview": "import dataclasses\nimport logging\nimport math\nimport os\nimport io\nimport sys\nimport time\nimport json\nfrom typing import "
}
]
// ... and 1 more files (download for full content)
About this extraction
This page contains the full source code of the nlpxucan/WizardLM GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 55 files (26.9 MB), approximately 1.4M tokens, and a symbol index with 151 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.