Copy disabled (too large)
Download .txt
Showing preview only (18,974K chars total). Download the full file to get everything.
Repository: Lichang-Chen/InstructZero
Branch: main
Commit: 911f01912bb5
Files: 110
Total size: 18.1 MB
Directory structure:
gitextract_xr63eqto/
├── .gitignore
├── InstructZero/
│ └── experiments/
│ ├── args.py
│ ├── automatic_prompt_engineer/
│ │ ├── __init__.py
│ │ ├── ape.py
│ │ ├── config.py
│ │ ├── data.py
│ │ ├── evaluate.py
│ │ ├── generate.py
│ │ ├── llm.py
│ │ └── template.py
│ ├── configs/
│ │ └── instruction_induction.yaml
│ ├── data/
│ │ └── instruction_induction/
│ │ ├── README.md
│ │ ├── induction_input/
│ │ │ ├── active_to_passive.json
│ │ │ ├── antonyms.json
│ │ │ ├── cause_and_effect.json
│ │ │ ├── common_concept.json
│ │ │ ├── diff.json
│ │ │ ├── first_word_letter.json
│ │ │ ├── informal_to_formal.json
│ │ │ ├── larger_animal.json
│ │ │ ├── letters_list.json
│ │ │ ├── negation.json
│ │ │ ├── num_to_verbal.json
│ │ │ ├── orthography_starts_with.json
│ │ │ ├── rhymes.json
│ │ │ ├── second_word_letter.json
│ │ │ ├── sentence_similarity.json
│ │ │ ├── sentiment.json
│ │ │ ├── singular_to_plural.json
│ │ │ ├── sum.json
│ │ │ ├── synonyms.json
│ │ │ ├── taxonomy_animal.json
│ │ │ ├── translation_en-de.json
│ │ │ ├── translation_en-es.json
│ │ │ ├── translation_en-fr.json
│ │ │ └── word_in_context.json
│ │ ├── load_data.py
│ │ └── raw/
│ │ ├── execute/
│ │ │ ├── active_to_passive.json
│ │ │ ├── antonyms.json
│ │ │ ├── auto_categorization.json
│ │ │ ├── auto_debugging.json
│ │ │ ├── cause_and_effect.json
│ │ │ ├── common_concept.json
│ │ │ ├── diff.json
│ │ │ ├── first_word_letter.json
│ │ │ ├── informal_to_formal.json
│ │ │ ├── larger_animal.json
│ │ │ ├── letters_list.json
│ │ │ ├── negation.json
│ │ │ ├── num_to_verbal.json
│ │ │ ├── object_counting.json
│ │ │ ├── odd_one_out.json
│ │ │ ├── orthography_starts_with.json
│ │ │ ├── periodic_elements.json
│ │ │ ├── rhymes.json
│ │ │ ├── second_word_letter.json
│ │ │ ├── sentence_similarity.json
│ │ │ ├── sentiment.json
│ │ │ ├── singular_to_plural.json
│ │ │ ├── sum.json
│ │ │ ├── synonyms.json
│ │ │ ├── taxonomy_animal.json
│ │ │ ├── translation_en-de.json
│ │ │ ├── translation_en-es.json
│ │ │ ├── translation_en-fr.json
│ │ │ ├── word_in_context.json
│ │ │ ├── word_sorting.json
│ │ │ └── word_unscrambling.json
│ │ └── induce/
│ │ ├── active_to_passive.json
│ │ ├── antonyms.json
│ │ ├── ascii_word_recognition.json
│ │ ├── auto_categorization.json
│ │ ├── auto_debugging.json
│ │ ├── cause_and_effect.json
│ │ ├── common_concept.json
│ │ ├── cs_algorithms.json
│ │ ├── diff.json
│ │ ├── first_word_letter.json
│ │ ├── informal_to_formal.json
│ │ ├── larger_animal.json
│ │ ├── letters_list.json
│ │ ├── negation.json
│ │ ├── num_to_verbal.json
│ │ ├── object_counting.json
│ │ ├── odd_one_out.json
│ │ ├── orthography_starts_with.json
│ │ ├── periodic_elements.json
│ │ ├── rhymes.json
│ │ ├── second_word_letter.json
│ │ ├── sentence_similarity.json
│ │ ├── sentiment.json
│ │ ├── singular_to_plural.json
│ │ ├── sum.json
│ │ ├── synonyms.json
│ │ ├── taxonomy_animal.json
│ │ ├── translation_en-de.json
│ │ ├── translation_en-es.json
│ │ ├── translation_en-fr.json
│ │ ├── word_in_context.json
│ │ ├── word_sorting.json
│ │ └── word_unscrambling.json
│ ├── evaluation/
│ │ └── instruction_induction/
│ │ ├── __init__.py
│ │ ├── exec_accuracy.py
│ │ └── utility.py
│ ├── instruction_coupled_kernel.py
│ ├── misc.py
│ ├── run_instructzero.py
│ └── run_instructzero.sh
├── README.md
└── requirements.txt
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
InstructZero/automatic_prompt_engineer.egg-info
InstructZero/build
InstructZero/dist
.vscode/
InstructZero/experiments/__pycache__
================================================
FILE: InstructZero/experiments/args.py
================================================
import argparse
def parse_args():
parser = argparse.ArgumentParser(description="InstructZero pipeline")
parser.add_argument(
"--task",
type=str,
default=None,
help="The name of the dataset to use (via the datasets library).",
)
parser.add_argument(
"--random_proj",
type=str,
default="uniform",
help="The initialization of the projection matrix A."
)
parser.add_argument(
"--intrinsic_dim",
type=int,
default=10,
help="The instrinsic dimension of the projection matrix"
)
parser.add_argument(
"--n_prompt_tokens",
type=int,
default=5,
help="The number of prompt tokens."
)
parser.add_argument(
"--HF_cache_dir",
type=str,
default="/data/bobchen/vicuna-13b",
help="Your vicuna directory"
)
parser.add_argument(
"--seed",
type=int,
default=0,
help="Set the seed."
)
parser.add_argument(
"--alpha",
type=float,
default=1.0,
help="Set the alpha if the initialization of the projection matrix A is std."
)
parser.add_argument(
"--beta",
type=float,
default=3.0,
help="Set the beta if the initialization of the projection matrix A is std."
)
parser.add_argument(
"--api_model",
type=str,
default='chatgpt',
help="The black-box api model."
)
parser.add_argument(
"--model_name",
type=str,
default='vicuna',
help="The model name of the open-source LLM."
)
args = parser.parse_args()
return args
================================================
FILE: InstructZero/experiments/automatic_prompt_engineer/__init__.py
================================================
================================================
FILE: InstructZero/experiments/automatic_prompt_engineer/ape.py
================================================
import random
from . import generate, evaluate, config, template, data, llm
def get_simple_prompt_gen_template(prompt_gen_template, prompt_gen_mode):
if prompt_gen_template is None:
if prompt_gen_mode == 'forward':
prompt_gen_template = "I gave a friend an instruction. Based on the instruction they produced the following input-output pairs:\n\n[full_DEMO]\n\nThe instruction was to [APE]"
elif prompt_gen_mode == 'insert':
prompt_gen_template = None
else:
raise ValueError(
'Invalid prompt_gen_mode: {}'.format(prompt_gen_mode))
return prompt_gen_template
def simple_ape(dataset,
eval_template='Instruction: [PROMPT]\nInput: [INPUT]\nOutput: [OUTPUT]',
prompt_gen_template=None,
demos_template='Input: [INPUT]\nOutput: [OUTPUT]',
eval_model='text-davinci-002',
prompt_gen_model='text-davinci-002',
prompt_gen_mode='forward',
num_prompts=50,
eval_rounds=20,
prompt_gen_batch_size=200,
eval_batch_size=500):
"""
Function that wraps the find_prompts function to make it easier to use.
Design goals: include default values for most parameters, and automatically
fill out the config dict for the user in a way that fits almost all use cases.
The main shortcuts this function takes are:
- Uses the same dataset for prompt generation, evaluation, and few shot demos
- Uses UCB algorithm for evaluation
- Fixes the number of prompts per round to num_prompts // 3 (so the first three rounds will
sample every prompt once)
- Fixes the number of samples per prompt per round to 5
Parameters:
dataset: The dataset to use for evaluation.
eval_template: The template for the evaluation queries.
prompt_gen_template: The template to use for prompt generation.
demos_template: The template for the demos.
eval_model: The model to use for evaluation.
prompt_gen_model: The model to use for prompt generation.
prompt_gen_mode: The mode to use for prompt generation.
num_prompts: The number of prompts to generate during the search.
eval_rounds: The number of evaluation rounds to run.
Returns:
An evaluation result and a function to evaluate the prompts with new inputs.
"""
prompt_gen_template = get_simple_prompt_gen_template(
prompt_gen_template, prompt_gen_mode)
conf = config.simple_config(
eval_model, prompt_gen_model, prompt_gen_mode, num_prompts, eval_rounds, prompt_gen_batch_size, eval_batch_size)
return find_prompts(eval_template, demos_template, dataset, dataset, conf, prompt_gen_template=prompt_gen_template)
def simple_eval(dataset,
prompts,
eval_template='Instruction: [PROMPT]\nInput: [INPUT]\nOutput: [OUTPUT]',
demos_template='Input: [INPUT]\nOutput: [OUTPUT]',
eval_model='text-davinci-002',
num_samples=50):
"""
Function that wraps the evaluate_prompts function to make it easier to use.
Parameters:
dataset: The dataset to use for evaluation.
prompts: The list of prompts to evaluate.
eval_template: The template for the evaluation queries.
demos_template: The template for the demos.
eval_model: The model to use for evaluation.
Returns:
An evaluation result.
"""
eval_template = template.EvalTemplate(eval_template)
demos_template = template.DemosTemplate(demos_template)
conf = config.update_config({}, 'configs/default.yaml')
conf['evaluation']['model']['gpt_config']['model'] = eval_model
conf['evaluation']['num_samples'] = min(len(dataset[0]), num_samples)
res = evaluate.evalute_prompts(
prompts, eval_template, dataset, demos_template, dataset, conf['evaluation']['method'], conf['evaluation'])
return res
def simple_estimate_cost(dataset,
eval_template='Instruction: [PROMPT]\nInput: [INPUT]\nOutput: [OUTPUT]',
prompt_gen_template=None,
demos_template='Input: [INPUT]\nOutput: [OUTPUT]',
eval_model='text-davinci-002',
prompt_gen_model='text-davinci-002',
prompt_gen_mode='forward',
num_prompts=50,
eval_rounds=20,
prompt_gen_batch_size=200,
eval_batch_size=500):
prompt_gen_template = get_simple_prompt_gen_template(
prompt_gen_template, prompt_gen_mode)
conf = config.simple_config(
eval_model, prompt_gen_model, prompt_gen_mode, num_prompts, eval_rounds, prompt_gen_batch_size, eval_batch_size)
return estimate_cost(eval_template, demos_template, dataset, dataset, conf, prompt_gen_template=prompt_gen_template)
def find_prompts(eval_template,
demos_template,
prompt_gen_data,
eval_data,
conf,
base_conf='configs/default.yaml',
few_shot_data=None,
prompt_gen_template=None):
"""
Function to generate prompts using APE.
Parameters:
eval_template: The template for the evaluation queries.
demos_template: The template for the demos.
prompt_gen_data: The data to use for prompt generation.
eval_data: The data to use for evaluation.
conf: The configuration dictionary.
few_shot_data: The data to use for demonstrations during eval (not implemented yet).
eval_method: The evaluation method to use. ('likelihood')
prompt_gen_template: The template to use for prompt generation.
verbosity: The verbosity level.
Returns:
An evaluation result. Also returns a function to evaluate the prompts with new inputs.
"""
conf = config.update_config(conf, base_conf)
# Generate prompts
eval_template = template.EvalTemplate(eval_template)
demos_template = template.DemosTemplate(demos_template)
if prompt_gen_template is None:
prompt_gen_template = eval_template.convert_to_generation_template()
else:
prompt_gen_template = template.GenerationTemplate(prompt_gen_template)
if few_shot_data is None:
few_shot_data = prompt_gen_data
import pdb; pdb.set_trace()
print('Generating prompts...')
prompts = generate.generate_prompts(
prompt_gen_template, demos_template, prompt_gen_data, conf['generation'])
print('Model returned {} prompts. Deduplicating...'.format(len(prompts)))
prompts = list(set(prompts))
print('Deduplicated to {} prompts.'.format(len(prompts)))
print('Evaluating prompts...')
res = evaluate.evalute_prompts(prompts, eval_template, eval_data, demos_template, few_shot_data,
conf['evaluation']['method'], conf['evaluation'])
print('Finished evaluating.')
demo_fn = evaluate.demo_function(eval_template, conf['demo'])
return res, demo_fn
def evaluate_prompts(prompts, eval_template, eval_data, demos_template, few_shot_data, conf,
base_conf='configs/default.yaml'):
"""
Function to evaluate a list of prompts.
Parameters:
prompts: The list of prompts to evaluate.
eval_template: The template for the evaluation queries.
eval_data: The data to use for evaluation.
eval_method: The evaluation method to use. ('likelihood')
conf: The configuration dictionary.
base_conf: The base configuration file.
Returns:
A list of prompts and their scores, sorted by score.
"""
conf = config.update_config(conf, base_conf)
# Generate prompts
eval_template = template.EvalTemplate(eval_template)
demos_template = template.DemosTemplate(demos_template)
print('Evaluating prompts...')
res = evaluate.evaluate_prompts(
prompts, eval_template, eval_data, demos_template, few_shot_data, conf['evaluation']['method'],
conf['evaluation'])
print('Finished evaluating.')
return res
def estimate_cost(eval_template,
demos_template,
prompt_gen_data,
eval_data,
conf,
base_conf='configs/default.yaml',
few_shot_data=None,
prompt_gen_template=None,
eval_query=None):
conf = config.update_config(conf, base_conf)
max_prompt_len = conf['generation']['model']['gpt_config']['max_tokens']
num_prompts = conf['generation']['num_prompts_per_subsample'] * \
conf['generation']['num_subsamples']
eval_method = conf['evaluation']['method']
if eval_method == 'bandits':
num_prompts_per_round = conf['evaluation']['num_prompts_per_round']
if num_prompts_per_round < 1:
num_prompts_per_round = int(
num_prompts * num_prompts_per_round)
num_evals = conf['evaluation']['rounds'] * \
num_prompts_per_round * \
conf['evaluation']['base_eval_config']['num_samples']
else:
num_evals = conf['evaluation']['num_samples'] * num_prompts
# Compute cost of prompt generation
queries = get_generation_query(
eval_template, demos_template, conf, prompt_gen_data, prompt_gen_template, num_query=50)
query_cost = 0
for query in queries:
query_cost += llm.gpt_get_estimated_cost(
conf['generation']['model'], query, max_prompt_len)
total_query_cost = query_cost / len(queries) * num_prompts
# Compute cost of evaluation
if few_shot_data is None:
few_shot_data = prompt_gen_data
queries = get_evaluation_query(
eval_template, demos_template, conf, eval_data, few_shot_data, eval_query, num_query=50)
if conf['evaluation']['method'] == 'bandits':
model_name = conf['evaluation']['base_eval_config']['model']
else:
model_name = conf['evaluation']['model']
query_cost = 0
for query in queries:
query_cost += llm.gpt_get_estimated_cost(model_name, query, 0)
total_eval_cost = query_cost / len(queries) * num_evals
return total_query_cost + total_eval_cost
def get_generation_query(eval_template,
demos_template,
conf,
prompt_gen_data,
prompt_gen_template=None,
num_query=1):
# Generate prompts
eval_template = template.EvalTemplate(eval_template)
demos_template = template.DemosTemplate(demos_template)
if prompt_gen_template is None:
prompt_gen_template = eval_template.convert_to_generation_template()
else:
prompt_gen_template = template.GenerationTemplate(prompt_gen_template)
# First, generate a few prompt queries:
queries = []
for _ in range(num_query):
subsampled_data = data.subsample_data(
prompt_gen_data, conf['generation']['num_demos'])
queries.append(generate.get_query(prompt_gen_template,
demos_template, subsampled_data))
return queries
def get_evaluation_query(eval_template,
demos_template,
conf,
eval_data,
few_shot_data,
eval_query=None,
num_query=1
):
eval_template = template.EvalTemplate(eval_template)
demos_template = template.DemosTemplate(demos_template)
if conf['evaluation']['method'] == 'bandits':
eval_base_method = conf['evaluation']['base_eval_method']
num_few_shot = conf['evaluation']['base_eval_config']['num_few_shot']
else:
eval_base_method = conf['evaluation']['method']
num_few_shot = conf['evaluation']['num_few_shot']
if eval_query is None:
if eval_base_method == 'likelihood':
from automatic_prompt_engineer.evaluation import likelihood
eval_query = likelihood.get_query
else:
raise ValueError(
'Cannot estimate costs for: {}'.format(eval_base_method))
max_prompt_len = conf['generation']['model']['gpt_config']['max_tokens']
filler_prompt = 'GGGG' * max_prompt_len
queries = []
for _ in range(num_query):
idx = random.randint(0, len(eval_data[0]) - 1)
input_, output_ = eval_data[0][idx], eval_data[1][idx]
demo_data = data.subsample_data(few_shot_data, num_few_shot)
query = eval_query(filler_prompt, eval_template, input_,
output_, demo_data, demos_template)[0]
queries.append(query)
return queries
================================================
FILE: InstructZero/experiments/automatic_prompt_engineer/config.py
================================================
import os
import yaml
def update_config(config, base_config='configs/default.yaml'):
# Get default config from yaml
with open(os.path.join(os.path.dirname(__file__), base_config)) as f:
default_config = yaml.safe_load(f)
# Update default config with user config
# Note that the config is a nested dictionary, so we need to update it recursively
def update(d, u):
for k, v in u.items():
if isinstance(v, dict):
d[k] = update(d.get(k, {}), v)
else:
d[k] = v
return d
return update(default_config, config)
def simple_config(eval_model, prompt_gen_model, prompt_gen_mode, num_prompts, eval_rounds, prompt_gen_batch_size, eval_batch_size):
"""Returns a config and splits the data into sensible chunks."""
conf = update_config({}, 'configs/bandits.yaml')
conf['generation']['model']['gpt_config']['model'] = prompt_gen_model
if prompt_gen_mode == 'insert':
conf['generation']['model']['name'] = 'GPT_insert'
conf['generation']['model']['batch_size'] = 1
elif prompt_gen_mode == 'forward':
conf['generation']['model']['name'] = 'GPT_forward'
conf['generation']['model']['batch_size'] = prompt_gen_batch_size
conf['generation']['num_subsamples'] = num_prompts // 10
conf['generation']['num_prompts_per_subsample'] = 10
conf['evaluation']['base_eval_config']['model']['gpt_config']['model'] = eval_model
conf['evaluation']['base_eval_config']['model']['batch_size'] = eval_batch_size
conf['evaluation']['num_prompts_per_round'] = 0.334
conf['evaluation']['rounds'] = eval_rounds
conf['evaluation']['base_eval_config']['num_samples'] = 5
return conf
================================================
FILE: InstructZero/experiments/automatic_prompt_engineer/data.py
================================================
import random
SEED=0
def subsample_data(data, subsample_size):
"""
Subsample data. Data is in the form of a tuple of lists.
"""
#random.seed(SEED)
inputs, outputs = data
assert len(inputs) == len(outputs)
indices = random.sample(range(len(inputs)), subsample_size)
inputs = [inputs[i] for i in indices]
outputs = [outputs[i] for i in indices]
return inputs, outputs
def create_split(data, split_size):
"""
Split data into two parts. Data is in the form of a tuple of lists.
"""
random.seed(SEED)
inputs, outputs = data
assert len(inputs) == len(outputs)
indices = random.sample(range(len(inputs)), split_size)
inputs1 = [inputs[i] for i in indices]
outputs1 = [outputs[i] for i in indices]
inputs2 = [inputs[i] for i in range(len(inputs)) if i not in indices]
outputs2 = [outputs[i] for i in range(len(inputs)) if i not in indices]
return (inputs1, outputs1), (inputs2, outputs2)
================================================
FILE: InstructZero/experiments/automatic_prompt_engineer/evaluate.py
================================================
from abc import ABC, abstractmethod
from . import llm
def get_eval_method(eval_method):
"""
Returns the evaluation method object.
Parameters:
eval_method: The evaluation method to use. ('likelihood')
Returns:
An evaluation method object.
"""
if callable(eval_method):
return eval_method
if eval_method == 'likelihood':
from automatic_prompt_engineer.evaluation import likelihood
return likelihood.likelihood_evaluator
elif eval_method == 'bandits':
from automatic_prompt_engineer.evaluation import bandits
return bandits.bandits_evaluator
else:
raise ValueError('Invalid evaluation method.')
def evaluate_prompts(prompts, eval_template, eval_data, demos_template, few_shot_data, eval_method, config):
"""
Returns the scores for a list of prompts.
Parameters:
prompts: A list of prompts.
eval_template: The template for the evaluation queries.
eval_data: The data to use for evaluation.
eval_method: The evaluation method to use. ('likelihood')
config: The configuration dictionary.
Returns:
An evaluation result object.
"""
eval_method = get_eval_method(eval_method)
return eval_method(prompts, eval_template, eval_data, demos_template, few_shot_data, config)
def demo_function(eval_template, config):
"""
Returns a function that can be manually test the LLM with a chosen prompt.
Parameters:
eval_template: The template for the evaluation queries.
config: The configuration dictionary.
Returns:
A function that takes a prompt and returns a demo.
"""
model = llm.model_from_config(config['model'])
def fn(prompt, inputs):
if not isinstance(inputs, list):
inputs = [inputs]
queries = []
for input_ in inputs:
query = eval_template.fill(prompt=prompt, input=input_)
queries.append(query)
outputs = model.generate_text(
queries, n=1)
return [out.strip().split('\n')[0] for out in outputs]
return fn
class EvaluationResult(ABC):
@abstractmethod
def sorted(self, method='default'):
"""Get the results in the form of a sorted prompt and score list.
Has a method argument to support sorting by various metrics."""
pass
@abstractmethod
def in_place(self, method='default'):
"""Get the results in the form of a list of prompts and scores without sorting."""
pass
================================================
FILE: InstructZero/experiments/automatic_prompt_engineer/generate.py
================================================
from . import data, llm
def get_query(prompt_gen_template, demos_template, subsampled_data):
"""
Returns a query for the prompt generator. A query is the prompt that is sent to the LLM.
Parameters:
prompt_gen_template: The template for the prompt generator queries.
demos_template: The template for the demonstrations.
subsampled_data: The data to use for the demonstrations.
Returns:
A query for the prompt generator.
"""
inputs, outputs = subsampled_data
demos = demos_template.fill(subsampled_data)
return prompt_gen_template.fill(input=inputs[0], output=outputs[0], full_demo=demos)
def generate_prompts(prompt_gen_template, demos_template, prompt_gen_data, config):
"""
Generates prompts using the prompt generator.
Parameters:
prompt_gen_template: The template for the prompt generator queries.
demos_template: The template for the demonstrations.
prompt_gen_data: The data to use for prompt generation.
config: The configuration dictionary.
Returns:
A list of prompts.
"""
queries = []
for _ in range(config['num_subsamples']):
subsampled_data = data.subsample_data(
prompt_gen_data, config['num_demos'])
queries.append(get_query(prompt_gen_template,
demos_template, subsampled_data))
# Instantiate the LLM
model = llm.model_from_config(config['model'], disable_tqdm=False)
prompts = model.generate_text(
queries, n=config['num_prompts_per_subsample'])
return prompts
================================================
FILE: InstructZero/experiments/automatic_prompt_engineer/llm.py
================================================
"""Contains classes for querying large language models."""
import os
import time
from tqdm import tqdm
from abc import ABC, abstractmethod
from transformers import LlamaForCausalLM, LlamaTokenizer
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
import openai
import torch
import asyncio
from typing import Any
gpt_costs_per_thousand = {
'davinci': 0.0200,
'curie': 0.0020,
'babbage': 0.0005,
'ada': 0.0004
}
async def dispatch_openai_requests(
messages_list: list[list[dict[str,Any]]],
model: str,
temperature: float,
max_tokens: int,
frequency_penalty: int,
presence_penalty: int
) -> list[str]:
"""Dispatches requests to OpenAI API asynchronously.
Args:
messages_list: List of messages to be sent to OpenAI ChatCompletion API.
model: OpenAI model to use.
temperature: Temperature to use for the model.
max_tokens: Maximum number of tokens to generate.
top_p: Top p to use for the model.
Returns:
List of responses from OpenAI API.
"""
# for x in messages_list:
# try:
async_responses = [openai.ChatCompletion.acreate(
model=model,
messages=x,
temperature=temperature,
max_tokens=max_tokens,
frequency_penalty=frequency_penalty,
presence_penalty=presence_penalty) for x in messages_list]
return await asyncio.gather(*async_responses)
def model_from_config(config, disable_tqdm=True):
"""Returns a model based on the config."""
model_type = config["name"]
if model_type == "GPT_forward":
return GPT_Forward(config, disable_tqdm=disable_tqdm)
elif model_type == "GPT_insert":
return GPT_Insert(config, disable_tqdm=disable_tqdm)
elif model_type == "Llama_Forward":
return Llama_Forward(config, disable_tqdm=disable_tqdm)
raise ValueError(f"Unknown model type: {model_type}")
class LLM(ABC):
"""Abstract base class for large language models."""
@abstractmethod
def generate_text(self, prompt):
"""Generates text from the model.
Parameters:
prompt: The prompt to use. This can be a string or a list of strings.
Returns:
A list of strings.
"""
pass
@abstractmethod
def log_probs(self, text, log_prob_range):
"""Returns the log probs of the text.
Parameters:
text: The text to get the log probs of. This can be a string or a list of strings.
log_prob_range: The range of characters within each string to get the log_probs of.
This is a list of tuples of the form (start, end).
Returns:
A list of log probs.
"""
pass
class Llama_Forward(LLM):
"""Wrapper for llama."""
def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
"""Initializes the model."""
SIZE=13
MODEL_DIR = '/data/bobchen/llama4trans/llama-{}b'.format(SIZE)
TOKENIZER_DIR = '/data/bobchen/llama4trans/tokenizer'
self.config = config
self.needs_confirmation = needs_confirmation
self.disable_tqdm = disable_tqdm
self.model=LlamaForCausalLM.from_pretrained(MODEL_DIR, device_map="auto")
self.tokenizer=LlamaTokenizer.from_pretrained(TOKENIZER_DIR)
def auto_reduce_n(self, fn, prompt, n):
"""Reduces n by half until the function succeeds."""
try:
return fn(prompt, n)
except BatchSizeException as e:
if n == 1:
raise e
return self.auto_reduce_n(fn, prompt, n // 2) + self.auto_reduce_n(fn, prompt, n // 2)
def generate_text(self, prompts, n):
if not isinstance(prompts, list):
prompts = [prompts]
text = []
for prompt in prompts:
input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids.cuda()
# Generate
generate_ids = self.model.generate(input_ids, max_new_tokens=32)
text.append(self.tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0])
return text
def complete(self, prompt, n):
"""Generates text from the model and returns the log prob data."""
if not isinstance(prompt, list):
prompt = [prompt]
batch_size = self.config['batch_size']
prompt_batches = [prompt[i:i + batch_size]
for i in range(0, len(prompt), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Generating {len(prompt) * n} completions, "
f"split into {len(prompt_batches)} batches of size {batch_size * n}")
res = []
for prompt_batch in tqdm(prompt_batches, disable=self.disable_tqdm):
res += self.__complete(prompt_batch, n)
return res
def log_probs(self, text, log_prob_range=None):
"""Returns the log probs of the text."""
if not isinstance(text, list):
text = [text]
if self.needs_confirmation:
self.confirm_cost(text, 1, 0)
batch_size = self.config['batch_size']
text_batches = [text[i:i + batch_size]
for i in range(0, len(text), batch_size)]
if log_prob_range is None:
log_prob_range_batches = [None] * len(text)
else:
assert len(log_prob_range) == len(text)
log_prob_range_batches = [log_prob_range[i:i + batch_size]
for i in range(0, len(log_prob_range), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Getting log probs for {len(text)} strings, "
f"split into {len(text_batches)} batches of (maximum) size {batch_size}")
log_probs = []
tokens = []
for text_batch, log_prob_range in tqdm(list(zip(text_batches, log_prob_range_batches)),
disable=self.disable_tqdm):
log_probs_batch, tokens_batch = self.__log_probs(
text_batch, log_prob_range)
log_probs += log_probs_batch
tokens += tokens_batch
return log_probs, tokens
class Flan_T5(LLM):
"""Wrapper for llama."""
def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
"""Initializes the model."""
self.device="cuda:1"
self.config = config
self.needs_confirmation = needs_confirmation
self.disable_tqdm = disable_tqdm
self.model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-xxl",
torch_dtype=torch.float16).to(device=self.device)
self.tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
def auto_reduce_n(self, fn, prompt, n):
"""Reduces n by half until the function succeeds."""
try:
return fn(prompt, n)
except BatchSizeException as e:
if n == 1:
raise e
return self.auto_reduce_n(fn, prompt, n // 2) + self.auto_reduce_n(fn, prompt, n // 2)
def generate_text(self, prompts, n):
if not isinstance(prompts, list):
prompts = [prompts]
# prompt_batches = [prompt[i:i + batch_size]
# for i in range(0, len(prompt), batch_size)]
# if not self.disable_tqdm:
# print(
# f"[{self.config['name']}] Generating {len(prompt) * n} completions, "
# f"split into {len(prompt_batches)} batches of size {batch_size * n}")
text = []
batch_size=10
for i in range(len(prompts) // batch_size):
tmp_prompts = prompts[i*batch_size:(i+1)*batch_size]
input_ids = self.tokenizer(tmp_prompts, padding='longest', return_tensors="pt").input_ids.to(device=self.device)
outputs = self.model.generate(input_ids, max_new_tokens=32)
text += self.tokenizer.batch_decode(outputs, skip_special_tokens=True)
# batch_size = int(len(prompts)/2)
# prompts1 = prompts[:batch_size]
# prompts2 = prompts[batch_size:]
# input_ids1 = self.tokenizer(prompts1, padding='longest', return_tensors="pt").input_ids.to(device=self.device)
# input_ids2 = self.tokenizer(prompts2, padding='longest', return_tensors="pt").input_ids.to(device=self.device)
# outputs1 = self.model.generate(input_ids1, max_new_tokens=20)
# text += self.tokenizer.batch_decode(outputs1, skip_special_tokens=True)
# outputs2 = self.model.generate(input_ids2, max_new_tokens=20)
# text += self.tokenizer.batch_decode(outputs2, skip_special_tokens=True)
# batch_size=1
return text
def complete(self, prompt, n):
"""Generates text from the model and returns the log prob data."""
if not isinstance(prompt, list):
prompt = [prompt]
batch_size = self.config['batch_size']
prompt_batches = [prompt[i:i + batch_size]
for i in range(0, len(prompt), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Generating {len(prompt) * n} completions, "
f"split into {len(prompt_batches)} batches of size {batch_size * n}")
res = []
for prompt_batch in tqdm(prompt_batches, disable=self.disable_tqdm):
res += self.__complete(prompt_batch, n)
return res
def log_probs(self, text, log_prob_range=None):
"""Returns the log probs of the text."""
if not isinstance(text, list):
text = [text]
if self.needs_confirmation:
self.confirm_cost(text, 1, 0)
batch_size = self.config['batch_size']
text_batches = [text[i:i + batch_size]
for i in range(0, len(text), batch_size)]
if log_prob_range is None:
log_prob_range_batches = [None] * len(text)
else:
assert len(log_prob_range) == len(text)
log_prob_range_batches = [log_prob_range[i:i + batch_size]
for i in range(0, len(log_prob_range), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Getting log probs for {len(text)} strings, "
f"split into {len(text_batches)} batches of (maximum) size {batch_size}")
log_probs = []
tokens = []
for text_batch, log_prob_range in tqdm(list(zip(text_batches, log_prob_range_batches)),
disable=self.disable_tqdm):
log_probs_batch, tokens_batch = self.__log_probs(
text_batch, log_prob_range)
log_probs += log_probs_batch
tokens += tokens_batch
return log_probs, tokens
class GPT_Forward(LLM):
"""Wrapper for ChatGPT."""
def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
"""Initializes the model."""
self.config = config
self.needs_confirmation = needs_confirmation
self.disable_tqdm = disable_tqdm
def confirm_cost(self, texts, n, max_tokens):
total_estimated_cost = 0
for text in texts:
total_estimated_cost += gpt_get_estimated_cost(
self.config, text, max_tokens) * n
print(f"Estimated cost: ${total_estimated_cost:.2f}")
# Ask the user to confirm in the command line
if os.getenv("LLM_SKIP_CONFIRM") is None:
confirm = input("Continue? (y/n) ")
if confirm != 'y':
raise Exception("Aborted.")
def auto_reduce_n(self, fn, prompt, n):
"""Reduces n by half until the function succeeds."""
try:
return fn(prompt, n)
except BatchSizeException as e:
if n == 1:
raise e
return self.auto_reduce_n(fn, prompt, n // 2) + self.auto_reduce_n(fn, prompt, n // 2)
def generate_text(self, prompt, n):
if not isinstance(prompt, list):
prompt = [prompt]
if self.needs_confirmation:
self.confirm_cost(
prompt, n, self.config['gpt_config']['max_tokens'])
batch_size = self.config['batch_size']
prompt_batches = [prompt[i:i + batch_size]
for i in range(0, len(prompt), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Generating {len(prompt) * n} completions, "
f"split into {len(prompt_batches)} batches of size {batch_size * n}")
text = []
for prompt_batch in tqdm(prompt_batches, disable=self.disable_tqdm):
# text += self.auto_reduce_n(self.__generate_text, prompt_batch, n)
text += self.__async_generate(prompt_batch, n)
return text
def complete(self, prompt, n):
"""Generates text from the model and returns the log prob data."""
if not isinstance(prompt, list):
prompt = [prompt]
batch_size = self.config['batch_size']
prompt_batches = [prompt[i:i + batch_size]
for i in range(0, len(prompt), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Generating {len(prompt) * n} completions, "
f"split into {len(prompt_batches)} batches of size {batch_size * n}")
res = []
for prompt_batch in tqdm(prompt_batches, disable=self.disable_tqdm):
res += self.__complete(prompt_batch, n)
return res
def log_probs(self, text, log_prob_range=None):
"""Returns the log probs of the text."""
if not isinstance(text, list):
text = [text]
if self.needs_confirmation:
self.confirm_cost(text, 1, 0)
batch_size = self.config['batch_size']
text_batches = [text[i:i + batch_size]
for i in range(0, len(text), batch_size)]
if log_prob_range is None:
log_prob_range_batches = [None] * len(text)
else:
assert len(log_prob_range) == len(text)
log_prob_range_batches = [log_prob_range[i:i + batch_size]
for i in range(0, len(log_prob_range), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Getting log probs for {len(text)} strings, "
f"split into {len(text_batches)} batches of (maximum) size {batch_size}")
log_probs = []
tokens = []
for text_batch, log_prob_range in tqdm(list(zip(text_batches, log_prob_range_batches)),
disable=self.disable_tqdm):
log_probs_batch, tokens_batch = self.__log_probs(
text_batch, log_prob_range)
log_probs += log_probs_batch
tokens += tokens_batch
return log_probs, tokens
def __generate_text(self, prompt, n):
"""Generates text from the model."""
if not isinstance(prompt, list):
text = [prompt]
config = self.config['gpt_config'].copy()
config['n'] = n
answer = []
# If there are any [APE] tokens in the prompts, remove them
for i in range(len(prompt)):
prompt_single = prompt[i].replace('[APE]', '').strip()
response = None
while response is None:
try:
response = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": prompt_single}],
temperature=0.0,
max_tokens=256,
frequency_penalty=0,
presence_penalty=0)
except Exception as e:
if 'is greater than the maximum' in str(e):
raise BatchSizeException()
print(e)
print('Retrying...')
time.sleep(5)
try:
# print(response['choices'][0]["message"]["content"])
answer.append(response['choices'][0]["message"]["content"])
except Exception:
answer.append('do not have reponse from chatgpt')
return answer
def __async_generate(self, prompt, n):
ml = [[{"role": "user", "content": p.replace('[APE]', '').strip()}] for p in prompt]
answer = None
while answer is None:
try:
predictions = asyncio.run(dispatch_openai_requests(
messages_list = ml,
model='gpt-3.5-turbo',
temperature=0,
max_tokens=256,
frequency_penalty=0,
presence_penalty=0
)
)
except Exception as e:
# if 'is greater than the maximum' in str(e):
# raise BatchSizeException()
print(e)
print("Retrying....")
time.sleep(20)
try:
answer = [x['choices'][0]['message']['content'] for x in predictions]
except Exception:
print("Please Wait!")
return answer
# try:
# reply = openai.ChatCompletion.create(
# model="gpt-3.5-turbo",
# messages=[{"role": "user", "content": input}],
# temperature=0.0,
# max_tokens=args.max_length_cot,
# frequency_penalty=0,
# presence_penalty=0)
# reply = reply['choices'][0]["message"]["content"].replace('\n\n', '')
def __complete(self, prompt, n):
"""Generates text from the model and returns the log prob data."""
if not isinstance(prompt, list):
text = [prompt]
config = self.config['gpt_config'].copy()
config['n'] = n
# If there are any [APE] tokens in the prompts, remove them
for i in range(len(prompt)):
prompt[i] = prompt[i].replace('[APE]', '').strip()
response = None
while response is None:
try:
response = openai.Completion.create(
**config, prompt=prompt)
except Exception as e:
print(e)
print('Retrying...')
time.sleep(5)
return response['choices']
def __log_probs(self, text, log_prob_range=None):
"""Returns the log probs of the text."""
if not isinstance(text, list):
text = [text]
if log_prob_range is not None:
for i in range(len(text)):
lower_index, upper_index = log_prob_range[i]
assert lower_index < upper_index
assert lower_index >= 0
assert upper_index - 1 < len(text[i])
config = self.config['gpt_config'].copy()
config['logprobs'] = 1
config['echo'] = True
config['max_tokens'] = 0
if isinstance(text, list):
text = [f'\n{text[i]}' for i in range(len(text))]
else:
text = f'\n{text}'
response = None
while response is None:
try:
response = openai.Completion.create(
**config, prompt=text)
# import pdb;pdb.set_trace()
except Exception as e:
print(e)
print('Retrying...')
time.sleep(5)
log_probs = [response['choices'][i]['logprobs']['token_logprobs'][1:]
for i in range(len(response['choices']))]
tokens = [response['choices'][i]['logprobs']['tokens'][1:]
for i in range(len(response['choices']))]
offsets = [response['choices'][i]['logprobs']['text_offset'][1:]
for i in range(len(response['choices']))]
# Subtract 1 from the offsets to account for the newline
for i in range(len(offsets)):
offsets[i] = [offset - 1 for offset in offsets[i]]
if log_prob_range is not None:
# First, we need to find the indices of the tokens in the log probs
# that correspond to the tokens in the log_prob_range
for i in range(len(log_probs)):
lower_index, upper_index = self.get_token_indices(
offsets[i], log_prob_range[i])
log_probs[i] = log_probs[i][lower_index:upper_index]
tokens[i] = tokens[i][lower_index:upper_index]
return log_probs, tokens
def get_token_indices(self, offsets, log_prob_range):
"""Returns the indices of the tokens in the log probs that correspond to the tokens in the log_prob_range."""
# For the lower index, find the highest index that is less than or equal to the lower index
lower_index = 0
for i in range(len(offsets)):
if offsets[i] <= log_prob_range[0]:
lower_index = i
else:
break
upper_index = len(offsets)
for i in range(len(offsets)):
if offsets[i] >= log_prob_range[1]:
upper_index = i
break
return lower_index, upper_index
class Claude_Forward(LLM):
"""Wrapper for Claude."""
def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
"""Initializes the model."""
self.config = config
self.needs_confirmation = needs_confirmation
self.disable_tqdm = disable_tqdm
def confirm_cost(self, texts, n, max_tokens):
total_estimated_cost = 0
for text in texts:
total_estimated_cost += gpt_get_estimated_cost(
self.config, text, max_tokens) * n
print(f"Estimated cost: ${total_estimated_cost:.2f}")
# Ask the user to confirm in the command line
if os.getenv("LLM_SKIP_CONFIRM") is None:
confirm = input("Continue? (y/n) ")
if confirm != 'y':
raise Exception("Aborted.")
def auto_reduce_n(self, fn, prompt, n):
"""Reduces n by half until the function succeeds."""
try:
return fn(prompt, n)
except BatchSizeException as e:
if n == 1:
raise e
return self.auto_reduce_n(fn, prompt, n // 2) + self.auto_reduce_n(fn, prompt, n // 2)
def generate_text(self, prompt, n):
if not isinstance(prompt, list):
prompt = [prompt]
if self.needs_confirmation:
self.confirm_cost(
prompt, n, self.config['gpt_config']['max_tokens'])
batch_size = self.config['batch_size']
prompt_batches = [prompt[i:i + batch_size]
for i in range(0, len(prompt), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Generating {len(prompt) * n} completions, "
f"split into {len(prompt_batches)} batches of size {batch_size * n}")
text = []
for prompt_batch in tqdm(prompt_batches, disable=self.disable_tqdm):
# text += self.auto_reduce_n(self.__generate_text, prompt_batch, n)
text += self.__async_generate(prompt_batch, n)
return text
def complete(self, prompt, n):
"""Generates text from the model and returns the log prob data."""
if not isinstance(prompt, list):
prompt = [prompt]
batch_size = self.config['batch_size']
prompt_batches = [prompt[i:i + batch_size]
for i in range(0, len(prompt), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Generating {len(prompt) * n} completions, "
f"split into {len(prompt_batches)} batches of size {batch_size * n}")
res = []
for prompt_batch in tqdm(prompt_batches, disable=self.disable_tqdm):
res += self.__complete(prompt_batch, n)
return res
def log_probs(self, text, log_prob_range=None):
"""Returns the log probs of the text."""
if not isinstance(text, list):
text = [text]
if self.needs_confirmation:
self.confirm_cost(text, 1, 0)
batch_size = self.config['batch_size']
text_batches = [text[i:i + batch_size]
for i in range(0, len(text), batch_size)]
if log_prob_range is None:
log_prob_range_batches = [None] * len(text)
else:
assert len(log_prob_range) == len(text)
log_prob_range_batches = [log_prob_range[i:i + batch_size]
for i in range(0, len(log_prob_range), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Getting log probs for {len(text)} strings, "
f"split into {len(text_batches)} batches of (maximum) size {batch_size}")
log_probs = []
tokens = []
for text_batch, log_prob_range in tqdm(list(zip(text_batches, log_prob_range_batches)),
disable=self.disable_tqdm):
log_probs_batch, tokens_batch = self.__log_probs(
text_batch, log_prob_range)
log_probs += log_probs_batch
tokens += tokens_batch
return log_probs, tokens
def __generate_text(self, prompt, n):
"""Generates text from the model."""
if not isinstance(prompt, list):
text = [prompt]
config = self.config['gpt_config'].copy()
config['n'] = n
answer = []
# If there are any [APE] tokens in the prompts, remove them
for i in range(len(prompt)):
prompt_single = prompt[i].replace('[APE]', '').strip()
response = None
while response is None:
try:
response = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": prompt_single}],
temperature=0.0,
max_tokens=256,
frequency_penalty=0,
presence_penalty=0)
except Exception as e:
if 'is greater than the maximum' in str(e):
raise BatchSizeException()
print(e)
print('Retrying...')
time.sleep(5)
try:
# print(response['choices'][0]["message"]["content"])
answer.append(response['choices'][0]["message"]["content"])
except Exception:
answer.append('do not have reponse from chatgpt')
return answer
def __async_generate(self, prompt, n):
ml = [[{"role": "user", "content": p.replace('[APE]', '').strip()}] for p in prompt]
answer = None
while answer is None:
try:
predictions = asyncio.run(dispatch_openai_requests(
messages_list = ml,
model='gpt-3.5-turbo',
temperature=0,
max_tokens=256,
frequency_penalty=0,
presence_penalty=0
)
)
except Exception as e:
# if 'is greater than the maximum' in str(e):
# raise BatchSizeException()
print(e)
print("Retrying....")
time.sleep(20)
try:
answer = [x['choices'][0]['message']['content'] for x in predictions]
except Exception:
print("Please Wait!")
return answer
# try:
# reply = openai.ChatCompletion.create(
# model="gpt-3.5-turbo",
# messages=[{"role": "user", "content": input}],
# temperature=0.0,
# max_tokens=args.max_length_cot,
# frequency_penalty=0,
# presence_penalty=0)
# reply = reply['choices'][0]["message"]["content"].replace('\n\n', '')
def __complete(self, prompt, n):
"""Generates text from the model and returns the log prob data."""
if not isinstance(prompt, list):
text = [prompt]
config = self.config['gpt_config'].copy()
config['n'] = n
# If there are any [APE] tokens in the prompts, remove them
for i in range(len(prompt)):
prompt[i] = prompt[i].replace('[APE]', '').strip()
response = None
while response is None:
try:
response = openai.Completion.create(
**config, prompt=prompt)
except Exception as e:
print(e)
print('Retrying...')
time.sleep(5)
return response['choices']
def __log_probs(self, text, log_prob_range=None):
"""Returns the log probs of the text."""
if not isinstance(text, list):
text = [text]
if log_prob_range is not None:
for i in range(len(text)):
lower_index, upper_index = log_prob_range[i]
assert lower_index < upper_index
assert lower_index >= 0
assert upper_index - 1 < len(text[i])
config = self.config['gpt_config'].copy()
config['logprobs'] = 1
config['echo'] = True
config['max_tokens'] = 0
if isinstance(text, list):
text = [f'\n{text[i]}' for i in range(len(text))]
else:
text = f'\n{text}'
response = None
while response is None:
try:
response = openai.Completion.create(
**config, prompt=text)
# import pdb;pdb.set_trace()
except Exception as e:
print(e)
print('Retrying...')
time.sleep(5)
log_probs = [response['choices'][i]['logprobs']['token_logprobs'][1:]
for i in range(len(response['choices']))]
tokens = [response['choices'][i]['logprobs']['tokens'][1:]
for i in range(len(response['choices']))]
offsets = [response['choices'][i]['logprobs']['text_offset'][1:]
for i in range(len(response['choices']))]
# Subtract 1 from the offsets to account for the newline
for i in range(len(offsets)):
offsets[i] = [offset - 1 for offset in offsets[i]]
if log_prob_range is not None:
# First, we need to find the indices of the tokens in the log probs
# that correspond to the tokens in the log_prob_range
for i in range(len(log_probs)):
lower_index, upper_index = self.get_token_indices(
offsets[i], log_prob_range[i])
log_probs[i] = log_probs[i][lower_index:upper_index]
tokens[i] = tokens[i][lower_index:upper_index]
return log_probs, tokens
def get_token_indices(self, offsets, log_prob_range):
"""Returns the indices of the tokens in the log probs that correspond to the tokens in the log_prob_range."""
# For the lower index, find the highest index that is less than or equal to the lower index
lower_index = 0
for i in range(len(offsets)):
if offsets[i] <= log_prob_range[0]:
lower_index = i
else:
break
upper_index = len(offsets)
for i in range(len(offsets)):
if offsets[i] >= log_prob_range[1]:
upper_index = i
break
return lower_index, upper_index
class GPT_Insert(LLM):
def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
"""Initializes the model."""
self.config = config
self.needs_confirmation = needs_confirmation
self.disable_tqdm = disable_tqdm
def confirm_cost(self, texts, n, max_tokens):
total_estimated_cost = 0
for text in texts:
total_estimated_cost += gpt_get_estimated_cost(
self.config, text, max_tokens) * n
print(f"Estimated cost: ${total_estimated_cost:.2f}")
# Ask the user to confirm in the command line
if os.getenv("LLM_SKIP_CONFIRM") is None:
confirm = input("Continue? (y/n) ")
if confirm != 'y':
raise Exception("Aborted.")
def auto_reduce_n(self, fn, prompt, n):
"""Reduces n by half until the function succeeds."""
try:
return fn(prompt, n)
except BatchSizeException as e:
if n == 1:
raise e
return self.auto_reduce_n(fn, prompt, n // 2) + self.auto_reduce_n(fn, prompt, n // 2)
def generate_text(self, prompt, n):
if not isinstance(prompt, list):
prompt = [prompt]
if self.needs_confirmation:
self.confirm_cost(
prompt, n, self.config['gpt_config']['max_tokens'])
batch_size = self.config['batch_size']
assert batch_size == 1
prompt_batches = [prompt[i:i + batch_size]
for i in range(0, len(prompt), batch_size)]
if not self.disable_tqdm:
print(
f"[{self.config['name']}] Generating {len(prompt) * n} completions, split into {len(prompt_batches)} batches of (maximum) size {batch_size * n}")
text = []
for prompt_batch in tqdm(prompt_batches, disable=self.disable_tqdm):
text += self.auto_reduce_n(self.__generate_text, prompt_batch, n)
return text
def log_probs(self, text, log_prob_range=None):
raise NotImplementedError
def __generate_text(self, prompt, n):
"""Generates text from the model."""
config = self.config['gpt_config'].copy()
config['n'] = n
# Split prompts into prefixes and suffixes with the [APE] token (do not include the [APE] token in the suffix)
prefix = prompt[0].split('[APE]')[0]
suffix = prompt[0].split('[APE]')[1]
response = None
while response is None:
try:
response = openai.ChatCompletion.create(
**config, prompt=prefix, suffix=suffix)
except Exception as e:
print(e)
print('Retrying...')
time.sleep(5)
# Remove suffix from the generated text
texts = [response['choices'][i]['text'].replace(suffix, '') for i in range(len(response['choices']))]
return texts
def gpt_get_estimated_cost(config, prompt, max_tokens):
"""Uses the current API costs/1000 tokens to estimate the cost of generating text from the model."""
# Get rid of [APE] token
prompt = prompt.replace('[APE]', '')
# Get the number of tokens in the prompt
n_prompt_tokens = len(prompt) // 4
# Get the number of tokens in the generated text
total_tokens = n_prompt_tokens + max_tokens
engine = config['gpt_config']['model'].split('-')[1]
costs_per_thousand = gpt_costs_per_thousand
if engine not in costs_per_thousand:
# Try as if it is a fine-tuned model
engine = config['gpt_config']['model'].split(':')[0]
costs_per_thousand = {
'davinci': 0.1200,
'curie': 0.0120,
'babbage': 0.0024,
'ada': 0.0016
}
price = costs_per_thousand[engine] * total_tokens / 1000
return price
class BatchSizeException(Exception):
pass
================================================
FILE: InstructZero/experiments/automatic_prompt_engineer/template.py
================================================
class InitQATemplate:
"""
Takes a prompt template and provides methods for filling in blanks.
The format is as follows:
[APE] is where text will be generated by the LLM.
[full_DEMO] is where the full demo will be inserted.
[INPUT] is where the input to the first demo will be inserted.
[OUTPUT] is where the output from the first demo will be inserted.
"""
def __init__(self, template):
self.template = template
# Check that the template is valid
# There should be exactly one [APE] token
# assert self.template.count('[APE]') == 1
def fill(self, full_demo=''):
"""
Fills in the template with the given values.
"""
return self.template.replace('[full_DEMO]', full_demo)
class GenerationTemplate:
"""
Takes a prompt template and provides methods for filling in blanks.
The format is as follows:
[APE] is where text will be generated by the LLM.
[full_DEMO] is where the full demo will be inserted.
[INPUT] is where the input to the first demo will be inserted.
[OUTPUT] is where the output from the first demo will be inserted.
"""
def __init__(self, template):
self.template = template
# Check that the template is valid
# There should be exactly one [APE] token
# assert self.template.count('[APE]') == 1
def fill(self, full_demo='', input='', output=''):
"""
Fills in the template with the given values.
"""
return self.template.replace('[full_DEMO]', full_demo).replace(
'[INPUT]', input).replace('[OUTPUT]', output)
class EvalTemplate:
"""
Takes a prompt template and provides methods for filling in blanks.
The format is as follows:
[PROMPT] is where the prompt will be inserted.
[full_DEMO] is where the full demo will be inserted.
[INPUT] is where the input to the first demo will be inserted.
[OUTPUT] is where the output from the first demo will be inserted.
"""
def __init__(self, template):
self.template = template
def fill(self, prompt='', full_demo='', input='', output=''):
"""
Fills in the template with the given values.
"""
return self.template.replace('[PROMPT]', prompt).replace(
'[full_DEMO]', full_demo).replace('[INPUT]', input).replace('[OUTPUT]', output)
def convert_to_generation_template(self):
"""
Converts the evaluation template to a generation template.
"""
return GenerationTemplate(self.template.replace('[PROMPT]', '[APE]'))
class DemosTemplate:
"""
Takes a template for the full demo and provides methods for filling in blanks.
The format is as follows:
[INPUT], [OUTPUT]
"""
def __init__(self, template, delimiter='\n\n'):
self.template = template
self.delimiter = delimiter
def fill(self, data):
"""
Fills in the template with the given values. Data is a tuple of lists.
"""
demos = ''
for i, (input_, output_) in enumerate(zip(*data)):
demos += self.template.replace('[INPUT]', input_).replace(
'[OUTPUT]', output_)
if i != len(data[0]) - 1:
demos += self.delimiter
return demos
================================================
FILE: InstructZero/experiments/configs/instruction_induction.yaml
================================================
generation:
num_subsamples: 3
num_demos: 5
num_prompts_per_subsample: 30
model:
name: GPT_forward
batch_size: 500
gpt_config:
model: text-davinci-002
temperature: 0.9
max_tokens: 50
top_p: 0.9
frequency_penalty: 0.0
presence_penalty: 0.0
evaluation:
method: exec_accuracy
num_samples: 30
num_few_shot: 5
model:
name: GPT_forward
batch_size: 20
gpt_config:
model: text-davinci-002
temperature: 0.7
max_tokens: 200
top_p: 1.0
frequency_penalty: 0.0
presence_penalty: 0.0
demo:
model:
name: GPT_forward
batch_size: 500
gpt_config:
model: text-davinci-002
temperature: 0.7
max_tokens: 200
top_p: 1.0
frequency_penalty: 0.0
presence_penalty: 0.0
================================================
FILE: InstructZero/experiments/data/instruction_induction/README.md
================================================
# Data
Data for the instruction induction experiments. Followed We collect another 8 instruction induction tasks besides 24 tasks in [Instruction Induction](https://arxiv.org/abs/2205.10782).
## Content
- induction_input: includes the inputs utilized in our experiments on instruction induction. Each input consists of our instruction induction prompt along with five demonstrations of task input-output.
- raw:
- induce: comprises input-output demonstrations that were utilized in the creation of the instruction induction inputs.
- execute: examples that were withheld for evaluating the accuracy of execution.
================================================
FILE: InstructZero/experiments/data/instruction_induction/induction_input/active_to_passive.json
================================================
{
"examples": {
"1": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The actor encouraged the scientists.\nOutput: The scientists were encouraged by the actor.\n\nInput: The authors thanked the actors.\nOutput: The actors were thanked by the authors.\n\nInput: The secretaries thanked the athlete.\nOutput: The athlete was thanked by the secretaries.\n\nInput: The secretary stopped the presidents.\nOutput: The presidents were stopped by the secretary.\n\nInput: The secretary stopped the manager.\nOutput: The manager was stopped by the secretary.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The actor encouraged the scientists.\nOutput: The scientists were encouraged by the actor.\n\nInput: The authors thanked the actors.\nOutput: The actors were thanked by the authors.\n\nInput: The secretaries thanked the athlete.\nOutput: The athlete was thanked by the secretaries.\n\nInput: The secretary stopped the presidents.\nOutput: The presidents were stopped by the secretary.\n\nInput: The secretary stopped the manager.\nOutput: The manager was stopped by the secretary."
}
},
"2": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The bankers advised the presidents.\nOutput: The presidents were advised by the bankers.\n\nInput: The athlete avoided the managers.\nOutput: The managers were avoided by the athlete.\n\nInput: The artist introduced the lawyer.\nOutput: The lawyer was introduced by the artist.\n\nInput: The authors advised the student.\nOutput: The student was advised by the authors.\n\nInput: The secretaries thanked the athletes.\nOutput: The athletes were thanked by the secretaries.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The bankers advised the presidents.\nOutput: The presidents were advised by the bankers.\n\nInput: The athlete avoided the managers.\nOutput: The managers were avoided by the athlete.\n\nInput: The artist introduced the lawyer.\nOutput: The lawyer was introduced by the artist.\n\nInput: The authors advised the student.\nOutput: The student was advised by the authors.\n\nInput: The secretaries thanked the athletes.\nOutput: The athletes were thanked by the secretaries."
}
},
"3": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The presidents supported the senator.\nOutput: The senator was supported by the presidents.\n\nInput: The secretary recommended the students.\nOutput: The students were recommended by the secretary.\n\nInput: The senators encouraged the lawyers.\nOutput: The lawyers were encouraged by the senators.\n\nInput: The authors encouraged the students.\nOutput: The students were encouraged by the authors.\n\nInput: The presidents introduced the athlete.\nOutput: The athlete was introduced by the presidents.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The presidents supported the senator.\nOutput: The senator was supported by the presidents.\n\nInput: The secretary recommended the students.\nOutput: The students were recommended by the secretary.\n\nInput: The senators encouraged the lawyers.\nOutput: The lawyers were encouraged by the senators.\n\nInput: The authors encouraged the students.\nOutput: The students were encouraged by the authors.\n\nInput: The presidents introduced the athlete.\nOutput: The athlete was introduced by the presidents."
}
},
"4": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The lawyer admired the actor.\nOutput: The actor was admired by the lawyer.\n\nInput: The athletes recognized the judge.\nOutput: The judge was recognized by the athletes.\n\nInput: The authors avoided the banker.\nOutput: The banker was avoided by the authors.\n\nInput: The professor helped the actor.\nOutput: The actor was helped by the professor.\n\nInput: The bankers admired the manager.\nOutput: The manager was admired by the bankers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The lawyer admired the actor.\nOutput: The actor was admired by the lawyer.\n\nInput: The athletes recognized the judge.\nOutput: The judge was recognized by the athletes.\n\nInput: The authors avoided the banker.\nOutput: The banker was avoided by the authors.\n\nInput: The professor helped the actor.\nOutput: The actor was helped by the professor.\n\nInput: The bankers admired the manager.\nOutput: The manager was admired by the bankers."
}
},
"5": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The artist avoided the judge.\nOutput: The judge was avoided by the artist.\n\nInput: The secretary believed the banker.\nOutput: The banker was believed by the secretary.\n\nInput: The professors introduced the senators.\nOutput: The senators were introduced by the professors.\n\nInput: The doctor avoided the lawyers.\nOutput: The lawyers were avoided by the doctor.\n\nInput: The senators recognized the actor.\nOutput: The actor was recognized by the senators.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The artist avoided the judge.\nOutput: The judge was avoided by the artist.\n\nInput: The secretary believed the banker.\nOutput: The banker was believed by the secretary.\n\nInput: The professors introduced the senators.\nOutput: The senators were introduced by the professors.\n\nInput: The doctor avoided the lawyers.\nOutput: The lawyers were avoided by the doctor.\n\nInput: The senators recognized the actor.\nOutput: The actor was recognized by the senators."
}
},
"6": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The senators admired the managers.\nOutput: The managers were admired by the senators.\n\nInput: The student encouraged the tourists.\nOutput: The tourists were encouraged by the student.\n\nInput: The doctor recommended the students.\nOutput: The students were recommended by the doctor.\n\nInput: The tourist helped the artists.\nOutput: The artists were helped by the tourist.\n\nInput: The manager contacted the actor.\nOutput: The actor was contacted by the manager.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The senators admired the managers.\nOutput: The managers were admired by the senators.\n\nInput: The student encouraged the tourists.\nOutput: The tourists were encouraged by the student.\n\nInput: The doctor recommended the students.\nOutput: The students were recommended by the doctor.\n\nInput: The tourist helped the artists.\nOutput: The artists were helped by the tourist.\n\nInput: The manager contacted the actor.\nOutput: The actor was contacted by the manager."
}
},
"7": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The author introduced the bankers.\nOutput: The bankers were introduced by the author.\n\nInput: The artists admired the athlete.\nOutput: The athlete was admired by the artists.\n\nInput: The senators introduced the banker.\nOutput: The banker was introduced by the senators.\n\nInput: The senators admired the athlete.\nOutput: The athlete was admired by the senators.\n\nInput: The actors believed the manager.\nOutput: The manager was believed by the actors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The author introduced the bankers.\nOutput: The bankers were introduced by the author.\n\nInput: The artists admired the athlete.\nOutput: The athlete was admired by the artists.\n\nInput: The senators introduced the banker.\nOutput: The banker was introduced by the senators.\n\nInput: The senators admired the athlete.\nOutput: The athlete was admired by the senators.\n\nInput: The actors believed the manager.\nOutput: The manager was believed by the actors."
}
},
"8": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The doctor recommended the students.\nOutput: The students were recommended by the doctor.\n\nInput: The senators advised the professors.\nOutput: The professors were advised by the senators.\n\nInput: The judge contacted the banker.\nOutput: The banker was contacted by the judge.\n\nInput: The presidents contacted the professor.\nOutput: The professor was contacted by the presidents.\n\nInput: The senators avoided the lawyer.\nOutput: The lawyer was avoided by the senators.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The doctor recommended the students.\nOutput: The students were recommended by the doctor.\n\nInput: The senators advised the professors.\nOutput: The professors were advised by the senators.\n\nInput: The judge contacted the banker.\nOutput: The banker was contacted by the judge.\n\nInput: The presidents contacted the professor.\nOutput: The professor was contacted by the presidents.\n\nInput: The senators avoided the lawyer.\nOutput: The lawyer was avoided by the senators."
}
},
"9": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The tourists mentioned the manager.\nOutput: The manager was mentioned by the tourists.\n\nInput: The lawyers encouraged the professors.\nOutput: The professors were encouraged by the lawyers.\n\nInput: The managers advised the professors.\nOutput: The professors were advised by the managers.\n\nInput: The actor thanked the senators.\nOutput: The senators were thanked by the actor.\n\nInput: The managers introduced the tourist.\nOutput: The tourist was introduced by the managers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The tourists mentioned the manager.\nOutput: The manager was mentioned by the tourists.\n\nInput: The lawyers encouraged the professors.\nOutput: The professors were encouraged by the lawyers.\n\nInput: The managers advised the professors.\nOutput: The professors were advised by the managers.\n\nInput: The actor thanked the senators.\nOutput: The senators were thanked by the actor.\n\nInput: The managers introduced the tourist.\nOutput: The tourist was introduced by the managers."
}
},
"10": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The lawyer helped the banker.\nOutput: The banker was helped by the lawyer.\n\nInput: The artists recognized the senator.\nOutput: The senator was recognized by the artists.\n\nInput: The lawyers recommended the athlete.\nOutput: The athlete was recommended by the lawyers.\n\nInput: The senator believed the banker.\nOutput: The banker was believed by the senator.\n\nInput: The tourist avoided the bankers.\nOutput: The bankers were avoided by the tourist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The lawyer helped the banker.\nOutput: The banker was helped by the lawyer.\n\nInput: The artists recognized the senator.\nOutput: The senator was recognized by the artists.\n\nInput: The lawyers recommended the athlete.\nOutput: The athlete was recommended by the lawyers.\n\nInput: The senator believed the banker.\nOutput: The banker was believed by the senator.\n\nInput: The tourist avoided the bankers.\nOutput: The bankers were avoided by the tourist."
}
},
"11": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The professor admired the lawyers.\nOutput: The lawyers were admired by the professor.\n\nInput: The scientist introduced the authors.\nOutput: The authors were introduced by the scientist.\n\nInput: The actors stopped the manager.\nOutput: The manager was stopped by the actors.\n\nInput: The athlete helped the artist.\nOutput: The artist was helped by the athlete.\n\nInput: The artist thanked the lawyers.\nOutput: The lawyers were thanked by the artist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The professor admired the lawyers.\nOutput: The lawyers were admired by the professor.\n\nInput: The scientist introduced the authors.\nOutput: The authors were introduced by the scientist.\n\nInput: The actors stopped the manager.\nOutput: The manager was stopped by the actors.\n\nInput: The athlete helped the artist.\nOutput: The artist was helped by the athlete.\n\nInput: The artist thanked the lawyers.\nOutput: The lawyers were thanked by the artist."
}
},
"12": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The professors advised the athlete.\nOutput: The athlete was advised by the professors.\n\nInput: The doctor believed the artists.\nOutput: The artists were believed by the doctor.\n\nInput: The scientist believed the author.\nOutput: The author was believed by the scientist.\n\nInput: The actor recommended the professor.\nOutput: The professor was recommended by the actor.\n\nInput: The secretary supported the actor.\nOutput: The actor was supported by the secretary.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The professors advised the athlete.\nOutput: The athlete was advised by the professors.\n\nInput: The doctor believed the artists.\nOutput: The artists were believed by the doctor.\n\nInput: The scientist believed the author.\nOutput: The author was believed by the scientist.\n\nInput: The actor recommended the professor.\nOutput: The professor was recommended by the actor.\n\nInput: The secretary supported the actor.\nOutput: The actor was supported by the secretary."
}
},
"13": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The actor helped the senators.\nOutput: The senators were helped by the actor.\n\nInput: The athletes mentioned the bankers.\nOutput: The bankers were mentioned by the athletes.\n\nInput: The tourists mentioned the manager.\nOutput: The manager was mentioned by the tourists.\n\nInput: The athletes mentioned the judges.\nOutput: The judges were mentioned by the athletes.\n\nInput: The students supported the lawyers.\nOutput: The lawyers were supported by the students.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The actor helped the senators.\nOutput: The senators were helped by the actor.\n\nInput: The athletes mentioned the bankers.\nOutput: The bankers were mentioned by the athletes.\n\nInput: The tourists mentioned the manager.\nOutput: The manager was mentioned by the tourists.\n\nInput: The athletes mentioned the judges.\nOutput: The judges were mentioned by the athletes.\n\nInput: The students supported the lawyers.\nOutput: The lawyers were supported by the students."
}
},
"14": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The professors introduced the senators.\nOutput: The senators were introduced by the professors.\n\nInput: The senators encouraged the lawyers.\nOutput: The lawyers were encouraged by the senators.\n\nInput: The senators avoided the doctor.\nOutput: The doctor was avoided by the senators.\n\nInput: The professor recognized the scientist.\nOutput: The scientist was recognized by the professor.\n\nInput: The lawyer admired the author.\nOutput: The author was admired by the lawyer.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The professors introduced the senators.\nOutput: The senators were introduced by the professors.\n\nInput: The senators encouraged the lawyers.\nOutput: The lawyers were encouraged by the senators.\n\nInput: The senators avoided the doctor.\nOutput: The doctor was avoided by the senators.\n\nInput: The professor recognized the scientist.\nOutput: The scientist was recognized by the professor.\n\nInput: The lawyer admired the author.\nOutput: The author was admired by the lawyer."
}
},
"15": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The banker avoided the doctors.\nOutput: The doctors were avoided by the banker.\n\nInput: The doctors avoided the athletes.\nOutput: The athletes were avoided by the doctors.\n\nInput: The actor avoided the banker.\nOutput: The banker was avoided by the actor.\n\nInput: The professor believed the senator.\nOutput: The senator was believed by the professor.\n\nInput: The authors recognized the scientist.\nOutput: The scientist was recognized by the authors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The banker avoided the doctors.\nOutput: The doctors were avoided by the banker.\n\nInput: The doctors avoided the athletes.\nOutput: The athletes were avoided by the doctors.\n\nInput: The actor avoided the banker.\nOutput: The banker was avoided by the actor.\n\nInput: The professor believed the senator.\nOutput: The senator was believed by the professor.\n\nInput: The authors recognized the scientist.\nOutput: The scientist was recognized by the authors."
}
},
"16": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The judge encouraged the banker.\nOutput: The banker was encouraged by the judge.\n\nInput: The actors avoided the authors.\nOutput: The authors were avoided by the actors.\n\nInput: The author recommended the artists.\nOutput: The artists were recommended by the author.\n\nInput: The managers introduced the student.\nOutput: The student was introduced by the managers.\n\nInput: The doctor recommended the students.\nOutput: The students were recommended by the doctor.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The judge encouraged the banker.\nOutput: The banker was encouraged by the judge.\n\nInput: The actors avoided the authors.\nOutput: The authors were avoided by the actors.\n\nInput: The author recommended the artists.\nOutput: The artists were recommended by the author.\n\nInput: The managers introduced the student.\nOutput: The student was introduced by the managers.\n\nInput: The doctor recommended the students.\nOutput: The students were recommended by the doctor."
}
},
"17": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The manager recognized the president.\nOutput: The president was recognized by the manager.\n\nInput: The lawyers supported the judge.\nOutput: The judge was supported by the lawyers.\n\nInput: The senators stopped the manager.\nOutput: The manager was stopped by the senators.\n\nInput: The student encouraged the tourists.\nOutput: The tourists were encouraged by the student.\n\nInput: The actor contacted the athlete.\nOutput: The athlete was contacted by the actor.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The manager recognized the president.\nOutput: The president was recognized by the manager.\n\nInput: The lawyers supported the judge.\nOutput: The judge was supported by the lawyers.\n\nInput: The senators stopped the manager.\nOutput: The manager was stopped by the senators.\n\nInput: The student encouraged the tourists.\nOutput: The tourists were encouraged by the student.\n\nInput: The actor contacted the athlete.\nOutput: The athlete was contacted by the actor."
}
},
"18": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The senators mentioned the actor.\nOutput: The actor was mentioned by the senators.\n\nInput: The artist recognized the judges.\nOutput: The judges were recognized by the artist.\n\nInput: The presidents encouraged the tourists.\nOutput: The tourists were encouraged by the presidents.\n\nInput: The banker admired the lawyer.\nOutput: The lawyer was admired by the banker.\n\nInput: The managers introduced the senators.\nOutput: The senators were introduced by the managers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The senators mentioned the actor.\nOutput: The actor was mentioned by the senators.\n\nInput: The artist recognized the judges.\nOutput: The judges were recognized by the artist.\n\nInput: The presidents encouraged the tourists.\nOutput: The tourists were encouraged by the presidents.\n\nInput: The banker admired the lawyer.\nOutput: The lawyer was admired by the banker.\n\nInput: The managers introduced the senators.\nOutput: The senators were introduced by the managers."
}
},
"19": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The bankers supported the author.\nOutput: The author was supported by the bankers.\n\nInput: The senators recommended the tourists.\nOutput: The tourists were recommended by the senators.\n\nInput: The student stopped the senator.\nOutput: The senator was stopped by the student.\n\nInput: The managers stopped the actors.\nOutput: The actors were stopped by the managers.\n\nInput: The athletes recognized the doctors.\nOutput: The doctors were recognized by the athletes.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The bankers supported the author.\nOutput: The author was supported by the bankers.\n\nInput: The senators recommended the tourists.\nOutput: The tourists were recommended by the senators.\n\nInput: The student stopped the senator.\nOutput: The senator was stopped by the student.\n\nInput: The managers stopped the actors.\nOutput: The actors were stopped by the managers.\n\nInput: The athletes recognized the doctors.\nOutput: The doctors were recognized by the athletes."
}
},
"20": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The artists recognized the manager.\nOutput: The manager was recognized by the artists.\n\nInput: The senator contacted the actor.\nOutput: The actor was contacted by the senator.\n\nInput: The authors advised the president.\nOutput: The president was advised by the authors.\n\nInput: The professors contacted the banker.\nOutput: The banker was contacted by the professors.\n\nInput: The president advised the judges.\nOutput: The judges were advised by the president.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The artists recognized the manager.\nOutput: The manager was recognized by the artists.\n\nInput: The senator contacted the actor.\nOutput: The actor was contacted by the senator.\n\nInput: The authors advised the president.\nOutput: The president was advised by the authors.\n\nInput: The professors contacted the banker.\nOutput: The banker was contacted by the professors.\n\nInput: The president advised the judges.\nOutput: The judges were advised by the president."
}
},
"21": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The artist recognized the secretaries.\nOutput: The secretaries were recognized by the artist.\n\nInput: The author recommended the artists.\nOutput: The artists were recommended by the author.\n\nInput: The student admired the banker.\nOutput: The banker was admired by the student.\n\nInput: The judges admired the actors.\nOutput: The actors were admired by the judges.\n\nInput: The actors thanked the senators.\nOutput: The senators were thanked by the actors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The artist recognized the secretaries.\nOutput: The secretaries were recognized by the artist.\n\nInput: The author recommended the artists.\nOutput: The artists were recommended by the author.\n\nInput: The student admired the banker.\nOutput: The banker was admired by the student.\n\nInput: The judges admired the actors.\nOutput: The actors were admired by the judges.\n\nInput: The actors thanked the senators.\nOutput: The senators were thanked by the actors."
}
},
"22": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The president advised the judges.\nOutput: The judges were advised by the president.\n\nInput: The presidents thanked the bankers.\nOutput: The bankers were thanked by the presidents.\n\nInput: The lawyers believed the scientists.\nOutput: The scientists were believed by the lawyers.\n\nInput: The secretary encouraged the manager.\nOutput: The manager was encouraged by the secretary.\n\nInput: The judges admired the actors.\nOutput: The actors were admired by the judges.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The president advised the judges.\nOutput: The judges were advised by the president.\n\nInput: The presidents thanked the bankers.\nOutput: The bankers were thanked by the presidents.\n\nInput: The lawyers believed the scientists.\nOutput: The scientists were believed by the lawyers.\n\nInput: The secretary encouraged the manager.\nOutput: The manager was encouraged by the secretary.\n\nInput: The judges admired the actors.\nOutput: The actors were admired by the judges."
}
},
"23": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The author recommended the manager.\nOutput: The manager was recommended by the author.\n\nInput: The professor helped the actors.\nOutput: The actors were helped by the professor.\n\nInput: The secretaries believed the tourists.\nOutput: The tourists were believed by the secretaries.\n\nInput: The athletes mentioned the judges.\nOutput: The judges were mentioned by the athletes.\n\nInput: The secretaries recognized the senator.\nOutput: The senator was recognized by the secretaries.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The author recommended the manager.\nOutput: The manager was recommended by the author.\n\nInput: The professor helped the actors.\nOutput: The actors were helped by the professor.\n\nInput: The secretaries believed the tourists.\nOutput: The tourists were believed by the secretaries.\n\nInput: The athletes mentioned the judges.\nOutput: The judges were mentioned by the athletes.\n\nInput: The secretaries recognized the senator.\nOutput: The senator was recognized by the secretaries."
}
},
"24": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The artists recognized the banker.\nOutput: The banker was recognized by the artists.\n\nInput: The scientist helped the students.\nOutput: The students were helped by the scientist.\n\nInput: The professor recognized the scientist.\nOutput: The scientist was recognized by the professor.\n\nInput: The scientist recognized the author.\nOutput: The author was recognized by the scientist.\n\nInput: The tourists mentioned the manager.\nOutput: The manager was mentioned by the tourists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The artists recognized the banker.\nOutput: The banker was recognized by the artists.\n\nInput: The scientist helped the students.\nOutput: The students were helped by the scientist.\n\nInput: The professor recognized the scientist.\nOutput: The scientist was recognized by the professor.\n\nInput: The scientist recognized the author.\nOutput: The author was recognized by the scientist.\n\nInput: The tourists mentioned the manager.\nOutput: The manager was mentioned by the tourists."
}
},
"25": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The students helped the artists.\nOutput: The artists were helped by the students.\n\nInput: The judges helped the artists.\nOutput: The artists were helped by the judges.\n\nInput: The lawyers contacted the scientists.\nOutput: The scientists were contacted by the lawyers.\n\nInput: The artists thanked the athlete.\nOutput: The athlete was thanked by the artists.\n\nInput: The tourists contacted the bankers.\nOutput: The bankers were contacted by the tourists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The students helped the artists.\nOutput: The artists were helped by the students.\n\nInput: The judges helped the artists.\nOutput: The artists were helped by the judges.\n\nInput: The lawyers contacted the scientists.\nOutput: The scientists were contacted by the lawyers.\n\nInput: The artists thanked the athlete.\nOutput: The athlete was thanked by the artists.\n\nInput: The tourists contacted the bankers.\nOutput: The bankers were contacted by the tourists."
}
},
"26": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The author stopped the professor.\nOutput: The professor was stopped by the author.\n\nInput: The tourists advised the actor.\nOutput: The actor was advised by the tourists.\n\nInput: The lawyers encouraged the professors.\nOutput: The professors were encouraged by the lawyers.\n\nInput: The actors mentioned the student.\nOutput: The student was mentioned by the actors.\n\nInput: The president encouraged the professor.\nOutput: The professor was encouraged by the president.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The author stopped the professor.\nOutput: The professor was stopped by the author.\n\nInput: The tourists advised the actor.\nOutput: The actor was advised by the tourists.\n\nInput: The lawyers encouraged the professors.\nOutput: The professors were encouraged by the lawyers.\n\nInput: The actors mentioned the student.\nOutput: The student was mentioned by the actors.\n\nInput: The president encouraged the professor.\nOutput: The professor was encouraged by the president."
}
},
"27": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The professor avoided the doctors.\nOutput: The doctors were avoided by the professor.\n\nInput: The authors encouraged the athlete.\nOutput: The athlete was encouraged by the authors.\n\nInput: The senators recommended the tourists.\nOutput: The tourists were recommended by the senators.\n\nInput: The lawyer avoided the managers.\nOutput: The managers were avoided by the lawyer.\n\nInput: The tourists admired the presidents.\nOutput: The presidents were admired by the tourists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The professor avoided the doctors.\nOutput: The doctors were avoided by the professor.\n\nInput: The authors encouraged the athlete.\nOutput: The athlete was encouraged by the authors.\n\nInput: The senators recommended the tourists.\nOutput: The tourists were recommended by the senators.\n\nInput: The lawyer avoided the managers.\nOutput: The managers were avoided by the lawyer.\n\nInput: The tourists admired the presidents.\nOutput: The presidents were admired by the tourists."
}
},
"28": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The doctor avoided the student.\nOutput: The student was avoided by the doctor.\n\nInput: The presidents recommended the student.\nOutput: The student was recommended by the presidents.\n\nInput: The secretaries recommended the actors.\nOutput: The actors were recommended by the secretaries.\n\nInput: The president introduced the judge.\nOutput: The judge was introduced by the president.\n\nInput: The secretary mentioned the manager.\nOutput: The manager was mentioned by the secretary.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The doctor avoided the student.\nOutput: The student was avoided by the doctor.\n\nInput: The presidents recommended the student.\nOutput: The student was recommended by the presidents.\n\nInput: The secretaries recommended the actors.\nOutput: The actors were recommended by the secretaries.\n\nInput: The president introduced the judge.\nOutput: The judge was introduced by the president.\n\nInput: The secretary mentioned the manager.\nOutput: The manager was mentioned by the secretary."
}
},
"29": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The bankers thanked the professors.\nOutput: The professors were thanked by the bankers.\n\nInput: The judge recognized the doctors.\nOutput: The doctors were recognized by the judge.\n\nInput: The secretaries stopped the presidents.\nOutput: The presidents were stopped by the secretaries.\n\nInput: The senators thanked the professor.\nOutput: The professor was thanked by the senators.\n\nInput: The lawyer thanked the presidents.\nOutput: The presidents were thanked by the lawyer.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The bankers thanked the professors.\nOutput: The professors were thanked by the bankers.\n\nInput: The judge recognized the doctors.\nOutput: The doctors were recognized by the judge.\n\nInput: The secretaries stopped the presidents.\nOutput: The presidents were stopped by the secretaries.\n\nInput: The senators thanked the professor.\nOutput: The professor was thanked by the senators.\n\nInput: The lawyer thanked the presidents.\nOutput: The presidents were thanked by the lawyer."
}
},
"30": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The scientists recommended the tourist.\nOutput: The tourist was recommended by the scientists.\n\nInput: The artist avoided the lawyers.\nOutput: The lawyers were avoided by the artist.\n\nInput: The judges avoided the scientists.\nOutput: The scientists were avoided by the judges.\n\nInput: The presidents recommended the artist.\nOutput: The artist was recommended by the presidents.\n\nInput: The students helped the senator.\nOutput: The senator was helped by the students.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The scientists recommended the tourist.\nOutput: The tourist was recommended by the scientists.\n\nInput: The artist avoided the lawyers.\nOutput: The lawyers were avoided by the artist.\n\nInput: The judges avoided the scientists.\nOutput: The scientists were avoided by the judges.\n\nInput: The presidents recommended the artist.\nOutput: The artist was recommended by the presidents.\n\nInput: The students helped the senator.\nOutput: The senator was helped by the students."
}
},
"31": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The actors avoided the authors.\nOutput: The authors were avoided by the actors.\n\nInput: The president thanked the artists.\nOutput: The artists were thanked by the president.\n\nInput: The authors introduced the judge.\nOutput: The judge was introduced by the authors.\n\nInput: The scientists admired the secretaries.\nOutput: The secretaries were admired by the scientists.\n\nInput: The scientists encouraged the author.\nOutput: The author was encouraged by the scientists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The actors avoided the authors.\nOutput: The authors were avoided by the actors.\n\nInput: The president thanked the artists.\nOutput: The artists were thanked by the president.\n\nInput: The authors introduced the judge.\nOutput: The judge was introduced by the authors.\n\nInput: The scientists admired the secretaries.\nOutput: The secretaries were admired by the scientists.\n\nInput: The scientists encouraged the author.\nOutput: The author was encouraged by the scientists."
}
},
"32": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The bankers advised the presidents.\nOutput: The presidents were advised by the bankers.\n\nInput: The actor supported the president.\nOutput: The president was supported by the actor.\n\nInput: The doctors admired the judges.\nOutput: The judges were admired by the doctors.\n\nInput: The senator supported the actors.\nOutput: The actors were supported by the senator.\n\nInput: The tourists recommended the lawyer.\nOutput: The lawyer was recommended by the tourists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The bankers advised the presidents.\nOutput: The presidents were advised by the bankers.\n\nInput: The actor supported the president.\nOutput: The president was supported by the actor.\n\nInput: The doctors admired the judges.\nOutput: The judges were admired by the doctors.\n\nInput: The senator supported the actors.\nOutput: The actors were supported by the senator.\n\nInput: The tourists recommended the lawyer.\nOutput: The lawyer was recommended by the tourists."
}
},
"33": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The student stopped the presidents.\nOutput: The presidents were stopped by the student.\n\nInput: The secretary avoided the artists.\nOutput: The artists were avoided by the secretary.\n\nInput: The artists encouraged the judge.\nOutput: The judge was encouraged by the artists.\n\nInput: The students helped the managers.\nOutput: The managers were helped by the students.\n\nInput: The judge thanked the professor.\nOutput: The professor was thanked by the judge.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The student stopped the presidents.\nOutput: The presidents were stopped by the student.\n\nInput: The secretary avoided the artists.\nOutput: The artists were avoided by the secretary.\n\nInput: The artists encouraged the judge.\nOutput: The judge was encouraged by the artists.\n\nInput: The students helped the managers.\nOutput: The managers were helped by the students.\n\nInput: The judge thanked the professor.\nOutput: The professor was thanked by the judge."
}
},
"34": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The lawyers avoided the scientists.\nOutput: The scientists were avoided by the lawyers.\n\nInput: The secretaries introduced the doctors.\nOutput: The doctors were introduced by the secretaries.\n\nInput: The tourists advised the professors.\nOutput: The professors were advised by the tourists.\n\nInput: The artists recognized the scientist.\nOutput: The scientist was recognized by the artists.\n\nInput: The students supported the doctors.\nOutput: The doctors were supported by the students.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The lawyers avoided the scientists.\nOutput: The scientists were avoided by the lawyers.\n\nInput: The secretaries introduced the doctors.\nOutput: The doctors were introduced by the secretaries.\n\nInput: The tourists advised the professors.\nOutput: The professors were advised by the tourists.\n\nInput: The artists recognized the scientist.\nOutput: The scientist was recognized by the artists.\n\nInput: The students supported the doctors.\nOutput: The doctors were supported by the students."
}
},
"35": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The senator advised the doctors.\nOutput: The doctors were advised by the senator.\n\nInput: The students supported the professor.\nOutput: The professor was supported by the students.\n\nInput: The authors contacted the secretaries.\nOutput: The secretaries were contacted by the authors.\n\nInput: The authors contacted the judges.\nOutput: The judges were contacted by the authors.\n\nInput: The senators introduced the student.\nOutput: The student was introduced by the senators.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The senator advised the doctors.\nOutput: The doctors were advised by the senator.\n\nInput: The students supported the professor.\nOutput: The professor was supported by the students.\n\nInput: The authors contacted the secretaries.\nOutput: The secretaries were contacted by the authors.\n\nInput: The authors contacted the judges.\nOutput: The judges were contacted by the authors.\n\nInput: The senators introduced the student.\nOutput: The student was introduced by the senators."
}
},
"36": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The presidents encouraged the secretary.\nOutput: The secretary was encouraged by the presidents.\n\nInput: The banker mentioned the judge.\nOutput: The judge was mentioned by the banker.\n\nInput: The athletes admired the actor.\nOutput: The actor was admired by the athletes.\n\nInput: The actors stopped the athletes.\nOutput: The athletes were stopped by the actors.\n\nInput: The athletes encouraged the tourist.\nOutput: The tourist was encouraged by the athletes.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The presidents encouraged the secretary.\nOutput: The secretary was encouraged by the presidents.\n\nInput: The banker mentioned the judge.\nOutput: The judge was mentioned by the banker.\n\nInput: The athletes admired the actor.\nOutput: The actor was admired by the athletes.\n\nInput: The actors stopped the athletes.\nOutput: The athletes were stopped by the actors.\n\nInput: The athletes encouraged the tourist.\nOutput: The tourist was encouraged by the athletes."
}
},
"37": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The scientists contacted the tourists.\nOutput: The tourists were contacted by the scientists.\n\nInput: The actors thanked the doctor.\nOutput: The doctor was thanked by the actors.\n\nInput: The senator believed the judges.\nOutput: The judges were believed by the senator.\n\nInput: The scientist recognized the president.\nOutput: The president was recognized by the scientist.\n\nInput: The students stopped the doctor.\nOutput: The doctor was stopped by the students.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The scientists contacted the tourists.\nOutput: The tourists were contacted by the scientists.\n\nInput: The actors thanked the doctor.\nOutput: The doctor was thanked by the actors.\n\nInput: The senator believed the judges.\nOutput: The judges were believed by the senator.\n\nInput: The scientist recognized the president.\nOutput: The president was recognized by the scientist.\n\nInput: The students stopped the doctor.\nOutput: The doctor was stopped by the students."
}
},
"38": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The student admired the athlete.\nOutput: The athlete was admired by the student.\n\nInput: The judge recommended the scientists.\nOutput: The scientists were recommended by the judge.\n\nInput: The lawyers believed the student.\nOutput: The student was believed by the lawyers.\n\nInput: The lawyer thanked the athlete.\nOutput: The athlete was thanked by the lawyer.\n\nInput: The judge mentioned the tourist.\nOutput: The tourist was mentioned by the judge.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The student admired the athlete.\nOutput: The athlete was admired by the student.\n\nInput: The judge recommended the scientists.\nOutput: The scientists were recommended by the judge.\n\nInput: The lawyers believed the student.\nOutput: The student was believed by the lawyers.\n\nInput: The lawyer thanked the athlete.\nOutput: The athlete was thanked by the lawyer.\n\nInput: The judge mentioned the tourist.\nOutput: The tourist was mentioned by the judge."
}
},
"39": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The secretaries helped the professors.\nOutput: The professors were helped by the secretaries.\n\nInput: The athletes recognized the doctors.\nOutput: The doctors were recognized by the athletes.\n\nInput: The actor believed the secretaries.\nOutput: The secretaries were believed by the actor.\n\nInput: The athletes avoided the senator.\nOutput: The senator was avoided by the athletes.\n\nInput: The actors stopped the scientist.\nOutput: The scientist was stopped by the actors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The secretaries helped the professors.\nOutput: The professors were helped by the secretaries.\n\nInput: The athletes recognized the doctors.\nOutput: The doctors were recognized by the athletes.\n\nInput: The actor believed the secretaries.\nOutput: The secretaries were believed by the actor.\n\nInput: The athletes avoided the senator.\nOutput: The senator was avoided by the athletes.\n\nInput: The actors stopped the scientist.\nOutput: The scientist was stopped by the actors."
}
},
"40": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The tourist advised the doctors.\nOutput: The doctors were advised by the tourist.\n\nInput: The president advised the artists.\nOutput: The artists were advised by the president.\n\nInput: The artist advised the doctor.\nOutput: The doctor was advised by the artist.\n\nInput: The doctors thanked the author.\nOutput: The author was thanked by the doctors.\n\nInput: The secretary recommended the students.\nOutput: The students were recommended by the secretary.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The tourist advised the doctors.\nOutput: The doctors were advised by the tourist.\n\nInput: The president advised the artists.\nOutput: The artists were advised by the president.\n\nInput: The artist advised the doctor.\nOutput: The doctor was advised by the artist.\n\nInput: The doctors thanked the author.\nOutput: The author was thanked by the doctors.\n\nInput: The secretary recommended the students.\nOutput: The students were recommended by the secretary."
}
},
"41": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The lawyer avoided the president.\nOutput: The president was avoided by the lawyer.\n\nInput: The managers stopped the president.\nOutput: The president was stopped by the managers.\n\nInput: The senator mentioned the professor.\nOutput: The professor was mentioned by the senator.\n\nInput: The students helped the author.\nOutput: The author was helped by the students.\n\nInput: The professor believed the senator.\nOutput: The senator was believed by the professor.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The lawyer avoided the president.\nOutput: The president was avoided by the lawyer.\n\nInput: The managers stopped the president.\nOutput: The president was stopped by the managers.\n\nInput: The senator mentioned the professor.\nOutput: The professor was mentioned by the senator.\n\nInput: The students helped the author.\nOutput: The author was helped by the students.\n\nInput: The professor believed the senator.\nOutput: The senator was believed by the professor."
}
},
"42": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The actors thanked the doctor.\nOutput: The doctor was thanked by the actors.\n\nInput: The artist recognized the judges.\nOutput: The judges were recognized by the artist.\n\nInput: The doctor advised the actors.\nOutput: The actors were advised by the doctor.\n\nInput: The doctors believed the tourist.\nOutput: The tourist was believed by the doctors.\n\nInput: The students encouraged the athlete.\nOutput: The athlete was encouraged by the students.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The actors thanked the doctor.\nOutput: The doctor was thanked by the actors.\n\nInput: The artist recognized the judges.\nOutput: The judges were recognized by the artist.\n\nInput: The doctor advised the actors.\nOutput: The actors were advised by the doctor.\n\nInput: The doctors believed the tourist.\nOutput: The tourist was believed by the doctors.\n\nInput: The students encouraged the athlete.\nOutput: The athlete was encouraged by the students."
}
},
"43": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The scientist avoided the tourist.\nOutput: The tourist was avoided by the scientist.\n\nInput: The professors helped the president.\nOutput: The president was helped by the professors.\n\nInput: The manager stopped the lawyers.\nOutput: The lawyers were stopped by the manager.\n\nInput: The bankers contacted the doctor.\nOutput: The doctor was contacted by the bankers.\n\nInput: The artists admired the professor.\nOutput: The professor was admired by the artists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The scientist avoided the tourist.\nOutput: The tourist was avoided by the scientist.\n\nInput: The professors helped the president.\nOutput: The president was helped by the professors.\n\nInput: The manager stopped the lawyers.\nOutput: The lawyers were stopped by the manager.\n\nInput: The bankers contacted the doctor.\nOutput: The doctor was contacted by the bankers.\n\nInput: The artists admired the professor.\nOutput: The professor was admired by the artists."
}
},
"44": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The doctor encouraged the lawyers.\nOutput: The lawyers were encouraged by the doctor.\n\nInput: The actor admired the professor.\nOutput: The professor was admired by the actor.\n\nInput: The secretaries stopped the bankers.\nOutput: The bankers were stopped by the secretaries.\n\nInput: The students contacted the doctors.\nOutput: The doctors were contacted by the students.\n\nInput: The tourist recommended the secretary.\nOutput: The secretary was recommended by the tourist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The doctor encouraged the lawyers.\nOutput: The lawyers were encouraged by the doctor.\n\nInput: The actor admired the professor.\nOutput: The professor was admired by the actor.\n\nInput: The secretaries stopped the bankers.\nOutput: The bankers were stopped by the secretaries.\n\nInput: The students contacted the doctors.\nOutput: The doctors were contacted by the students.\n\nInput: The tourist recommended the secretary.\nOutput: The secretary was recommended by the tourist."
}
},
"45": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The managers recognized the author.\nOutput: The author was recognized by the managers.\n\nInput: The artists avoided the secretary.\nOutput: The secretary was avoided by the artists.\n\nInput: The doctors introduced the actor.\nOutput: The actor was introduced by the doctors.\n\nInput: The senators advised the actor.\nOutput: The actor was advised by the senators.\n\nInput: The judge introduced the professor.\nOutput: The professor was introduced by the judge.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The managers recognized the author.\nOutput: The author was recognized by the managers.\n\nInput: The artists avoided the secretary.\nOutput: The secretary was avoided by the artists.\n\nInput: The doctors introduced the actor.\nOutput: The actor was introduced by the doctors.\n\nInput: The senators advised the actor.\nOutput: The actor was advised by the senators.\n\nInput: The judge introduced the professor.\nOutput: The professor was introduced by the judge."
}
},
"46": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The artists encouraged the judge.\nOutput: The judge was encouraged by the artists.\n\nInput: The artists believed the athlete.\nOutput: The athlete was believed by the artists.\n\nInput: The scientist mentioned the judge.\nOutput: The judge was mentioned by the scientist.\n\nInput: The lawyer avoided the managers.\nOutput: The managers were avoided by the lawyer.\n\nInput: The artists avoided the secretary.\nOutput: The secretary was avoided by the artists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The artists encouraged the judge.\nOutput: The judge was encouraged by the artists.\n\nInput: The artists believed the athlete.\nOutput: The athlete was believed by the artists.\n\nInput: The scientist mentioned the judge.\nOutput: The judge was mentioned by the scientist.\n\nInput: The lawyer avoided the managers.\nOutput: The managers were avoided by the lawyer.\n\nInput: The artists avoided the secretary.\nOutput: The secretary was avoided by the artists."
}
},
"47": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The actors mentioned the student.\nOutput: The student was mentioned by the actors.\n\nInput: The president thanked the artists.\nOutput: The artists were thanked by the president.\n\nInput: The doctors stopped the scientists.\nOutput: The scientists were stopped by the doctors.\n\nInput: The secretaries admired the artist.\nOutput: The artist was admired by the secretaries.\n\nInput: The athletes encouraged the doctors.\nOutput: The doctors were encouraged by the athletes.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The actors mentioned the student.\nOutput: The student was mentioned by the actors.\n\nInput: The president thanked the artists.\nOutput: The artists were thanked by the president.\n\nInput: The doctors stopped the scientists.\nOutput: The scientists were stopped by the doctors.\n\nInput: The secretaries admired the artist.\nOutput: The artist was admired by the secretaries.\n\nInput: The athletes encouraged the doctors.\nOutput: The doctors were encouraged by the athletes."
}
},
"48": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The artists mentioned the actors.\nOutput: The actors were mentioned by the artists.\n\nInput: The banker admired the managers.\nOutput: The managers were admired by the banker.\n\nInput: The actors helped the judge.\nOutput: The judge was helped by the actors.\n\nInput: The students supported the lawyers.\nOutput: The lawyers were supported by the students.\n\nInput: The judges supported the lawyer.\nOutput: The lawyer was supported by the judges.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The artists mentioned the actors.\nOutput: The actors were mentioned by the artists.\n\nInput: The banker admired the managers.\nOutput: The managers were admired by the banker.\n\nInput: The actors helped the judge.\nOutput: The judge was helped by the actors.\n\nInput: The students supported the lawyers.\nOutput: The lawyers were supported by the students.\n\nInput: The judges supported the lawyer.\nOutput: The lawyer was supported by the judges."
}
},
"49": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The presidents recommended the author.\nOutput: The author was recommended by the presidents.\n\nInput: The scientists helped the senator.\nOutput: The senator was helped by the scientists.\n\nInput: The secretary avoided the doctors.\nOutput: The doctors were avoided by the secretary.\n\nInput: The lawyers encouraged the professors.\nOutput: The professors were encouraged by the lawyers.\n\nInput: The tourist helped the artists.\nOutput: The artists were helped by the tourist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The presidents recommended the author.\nOutput: The author was recommended by the presidents.\n\nInput: The scientists helped the senator.\nOutput: The senator was helped by the scientists.\n\nInput: The secretary avoided the doctors.\nOutput: The doctors were avoided by the secretary.\n\nInput: The lawyers encouraged the professors.\nOutput: The professors were encouraged by the lawyers.\n\nInput: The tourist helped the artists.\nOutput: The artists were helped by the tourist."
}
},
"50": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The presidents contacted the senator.\nOutput: The senator was contacted by the presidents.\n\nInput: The actors recognized the scientist.\nOutput: The scientist was recognized by the actors.\n\nInput: The judge believed the senators.\nOutput: The senators were believed by the judge.\n\nInput: The lawyer recognized the senators.\nOutput: The senators were recognized by the lawyer.\n\nInput: The president encouraged the bankers.\nOutput: The bankers were encouraged by the president.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The presidents contacted the senator.\nOutput: The senator was contacted by the presidents.\n\nInput: The actors recognized the scientist.\nOutput: The scientist was recognized by the actors.\n\nInput: The judge believed the senators.\nOutput: The senators were believed by the judge.\n\nInput: The lawyer recognized the senators.\nOutput: The senators were recognized by the lawyer.\n\nInput: The president encouraged the bankers.\nOutput: The bankers were encouraged by the president."
}
},
"51": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The president supported the actor.\nOutput: The actor was supported by the president.\n\nInput: The tourist believed the doctor.\nOutput: The doctor was believed by the tourist.\n\nInput: The authors recognized the banker.\nOutput: The banker was recognized by the authors.\n\nInput: The scientist recommended the senators.\nOutput: The senators were recommended by the scientist.\n\nInput: The doctors stopped the scientists.\nOutput: The scientists were stopped by the doctors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The president supported the actor.\nOutput: The actor was supported by the president.\n\nInput: The tourist believed the doctor.\nOutput: The doctor was believed by the tourist.\n\nInput: The authors recognized the banker.\nOutput: The banker was recognized by the authors.\n\nInput: The scientist recommended the senators.\nOutput: The senators were recommended by the scientist.\n\nInput: The doctors stopped the scientists.\nOutput: The scientists were stopped by the doctors."
}
},
"52": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The senator helped the judges.\nOutput: The judges were helped by the senator.\n\nInput: The doctors thanked the lawyers.\nOutput: The lawyers were thanked by the doctors.\n\nInput: The author advised the judge.\nOutput: The judge was advised by the author.\n\nInput: The tourists advised the professors.\nOutput: The professors were advised by the tourists.\n\nInput: The managers contacted the scientist.\nOutput: The scientist was contacted by the managers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The senator helped the judges.\nOutput: The judges were helped by the senator.\n\nInput: The doctors thanked the lawyers.\nOutput: The lawyers were thanked by the doctors.\n\nInput: The author advised the judge.\nOutput: The judge was advised by the author.\n\nInput: The tourists advised the professors.\nOutput: The professors were advised by the tourists.\n\nInput: The managers contacted the scientist.\nOutput: The scientist was contacted by the managers."
}
},
"53": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The president introduced the lawyer.\nOutput: The lawyer was introduced by the president.\n\nInput: The doctors thanked the lawyers.\nOutput: The lawyers were thanked by the doctors.\n\nInput: The students advised the professor.\nOutput: The professor was advised by the students.\n\nInput: The judge introduced the student.\nOutput: The student was introduced by the judge.\n\nInput: The tourist avoided the judges.\nOutput: The judges were avoided by the tourist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The president introduced the lawyer.\nOutput: The lawyer was introduced by the president.\n\nInput: The doctors thanked the lawyers.\nOutput: The lawyers were thanked by the doctors.\n\nInput: The students advised the professor.\nOutput: The professor was advised by the students.\n\nInput: The judge introduced the student.\nOutput: The student was introduced by the judge.\n\nInput: The tourist avoided the judges.\nOutput: The judges were avoided by the tourist."
}
},
"54": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The secretaries advised the artist.\nOutput: The artist was advised by the secretaries.\n\nInput: The authors introduced the student.\nOutput: The student was introduced by the authors.\n\nInput: The tourist helped the artists.\nOutput: The artists were helped by the tourist.\n\nInput: The judge thanked the author.\nOutput: The author was thanked by the judge.\n\nInput: The student advised the scientists.\nOutput: The scientists were advised by the student.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The secretaries advised the artist.\nOutput: The artist was advised by the secretaries.\n\nInput: The authors introduced the student.\nOutput: The student was introduced by the authors.\n\nInput: The tourist helped the artists.\nOutput: The artists were helped by the tourist.\n\nInput: The judge thanked the author.\nOutput: The author was thanked by the judge.\n\nInput: The student advised the scientists.\nOutput: The scientists were advised by the student."
}
},
"55": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The professors helped the doctor.\nOutput: The doctor was helped by the professors.\n\nInput: The actors mentioned the student.\nOutput: The student was mentioned by the actors.\n\nInput: The judges thanked the doctors.\nOutput: The doctors were thanked by the judges.\n\nInput: The professor recognized the scientist.\nOutput: The scientist was recognized by the professor.\n\nInput: The author avoided the doctors.\nOutput: The doctors were avoided by the author.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The professors helped the doctor.\nOutput: The doctor was helped by the professors.\n\nInput: The actors mentioned the student.\nOutput: The student was mentioned by the actors.\n\nInput: The judges thanked the doctors.\nOutput: The doctors were thanked by the judges.\n\nInput: The professor recognized the scientist.\nOutput: The scientist was recognized by the professor.\n\nInput: The author avoided the doctors.\nOutput: The doctors were avoided by the author."
}
},
"56": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The tourist supported the athlete.\nOutput: The athlete was supported by the tourist.\n\nInput: The banker recommended the tourists.\nOutput: The tourists were recommended by the banker.\n\nInput: The managers supported the bankers.\nOutput: The bankers were supported by the managers.\n\nInput: The artists thanked the scientist.\nOutput: The scientist was thanked by the artists.\n\nInput: The secretaries stopped the presidents.\nOutput: The presidents were stopped by the secretaries.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The tourist supported the athlete.\nOutput: The athlete was supported by the tourist.\n\nInput: The banker recommended the tourists.\nOutput: The tourists were recommended by the banker.\n\nInput: The managers supported the bankers.\nOutput: The bankers were supported by the managers.\n\nInput: The artists thanked the scientist.\nOutput: The scientist was thanked by the artists.\n\nInput: The secretaries stopped the presidents.\nOutput: The presidents were stopped by the secretaries."
}
},
"57": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The athlete believed the managers.\nOutput: The managers were believed by the athlete.\n\nInput: The managers mentioned the student.\nOutput: The student was mentioned by the managers.\n\nInput: The senators advised the presidents.\nOutput: The presidents were advised by the senators.\n\nInput: The banker mentioned the secretaries.\nOutput: The secretaries were mentioned by the banker.\n\nInput: The author recommended the artists.\nOutput: The artists were recommended by the author.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The athlete believed the managers.\nOutput: The managers were believed by the athlete.\n\nInput: The managers mentioned the student.\nOutput: The student was mentioned by the managers.\n\nInput: The senators advised the presidents.\nOutput: The presidents were advised by the senators.\n\nInput: The banker mentioned the secretaries.\nOutput: The secretaries were mentioned by the banker.\n\nInput: The author recommended the artists.\nOutput: The artists were recommended by the author."
}
},
"58": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The doctor advised the managers.\nOutput: The managers were advised by the doctor.\n\nInput: The secretary stopped the presidents.\nOutput: The presidents were stopped by the secretary.\n\nInput: The artist advised the author.\nOutput: The author was advised by the artist.\n\nInput: The athletes recognized the judge.\nOutput: The judge was recognized by the athletes.\n\nInput: The professors mentioned the presidents.\nOutput: The presidents were mentioned by the professors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The doctor advised the managers.\nOutput: The managers were advised by the doctor.\n\nInput: The secretary stopped the presidents.\nOutput: The presidents were stopped by the secretary.\n\nInput: The artist advised the author.\nOutput: The author was advised by the artist.\n\nInput: The athletes recognized the judge.\nOutput: The judge was recognized by the athletes.\n\nInput: The professors mentioned the presidents.\nOutput: The presidents were mentioned by the professors."
}
},
"59": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The secretaries supported the scientists.\nOutput: The scientists were supported by the secretaries.\n\nInput: The managers stopped the doctor.\nOutput: The doctor was stopped by the managers.\n\nInput: The actors mentioned the student.\nOutput: The student was mentioned by the actors.\n\nInput: The student contacted the secretaries.\nOutput: The secretaries were contacted by the student.\n\nInput: The lawyers believed the scientists.\nOutput: The scientists were believed by the lawyers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The secretaries supported the scientists.\nOutput: The scientists were supported by the secretaries.\n\nInput: The managers stopped the doctor.\nOutput: The doctor was stopped by the managers.\n\nInput: The actors mentioned the student.\nOutput: The student was mentioned by the actors.\n\nInput: The student contacted the secretaries.\nOutput: The secretaries were contacted by the student.\n\nInput: The lawyers believed the scientists.\nOutput: The scientists were believed by the lawyers."
}
},
"60": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The scientist recognized the student.\nOutput: The student was recognized by the scientist.\n\nInput: The professors helped the managers.\nOutput: The managers were helped by the professors.\n\nInput: The tourist mentioned the secretary.\nOutput: The secretary was mentioned by the tourist.\n\nInput: The banker advised the doctor.\nOutput: The doctor was advised by the banker.\n\nInput: The managers introduced the senators.\nOutput: The senators were introduced by the managers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The scientist recognized the student.\nOutput: The student was recognized by the scientist.\n\nInput: The professors helped the managers.\nOutput: The managers were helped by the professors.\n\nInput: The tourist mentioned the secretary.\nOutput: The secretary was mentioned by the tourist.\n\nInput: The banker advised the doctor.\nOutput: The doctor was advised by the banker.\n\nInput: The managers introduced the senators.\nOutput: The senators were introduced by the managers."
}
},
"61": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The tourist admired the managers.\nOutput: The managers were admired by the tourist.\n\nInput: The secretary avoided the author.\nOutput: The author was avoided by the secretary.\n\nInput: The judge recommended the secretary.\nOutput: The secretary was recommended by the judge.\n\nInput: The manager advised the scientist.\nOutput: The scientist was advised by the manager.\n\nInput: The actors recommended the presidents.\nOutput: The presidents were recommended by the actors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The tourist admired the managers.\nOutput: The managers were admired by the tourist.\n\nInput: The secretary avoided the author.\nOutput: The author was avoided by the secretary.\n\nInput: The judge recommended the secretary.\nOutput: The secretary was recommended by the judge.\n\nInput: The manager advised the scientist.\nOutput: The scientist was advised by the manager.\n\nInput: The actors recommended the presidents.\nOutput: The presidents were recommended by the actors."
}
},
"62": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The bankers supported the professor.\nOutput: The professor was supported by the bankers.\n\nInput: The actors stopped the athletes.\nOutput: The athletes were stopped by the actors.\n\nInput: The students helped the managers.\nOutput: The managers were helped by the students.\n\nInput: The students advised the professor.\nOutput: The professor was advised by the students.\n\nInput: The bankers stopped the lawyer.\nOutput: The lawyer was stopped by the bankers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The bankers supported the professor.\nOutput: The professor was supported by the bankers.\n\nInput: The actors stopped the athletes.\nOutput: The athletes were stopped by the actors.\n\nInput: The students helped the managers.\nOutput: The managers were helped by the students.\n\nInput: The students advised the professor.\nOutput: The professor was advised by the students.\n\nInput: The bankers stopped the lawyer.\nOutput: The lawyer was stopped by the bankers."
}
},
"63": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The managers contacted the scientist.\nOutput: The scientist was contacted by the managers.\n\nInput: The secretary recognized the tourists.\nOutput: The tourists were recognized by the secretary.\n\nInput: The scientist recognized the author.\nOutput: The author was recognized by the scientist.\n\nInput: The professor contacted the lawyer.\nOutput: The lawyer was contacted by the professor.\n\nInput: The banker thanked the athlete.\nOutput: The athlete was thanked by the banker.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The managers contacted the scientist.\nOutput: The scientist was contacted by the managers.\n\nInput: The secretary recognized the tourists.\nOutput: The tourists were recognized by the secretary.\n\nInput: The scientist recognized the author.\nOutput: The author was recognized by the scientist.\n\nInput: The professor contacted the lawyer.\nOutput: The lawyer was contacted by the professor.\n\nInput: The banker thanked the athlete.\nOutput: The athlete was thanked by the banker."
}
},
"64": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The president supported the actor.\nOutput: The actor was supported by the president.\n\nInput: The judges thanked the tourists.\nOutput: The tourists were thanked by the judges.\n\nInput: The secretary mentioned the manager.\nOutput: The manager was mentioned by the secretary.\n\nInput: The artists contacted the doctors.\nOutput: The doctors were contacted by the artists.\n\nInput: The judge advised the scientists.\nOutput: The scientists were advised by the judge.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The president supported the actor.\nOutput: The actor was supported by the president.\n\nInput: The judges thanked the tourists.\nOutput: The tourists were thanked by the judges.\n\nInput: The secretary mentioned the manager.\nOutput: The manager was mentioned by the secretary.\n\nInput: The artists contacted the doctors.\nOutput: The doctors were contacted by the artists.\n\nInput: The judge advised the scientists.\nOutput: The scientists were advised by the judge."
}
},
"65": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The presidents advised the lawyer.\nOutput: The lawyer was advised by the presidents.\n\nInput: The manager introduced the professors.\nOutput: The professors were introduced by the manager.\n\nInput: The lawyers mentioned the tourists.\nOutput: The tourists were mentioned by the lawyers.\n\nInput: The students recognized the banker.\nOutput: The banker was recognized by the students.\n\nInput: The lawyer encouraged the judges.\nOutput: The judges were encouraged by the lawyer.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The presidents advised the lawyer.\nOutput: The lawyer was advised by the presidents.\n\nInput: The manager introduced the professors.\nOutput: The professors were introduced by the manager.\n\nInput: The lawyers mentioned the tourists.\nOutput: The tourists were mentioned by the lawyers.\n\nInput: The students recognized the banker.\nOutput: The banker was recognized by the students.\n\nInput: The lawyer encouraged the judges.\nOutput: The judges were encouraged by the lawyer."
}
},
"66": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The scientists thanked the athletes.\nOutput: The athletes were thanked by the scientists.\n\nInput: The tourist encouraged the banker.\nOutput: The banker was encouraged by the tourist.\n\nInput: The professors believed the manager.\nOutput: The manager was believed by the professors.\n\nInput: The senators believed the presidents.\nOutput: The presidents were believed by the senators.\n\nInput: The actor admired the secretaries.\nOutput: The secretaries were admired by the actor.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The scientists thanked the athletes.\nOutput: The athletes were thanked by the scientists.\n\nInput: The tourist encouraged the banker.\nOutput: The banker was encouraged by the tourist.\n\nInput: The professors believed the manager.\nOutput: The manager was believed by the professors.\n\nInput: The senators believed the presidents.\nOutput: The presidents were believed by the senators.\n\nInput: The actor admired the secretaries.\nOutput: The secretaries were admired by the actor."
}
},
"67": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The manager contacted the president.\nOutput: The president was contacted by the manager.\n\nInput: The athlete believed the judge.\nOutput: The judge was believed by the athlete.\n\nInput: The judge avoided the artist.\nOutput: The artist was avoided by the judge.\n\nInput: The president helped the secretaries.\nOutput: The secretaries were helped by the president.\n\nInput: The bankers admired the author.\nOutput: The author was admired by the bankers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The manager contacted the president.\nOutput: The president was contacted by the manager.\n\nInput: The athlete believed the judge.\nOutput: The judge was believed by the athlete.\n\nInput: The judge avoided the artist.\nOutput: The artist was avoided by the judge.\n\nInput: The president helped the secretaries.\nOutput: The secretaries were helped by the president.\n\nInput: The bankers admired the author.\nOutput: The author was admired by the bankers."
}
},
"68": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The secretaries thanked the athletes.\nOutput: The athletes were thanked by the secretaries.\n\nInput: The senator contacted the actor.\nOutput: The actor was contacted by the senator.\n\nInput: The athlete supported the secretaries.\nOutput: The secretaries were supported by the athlete.\n\nInput: The professor helped the actor.\nOutput: The actor was helped by the professor.\n\nInput: The presidents contacted the senator.\nOutput: The senator was contacted by the presidents.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The secretaries thanked the athletes.\nOutput: The athletes were thanked by the secretaries.\n\nInput: The senator contacted the actor.\nOutput: The actor was contacted by the senator.\n\nInput: The athlete supported the secretaries.\nOutput: The secretaries were supported by the athlete.\n\nInput: The professor helped the actor.\nOutput: The actor was helped by the professor.\n\nInput: The presidents contacted the senator.\nOutput: The senator was contacted by the presidents."
}
},
"69": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The artist encouraged the student.\nOutput: The student was encouraged by the artist.\n\nInput: The doctors admired the presidents.\nOutput: The presidents were admired by the doctors.\n\nInput: The senators believed the presidents.\nOutput: The presidents were believed by the senators.\n\nInput: The president mentioned the professor.\nOutput: The professor was mentioned by the president.\n\nInput: The banker encouraged the athlete.\nOutput: The athlete was encouraged by the banker.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The artist encouraged the student.\nOutput: The student was encouraged by the artist.\n\nInput: The doctors admired the presidents.\nOutput: The presidents were admired by the doctors.\n\nInput: The senators believed the presidents.\nOutput: The presidents were believed by the senators.\n\nInput: The president mentioned the professor.\nOutput: The professor was mentioned by the president.\n\nInput: The banker encouraged the athlete.\nOutput: The athlete was encouraged by the banker."
}
},
"70": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The tourist contacted the doctor.\nOutput: The doctor was contacted by the tourist.\n\nInput: The authors helped the doctor.\nOutput: The doctor was helped by the authors.\n\nInput: The actors recognized the scientists.\nOutput: The scientists were recognized by the actors.\n\nInput: The president advised the judges.\nOutput: The judges were advised by the president.\n\nInput: The secretaries avoided the student.\nOutput: The student was avoided by the secretaries.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The tourist contacted the doctor.\nOutput: The doctor was contacted by the tourist.\n\nInput: The authors helped the doctor.\nOutput: The doctor was helped by the authors.\n\nInput: The actors recognized the scientists.\nOutput: The scientists were recognized by the actors.\n\nInput: The president advised the judges.\nOutput: The judges were advised by the president.\n\nInput: The secretaries avoided the student.\nOutput: The student was avoided by the secretaries."
}
},
"71": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The tourists mentioned the artist.\nOutput: The artist was mentioned by the tourists.\n\nInput: The artist mentioned the actors.\nOutput: The actors were mentioned by the artist.\n\nInput: The senator believed the banker.\nOutput: The banker was believed by the senator.\n\nInput: The artist recognized the judges.\nOutput: The judges were recognized by the artist.\n\nInput: The senator recognized the managers.\nOutput: The managers were recognized by the senator.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The tourists mentioned the artist.\nOutput: The artist was mentioned by the tourists.\n\nInput: The artist mentioned the actors.\nOutput: The actors were mentioned by the artist.\n\nInput: The senator believed the banker.\nOutput: The banker was believed by the senator.\n\nInput: The artist recognized the judges.\nOutput: The judges were recognized by the artist.\n\nInput: The senator recognized the managers.\nOutput: The managers were recognized by the senator."
}
},
"72": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The artist advised the doctor.\nOutput: The doctor was advised by the artist.\n\nInput: The judge introduced the student.\nOutput: The student was introduced by the judge.\n\nInput: The secretary contacted the tourist.\nOutput: The tourist was contacted by the secretary.\n\nInput: The judge recommended the doctors.\nOutput: The doctors were recommended by the judge.\n\nInput: The doctors advised the actor.\nOutput: The actor was advised by the doctors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The artist advised the doctor.\nOutput: The doctor was advised by the artist.\n\nInput: The judge introduced the student.\nOutput: The student was introduced by the judge.\n\nInput: The secretary contacted the tourist.\nOutput: The tourist was contacted by the secretary.\n\nInput: The judge recommended the doctors.\nOutput: The doctors were recommended by the judge.\n\nInput: The doctors advised the actor.\nOutput: The actor was advised by the doctors."
}
},
"73": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The lawyers introduced the senator.\nOutput: The senator was introduced by the lawyers.\n\nInput: The judges mentioned the bankers.\nOutput: The bankers were mentioned by the judges.\n\nInput: The lawyers supported the judge.\nOutput: The judge was supported by the lawyers.\n\nInput: The tourists recommended the bankers.\nOutput: The bankers were recommended by the tourists.\n\nInput: The scientist encouraged the doctors.\nOutput: The doctors were encouraged by the scientist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The lawyers introduced the senator.\nOutput: The senator was introduced by the lawyers.\n\nInput: The judges mentioned the bankers.\nOutput: The bankers were mentioned by the judges.\n\nInput: The lawyers supported the judge.\nOutput: The judge was supported by the lawyers.\n\nInput: The tourists recommended the bankers.\nOutput: The bankers were recommended by the tourists.\n\nInput: The scientist encouraged the doctors.\nOutput: The doctors were encouraged by the scientist."
}
},
"74": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The athletes mentioned the tourist.\nOutput: The tourist was mentioned by the athletes.\n\nInput: The artist advised the scientists.\nOutput: The scientists were advised by the artist.\n\nInput: The banker introduced the tourist.\nOutput: The tourist was introduced by the banker.\n\nInput: The students supported the authors.\nOutput: The authors were supported by the students.\n\nInput: The artists recommended the athlete.\nOutput: The athlete was recommended by the artists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The athletes mentioned the tourist.\nOutput: The tourist was mentioned by the athletes.\n\nInput: The artist advised the scientists.\nOutput: The scientists were advised by the artist.\n\nInput: The banker introduced the tourist.\nOutput: The tourist was introduced by the banker.\n\nInput: The students supported the authors.\nOutput: The authors were supported by the students.\n\nInput: The artists recommended the athlete.\nOutput: The athlete was recommended by the artists."
}
},
"75": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The secretary admired the judge.\nOutput: The judge was admired by the secretary.\n\nInput: The senator recognized the managers.\nOutput: The managers were recognized by the senator.\n\nInput: The secretary mentioned the manager.\nOutput: The manager was mentioned by the secretary.\n\nInput: The president thanked the actor.\nOutput: The actor was thanked by the president.\n\nInput: The artist stopped the president.\nOutput: The president was stopped by the artist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The secretary admired the judge.\nOutput: The judge was admired by the secretary.\n\nInput: The senator recognized the managers.\nOutput: The managers were recognized by the senator.\n\nInput: The secretary mentioned the manager.\nOutput: The manager was mentioned by the secretary.\n\nInput: The president thanked the actor.\nOutput: The actor was thanked by the president.\n\nInput: The artist stopped the president.\nOutput: The president was stopped by the artist."
}
},
"76": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The actors recommended the presidents.\nOutput: The presidents were recommended by the actors.\n\nInput: The secretary thanked the judges.\nOutput: The judges were thanked by the secretary.\n\nInput: The artist thanked the actors.\nOutput: The actors were thanked by the artist.\n\nInput: The bankers believed the manager.\nOutput: The manager was believed by the bankers.\n\nInput: The athlete recommended the president.\nOutput: The president was recommended by the athlete.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The actors recommended the presidents.\nOutput: The presidents were recommended by the actors.\n\nInput: The secretary thanked the judges.\nOutput: The judges were thanked by the secretary.\n\nInput: The artist thanked the actors.\nOutput: The actors were thanked by the artist.\n\nInput: The bankers believed the manager.\nOutput: The manager was believed by the bankers.\n\nInput: The athlete recommended the president.\nOutput: The president was recommended by the athlete."
}
},
"77": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The tourists recognized the lawyer.\nOutput: The lawyer was recognized by the tourists.\n\nInput: The tourist encouraged the professors.\nOutput: The professors were encouraged by the tourist.\n\nInput: The athletes recognized the judge.\nOutput: The judge was recognized by the athletes.\n\nInput: The secretaries advised the banker.\nOutput: The banker was advised by the secretaries.\n\nInput: The professors avoided the students.\nOutput: The students were avoided by the professors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The tourists recognized the lawyer.\nOutput: The lawyer was recognized by the tourists.\n\nInput: The tourist encouraged the professors.\nOutput: The professors were encouraged by the tourist.\n\nInput: The athletes recognized the judge.\nOutput: The judge was recognized by the athletes.\n\nInput: The secretaries advised the banker.\nOutput: The banker was advised by the secretaries.\n\nInput: The professors avoided the students.\nOutput: The students were avoided by the professors."
}
},
"78": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The doctor supported the secretary.\nOutput: The secretary was supported by the doctor.\n\nInput: The senator mentioned the professor.\nOutput: The professor was mentioned by the senator.\n\nInput: The artists contacted the doctors.\nOutput: The doctors were contacted by the artists.\n\nInput: The actor stopped the tourists.\nOutput: The tourists were stopped by the actor.\n\nInput: The president admired the lawyers.\nOutput: The lawyers were admired by the president.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The doctor supported the secretary.\nOutput: The secretary was supported by the doctor.\n\nInput: The senator mentioned the professor.\nOutput: The professor was mentioned by the senator.\n\nInput: The artists contacted the doctors.\nOutput: The doctors were contacted by the artists.\n\nInput: The actor stopped the tourists.\nOutput: The tourists were stopped by the actor.\n\nInput: The president admired the lawyers.\nOutput: The lawyers were admired by the president."
}
},
"79": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The student mentioned the banker.\nOutput: The banker was mentioned by the student.\n\nInput: The senators believed the presidents.\nOutput: The presidents were believed by the senators.\n\nInput: The artist mentioned the author.\nOutput: The author was mentioned by the artist.\n\nInput: The authors supported the artist.\nOutput: The artist was supported by the authors.\n\nInput: The actor recommended the professor.\nOutput: The professor was recommended by the actor.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The student mentioned the banker.\nOutput: The banker was mentioned by the student.\n\nInput: The senators believed the presidents.\nOutput: The presidents were believed by the senators.\n\nInput: The artist mentioned the author.\nOutput: The author was mentioned by the artist.\n\nInput: The authors supported the artist.\nOutput: The artist was supported by the authors.\n\nInput: The actor recommended the professor.\nOutput: The professor was recommended by the actor."
}
},
"80": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The doctors thanked the lawyers.\nOutput: The lawyers were thanked by the doctors.\n\nInput: The scientists admired the secretaries.\nOutput: The secretaries were admired by the scientists.\n\nInput: The secretary recommended the students.\nOutput: The students were recommended by the secretary.\n\nInput: The artist introduced the lawyer.\nOutput: The lawyer was introduced by the artist.\n\nInput: The authors contacted the judges.\nOutput: The judges were contacted by the authors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The doctors thanked the lawyers.\nOutput: The lawyers were thanked by the doctors.\n\nInput: The scientists admired the secretaries.\nOutput: The secretaries were admired by the scientists.\n\nInput: The secretary recommended the students.\nOutput: The students were recommended by the secretary.\n\nInput: The artist introduced the lawyer.\nOutput: The lawyer was introduced by the artist.\n\nInput: The authors contacted the judges.\nOutput: The judges were contacted by the authors."
}
},
"81": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The tourists admired the presidents.\nOutput: The presidents were admired by the tourists.\n\nInput: The manager thanked the president.\nOutput: The president was thanked by the manager.\n\nInput: The professor avoided the judges.\nOutput: The judges were avoided by the professor.\n\nInput: The doctors contacted the authors.\nOutput: The authors were contacted by the doctors.\n\nInput: The judge admired the managers.\nOutput: The managers were admired by the judge.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The tourists admired the presidents.\nOutput: The presidents were admired by the tourists.\n\nInput: The manager thanked the president.\nOutput: The president was thanked by the manager.\n\nInput: The professor avoided the judges.\nOutput: The judges were avoided by the professor.\n\nInput: The doctors contacted the authors.\nOutput: The authors were contacted by the doctors.\n\nInput: The judge admired the managers.\nOutput: The managers were admired by the judge."
}
},
"82": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The professors avoided the students.\nOutput: The students were avoided by the professors.\n\nInput: The doctors stopped the actor.\nOutput: The actor was stopped by the doctors.\n\nInput: The manager stopped the senator.\nOutput: The senator was stopped by the manager.\n\nInput: The senator advised the doctors.\nOutput: The doctors were advised by the senator.\n\nInput: The judge encouraged the banker.\nOutput: The banker was encouraged by the judge.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The professors avoided the students.\nOutput: The students were avoided by the professors.\n\nInput: The doctors stopped the actor.\nOutput: The actor was stopped by the doctors.\n\nInput: The manager stopped the senator.\nOutput: The senator was stopped by the manager.\n\nInput: The senator advised the doctors.\nOutput: The doctors were advised by the senator.\n\nInput: The judge encouraged the banker.\nOutput: The banker was encouraged by the judge."
}
},
"83": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The judges advised the doctors.\nOutput: The doctors were advised by the judges.\n\nInput: The manager recognized the president.\nOutput: The president was recognized by the manager.\n\nInput: The tourist advised the doctors.\nOutput: The doctors were advised by the tourist.\n\nInput: The authors supported the artist.\nOutput: The artist was supported by the authors.\n\nInput: The actor encouraged the scientists.\nOutput: The scientists were encouraged by the actor.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The judges advised the doctors.\nOutput: The doctors were advised by the judges.\n\nInput: The manager recognized the president.\nOutput: The president was recognized by the manager.\n\nInput: The tourist advised the doctors.\nOutput: The doctors were advised by the tourist.\n\nInput: The authors supported the artist.\nOutput: The artist was supported by the authors.\n\nInput: The actor encouraged the scientists.\nOutput: The scientists were encouraged by the actor."
}
},
"84": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The professor avoided the author.\nOutput: The author was avoided by the professor.\n\nInput: The artists believed the professor.\nOutput: The professor was believed by the artists.\n\nInput: The professors supported the scientist.\nOutput: The scientist was supported by the professors.\n\nInput: The manager introduced the president.\nOutput: The president was introduced by the manager.\n\nInput: The authors encouraged the athlete.\nOutput: The athlete was encouraged by the authors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The professor avoided the author.\nOutput: The author was avoided by the professor.\n\nInput: The artists believed the professor.\nOutput: The professor was believed by the artists.\n\nInput: The professors supported the scientist.\nOutput: The scientist was supported by the professors.\n\nInput: The manager introduced the president.\nOutput: The president was introduced by the manager.\n\nInput: The authors encouraged the athlete.\nOutput: The athlete was encouraged by the authors."
}
},
"85": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The bankers believed the manager.\nOutput: The manager was believed by the bankers.\n\nInput: The scientists recommended the tourist.\nOutput: The tourist was recommended by the scientists.\n\nInput: The actor supported the president.\nOutput: The president was supported by the actor.\n\nInput: The artists advised the secretaries.\nOutput: The secretaries were advised by the artists.\n\nInput: The managers introduced the bankers.\nOutput: The bankers were introduced by the managers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The bankers believed the manager.\nOutput: The manager was believed by the bankers.\n\nInput: The scientists recommended the tourist.\nOutput: The tourist was recommended by the scientists.\n\nInput: The actor supported the president.\nOutput: The president was supported by the actor.\n\nInput: The artists advised the secretaries.\nOutput: The secretaries were advised by the artists.\n\nInput: The managers introduced the bankers.\nOutput: The bankers were introduced by the managers."
}
},
"86": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The manager recognized the doctors.\nOutput: The doctors were recognized by the manager.\n\nInput: The judge encouraged the tourists.\nOutput: The tourists were encouraged by the judge.\n\nInput: The author believed the actor.\nOutput: The actor was believed by the author.\n\nInput: The managers helped the professors.\nOutput: The professors were helped by the managers.\n\nInput: The banker contacted the tourist.\nOutput: The tourist was contacted by the banker.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The manager recognized the doctors.\nOutput: The doctors were recognized by the manager.\n\nInput: The judge encouraged the tourists.\nOutput: The tourists were encouraged by the judge.\n\nInput: The author believed the actor.\nOutput: The actor was believed by the author.\n\nInput: The managers helped the professors.\nOutput: The professors were helped by the managers.\n\nInput: The banker contacted the tourist.\nOutput: The tourist was contacted by the banker."
}
},
"87": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The actor thanked the authors.\nOutput: The authors were thanked by the actor.\n\nInput: The doctors admired the presidents.\nOutput: The presidents were admired by the doctors.\n\nInput: The actor encouraged the scientists.\nOutput: The scientists were encouraged by the actor.\n\nInput: The judges recognized the senator.\nOutput: The senator was recognized by the judges.\n\nInput: The secretaries helped the judge.\nOutput: The judge was helped by the secretaries.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The actor thanked the authors.\nOutput: The authors were thanked by the actor.\n\nInput: The doctors admired the presidents.\nOutput: The presidents were admired by the doctors.\n\nInput: The actor encouraged the scientists.\nOutput: The scientists were encouraged by the actor.\n\nInput: The judges recognized the senator.\nOutput: The senator was recognized by the judges.\n\nInput: The secretaries helped the judge.\nOutput: The judge was helped by the secretaries."
}
},
"88": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The president introduced the lawyer.\nOutput: The lawyer was introduced by the president.\n\nInput: The authors recognized the banker.\nOutput: The banker was recognized by the authors.\n\nInput: The lawyers supported the secretary.\nOutput: The secretary was supported by the lawyers.\n\nInput: The artists avoided the secretary.\nOutput: The secretary was avoided by the artists.\n\nInput: The artist stopped the athlete.\nOutput: The athlete was stopped by the artist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The president introduced the lawyer.\nOutput: The lawyer was introduced by the president.\n\nInput: The authors recognized the banker.\nOutput: The banker was recognized by the authors.\n\nInput: The lawyers supported the secretary.\nOutput: The secretary was supported by the lawyers.\n\nInput: The artists avoided the secretary.\nOutput: The secretary was avoided by the artists.\n\nInput: The artist stopped the athlete.\nOutput: The athlete was stopped by the artist."
}
},
"89": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The students stopped the doctor.\nOutput: The doctor was stopped by the students.\n\nInput: The professors mentioned the tourist.\nOutput: The tourist was mentioned by the professors.\n\nInput: The tourist recommended the senators.\nOutput: The senators were recommended by the tourist.\n\nInput: The scientist mentioned the student.\nOutput: The student was mentioned by the scientist.\n\nInput: The doctor avoided the artist.\nOutput: The artist was avoided by the doctor.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The students stopped the doctor.\nOutput: The doctor was stopped by the students.\n\nInput: The professors mentioned the tourist.\nOutput: The tourist was mentioned by the professors.\n\nInput: The tourist recommended the senators.\nOutput: The senators were recommended by the tourist.\n\nInput: The scientist mentioned the student.\nOutput: The student was mentioned by the scientist.\n\nInput: The doctor avoided the artist.\nOutput: The artist was avoided by the doctor."
}
},
"90": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The lawyers advised the senator.\nOutput: The senator was advised by the lawyers.\n\nInput: The tourist thanked the senators.\nOutput: The senators were thanked by the tourist.\n\nInput: The tourists advised the actor.\nOutput: The actor was advised by the tourists.\n\nInput: The secretary recommended the scientists.\nOutput: The scientists were recommended by the secretary.\n\nInput: The managers contacted the scientist.\nOutput: The scientist was contacted by the managers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The lawyers advised the senator.\nOutput: The senator was advised by the lawyers.\n\nInput: The tourist thanked the senators.\nOutput: The senators were thanked by the tourist.\n\nInput: The tourists advised the actor.\nOutput: The actor was advised by the tourists.\n\nInput: The secretary recommended the scientists.\nOutput: The scientists were recommended by the secretary.\n\nInput: The managers contacted the scientist.\nOutput: The scientist was contacted by the managers."
}
},
"91": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The manager recognized the president.\nOutput: The president was recognized by the manager.\n\nInput: The doctors introduced the actor.\nOutput: The actor was introduced by the doctors.\n\nInput: The professor helped the actors.\nOutput: The actors were helped by the professor.\n\nInput: The scientist stopped the banker.\nOutput: The banker was stopped by the scientist.\n\nInput: The secretaries advised the scientists.\nOutput: The scientists were advised by the secretaries.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The manager recognized the president.\nOutput: The president was recognized by the manager.\n\nInput: The doctors introduced the actor.\nOutput: The actor was introduced by the doctors.\n\nInput: The professor helped the actors.\nOutput: The actors were helped by the professor.\n\nInput: The scientist stopped the banker.\nOutput: The banker was stopped by the scientist.\n\nInput: The secretaries advised the scientists.\nOutput: The scientists were advised by the secretaries."
}
},
"92": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The author stopped the professor.\nOutput: The professor was stopped by the author.\n\nInput: The actors admired the artist.\nOutput: The artist was admired by the actors.\n\nInput: The tourists avoided the secretaries.\nOutput: The secretaries were avoided by the tourists.\n\nInput: The manager believed the artist.\nOutput: The artist was believed by the manager.\n\nInput: The professor contacted the lawyer.\nOutput: The lawyer was contacted by the professor.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The author stopped the professor.\nOutput: The professor was stopped by the author.\n\nInput: The actors admired the artist.\nOutput: The artist was admired by the actors.\n\nInput: The tourists avoided the secretaries.\nOutput: The secretaries were avoided by the tourists.\n\nInput: The manager believed the artist.\nOutput: The artist was believed by the manager.\n\nInput: The professor contacted the lawyer.\nOutput: The lawyer was contacted by the professor."
}
},
"93": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The actors introduced the doctor.\nOutput: The doctor was introduced by the actors.\n\nInput: The secretary admired the scientist.\nOutput: The scientist was admired by the secretary.\n\nInput: The judges thanked the athlete.\nOutput: The athlete was thanked by the judges.\n\nInput: The students helped the authors.\nOutput: The authors were helped by the students.\n\nInput: The artists thanked the athlete.\nOutput: The athlete was thanked by the artists.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The actors introduced the doctor.\nOutput: The doctor was introduced by the actors.\n\nInput: The secretary admired the scientist.\nOutput: The scientist was admired by the secretary.\n\nInput: The judges thanked the athlete.\nOutput: The athlete was thanked by the judges.\n\nInput: The students helped the authors.\nOutput: The authors were helped by the students.\n\nInput: The artists thanked the athlete.\nOutput: The athlete was thanked by the artists."
}
},
"94": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The author believed the tourist.\nOutput: The tourist was believed by the author.\n\nInput: The doctors thanked the author.\nOutput: The author was thanked by the doctors.\n\nInput: The actors recognized the author.\nOutput: The author was recognized by the actors.\n\nInput: The lawyer encouraged the senators.\nOutput: The senators were encouraged by the lawyer.\n\nInput: The doctors advised the author.\nOutput: The author was advised by the doctors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The author believed the tourist.\nOutput: The tourist was believed by the author.\n\nInput: The doctors thanked the author.\nOutput: The author was thanked by the doctors.\n\nInput: The actors recognized the author.\nOutput: The author was recognized by the actors.\n\nInput: The lawyer encouraged the senators.\nOutput: The senators were encouraged by the lawyer.\n\nInput: The doctors advised the author.\nOutput: The author was advised by the doctors."
}
},
"95": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The presidents contacted the actor.\nOutput: The actor was contacted by the presidents.\n\nInput: The secretary stopped the lawyer.\nOutput: The lawyer was stopped by the secretary.\n\nInput: The manager advised the scientist.\nOutput: The scientist was advised by the manager.\n\nInput: The banker contacted the author.\nOutput: The author was contacted by the banker.\n\nInput: The professors recognized the athlete.\nOutput: The athlete was recognized by the professors.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The presidents contacted the actor.\nOutput: The actor was contacted by the presidents.\n\nInput: The secretary stopped the lawyer.\nOutput: The lawyer was stopped by the secretary.\n\nInput: The manager advised the scientist.\nOutput: The scientist was advised by the manager.\n\nInput: The banker contacted the author.\nOutput: The author was contacted by the banker.\n\nInput: The professors recognized the athlete.\nOutput: The athlete was recognized by the professors."
}
},
"96": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The lawyers helped the actor.\nOutput: The actor was helped by the lawyers.\n\nInput: The artists encouraged the professor.\nOutput: The professor was encouraged by the artists.\n\nInput: The senator contacted the actor.\nOutput: The actor was contacted by the senator.\n\nInput: The doctor supported the secretary.\nOutput: The secretary was supported by the doctor.\n\nInput: The manager recommended the author.\nOutput: The author was recommended by the manager.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The lawyers helped the actor.\nOutput: The actor was helped by the lawyers.\n\nInput: The artists encouraged the professor.\nOutput: The professor was encouraged by the artists.\n\nInput: The senator contacted the actor.\nOutput: The actor was contacted by the senator.\n\nInput: The doctor supported the secretary.\nOutput: The secretary was supported by the doctor.\n\nInput: The manager recommended the author.\nOutput: The author was recommended by the manager."
}
},
"97": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The manager thanked the student.\nOutput: The student was thanked by the manager.\n\nInput: The lawyer avoided the secretaries.\nOutput: The secretaries were avoided by the lawyer.\n\nInput: The managers contacted the scientist.\nOutput: The scientist was contacted by the managers.\n\nInput: The banker advised the author.\nOutput: The author was advised by the banker.\n\nInput: The lawyers supported the judge.\nOutput: The judge was supported by the lawyers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The manager thanked the student.\nOutput: The student was thanked by the manager.\n\nInput: The lawyer avoided the secretaries.\nOutput: The secretaries were avoided by the lawyer.\n\nInput: The managers contacted the scientist.\nOutput: The scientist was contacted by the managers.\n\nInput: The banker advised the author.\nOutput: The author was advised by the banker.\n\nInput: The lawyers supported the judge.\nOutput: The judge was supported by the lawyers."
}
},
"98": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The president believed the professors.\nOutput: The professors were believed by the president.\n\nInput: The actor advised the managers.\nOutput: The managers were advised by the actor.\n\nInput: The scientist recommended the senators.\nOutput: The senators were recommended by the scientist.\n\nInput: The scientist recognized the student.\nOutput: The student was recognized by the scientist.\n\nInput: The secretary recommended the professor.\nOutput: The professor was recommended by the secretary.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The president believed the professors.\nOutput: The professors were believed by the president.\n\nInput: The actor advised the managers.\nOutput: The managers were advised by the actor.\n\nInput: The scientist recommended the senators.\nOutput: The senators were recommended by the scientist.\n\nInput: The scientist recognized the student.\nOutput: The student was recognized by the scientist.\n\nInput: The secretary recommended the professor.\nOutput: The professor was recommended by the secretary."
}
},
"99": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The lawyer avoided the professor.\nOutput: The professor was avoided by the lawyer.\n\nInput: The doctor encouraged the lawyers.\nOutput: The lawyers were encouraged by the doctor.\n\nInput: The students recognized the tourist.\nOutput: The tourist was recognized by the students.\n\nInput: The actors recommended the presidents.\nOutput: The presidents were recommended by the actors.\n\nInput: The scientist recognized the student.\nOutput: The student was recognized by the scientist.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The lawyer avoided the professor.\nOutput: The professor was avoided by the lawyer.\n\nInput: The doctor encouraged the lawyers.\nOutput: The lawyers were encouraged by the doctor.\n\nInput: The students recognized the tourist.\nOutput: The tourist was recognized by the students.\n\nInput: The actors recommended the presidents.\nOutput: The presidents were recommended by the actors.\n\nInput: The scientist recognized the student.\nOutput: The student was recognized by the scientist."
}
},
"100": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: The presidents thanked the actors.\nOutput: The actors were thanked by the presidents.\n\nInput: The manager recognized the doctors.\nOutput: The doctors were recognized by the manager.\n\nInput: The actor helped the senators.\nOutput: The senators were helped by the actor.\n\nInput: The athlete avoided the judge.\nOutput: The judge was avoided by the athlete.\n\nInput: The managers avoided the actors.\nOutput: The actors were avoided by the managers.\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: The presidents thanked the actors.\nOutput: The actors were thanked by the presidents.\n\nInput: The manager recognized the doctors.\nOutput: The doctors were recognized by the manager.\n\nInput: The actor helped the senators.\nOutput: The senators were helped by the actor.\n\nInput: The athlete avoided the judge.\nOutput: The judge was avoided by the athlete.\n\nInput: The managers avoided the actors.\nOutput: The actors were avoided by the managers."
}
}
}
}
================================================
FILE: InstructZero/experiments/data/instruction_induction/induction_input/antonyms.json
================================================
{
"examples": {
"1": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: informed\nOutput: uninformed\n\nInput: shout\nOutput: whisper\n\nInput: distal\nOutput: proximal\n\nInput: defiant\nOutput: compliant\n\nInput: unrestricted\nOutput: restricted\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: informed\nOutput: uninformed\n\nInput: shout\nOutput: whisper\n\nInput: distal\nOutput: proximal\n\nInput: defiant\nOutput: compliant\n\nInput: unrestricted\nOutput: restricted"
}
},
"2": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: inoffensive\nOutput: offensive\n\nInput: untapped\nOutput: tapped\n\nInput: ordinal\nOutput: cardinal\n\nInput: unexpired\nOutput: expired\n\nInput: obscurity\nOutput: clarity\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: inoffensive\nOutput: offensive\n\nInput: untapped\nOutput: tapped\n\nInput: ordinal\nOutput: cardinal\n\nInput: unexpired\nOutput: expired\n\nInput: obscurity\nOutput: clarity"
}
},
"3": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: structured\nOutput: unstructured\n\nInput: designed\nOutput: undesigned\n\nInput: blond\nOutput: brunet\n\nInput: activator\nOutput: inhibitor\n\nInput: geographic\nOutput: magnetic\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: structured\nOutput: unstructured\n\nInput: designed\nOutput: undesigned\n\nInput: blond\nOutput: brunet\n\nInput: activator\nOutput: inhibitor\n\nInput: geographic\nOutput: magnetic"
}
},
"4": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: proactive\nOutput: retroactive\n\nInput: cheerful\nOutput: depressing\n\nInput: service\nOutput: disservice\n\nInput: coarsely\nOutput: finely\n\nInput: unashamed\nOutput: ashamed\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: proactive\nOutput: retroactive\n\nInput: cheerful\nOutput: depressing\n\nInput: service\nOutput: disservice\n\nInput: coarsely\nOutput: finely\n\nInput: unashamed\nOutput: ashamed"
}
},
"5": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: harmony\nOutput: dissonance\n\nInput: irreplaceable\nOutput: replaceable\n\nInput: inflexible\nOutput: flexible\n\nInput: discreet\nOutput: indiscreet\n\nInput: nonpolitical\nOutput: political\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: harmony\nOutput: dissonance\n\nInput: irreplaceable\nOutput: replaceable\n\nInput: inflexible\nOutput: flexible\n\nInput: discreet\nOutput: indiscreet\n\nInput: nonpolitical\nOutput: political"
}
},
"6": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: infinitely\nOutput: finitely\n\nInput: imprecision\nOutput: precision\n\nInput: frequent\nOutput: infrequent\n\nInput: wish\nOutput: begrudge\n\nInput: obstructed\nOutput: unobstructed\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: infinitely\nOutput: finitely\n\nInput: imprecision\nOutput: precision\n\nInput: frequent\nOutput: infrequent\n\nInput: wish\nOutput: begrudge\n\nInput: obstructed\nOutput: unobstructed"
}
},
"7": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: indecision\nOutput: decision\n\nInput: progressive\nOutput: regressive\n\nInput: dissonance\nOutput: harmony\n\nInput: incurious\nOutput: curious\n\nInput: strong\nOutput: weak\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: indecision\nOutput: decision\n\nInput: progressive\nOutput: regressive\n\nInput: dissonance\nOutput: harmony\n\nInput: incurious\nOutput: curious\n\nInput: strong\nOutput: weak"
}
},
"8": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: autologous\nOutput: homologous\n\nInput: tonic\nOutput: atonic\n\nInput: suffix\nOutput: prefix\n\nInput: unaddressed\nOutput: addressed\n\nInput: fatal\nOutput: nonfatal\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: autologous\nOutput: homologous\n\nInput: tonic\nOutput: atonic\n\nInput: suffix\nOutput: prefix\n\nInput: unaddressed\nOutput: addressed\n\nInput: fatal\nOutput: nonfatal"
}
},
"9": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: disorder\nOutput: order\n\nInput: unchangeable\nOutput: changeable\n\nInput: finite\nOutput: infinite\n\nInput: distant\nOutput: close\n\nInput: sophisticated\nOutput: naive\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: disorder\nOutput: order\n\nInput: unchangeable\nOutput: changeable\n\nInput: finite\nOutput: infinite\n\nInput: distant\nOutput: close\n\nInput: sophisticated\nOutput: naive"
}
},
"10": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: insolubility\nOutput: solubility\n\nInput: inflexibility\nOutput: flexibility\n\nInput: found\nOutput: lost\n\nInput: vertebrate\nOutput: invertebrate\n\nInput: demobilization\nOutput: mobilization\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: insolubility\nOutput: solubility\n\nInput: inflexibility\nOutput: flexibility\n\nInput: found\nOutput: lost\n\nInput: vertebrate\nOutput: invertebrate\n\nInput: demobilization\nOutput: mobilization"
}
},
"11": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: home\nOutput: away\n\nInput: unofficial\nOutput: official\n\nInput: insignificant\nOutput: significant\n\nInput: illogically\nOutput: logically\n\nInput: honesty\nOutput: dishonesty\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: home\nOutput: away\n\nInput: unofficial\nOutput: official\n\nInput: insignificant\nOutput: significant\n\nInput: illogically\nOutput: logically\n\nInput: honesty\nOutput: dishonesty"
}
},
"12": {
"input": "I gave a friend an instruction and five inputs. The friend read the instruction and wrote an output for every one of the inputs.\nHere are the input-output pairs:\n\nInput: personal\nOutput: impersonal\n\nInput: nonsurgical\nOutput: surgical\n\nInput: unauthorized\nOutput: authorized\n\nInput: inconsiderate\nOutput: considerate\n\nInput: vocal\nOutput: instrumental\n\nThe instruction was",
"metadata": {
"examples_seen": "Input: personal\nOutput: impersonal\n\nInput: nonsurgical\nOutput: surgical\n\nInput: unauthorize
gitextract_xr63eqto/ ├── .gitignore ├── InstructZero/ │ └── experiments/ │ ├── args.py │ ├── automatic_prompt_engineer/ │ │ ├── __init__.py │ │ ├── ape.py │ │ ├── config.py │ │ ├── data.py │ │ ├── evaluate.py │ │ ├── generate.py │ │ ├── llm.py │ │ └── template.py │ ├── configs/ │ │ └── instruction_induction.yaml │ ├── data/ │ │ └── instruction_induction/ │ │ ├── README.md │ │ ├── induction_input/ │ │ │ ├── active_to_passive.json │ │ │ ├── antonyms.json │ │ │ ├── cause_and_effect.json │ │ │ ├── common_concept.json │ │ │ ├── diff.json │ │ │ ├── first_word_letter.json │ │ │ ├── informal_to_formal.json │ │ │ ├── larger_animal.json │ │ │ ├── letters_list.json │ │ │ ├── negation.json │ │ │ ├── num_to_verbal.json │ │ │ ├── orthography_starts_with.json │ │ │ ├── rhymes.json │ │ │ ├── second_word_letter.json │ │ │ ├── sentence_similarity.json │ │ │ ├── sentiment.json │ │ │ ├── singular_to_plural.json │ │ │ ├── sum.json │ │ │ ├── synonyms.json │ │ │ ├── taxonomy_animal.json │ │ │ ├── translation_en-de.json │ │ │ ├── translation_en-es.json │ │ │ ├── translation_en-fr.json │ │ │ └── word_in_context.json │ │ ├── load_data.py │ │ └── raw/ │ │ ├── execute/ │ │ │ ├── active_to_passive.json │ │ │ ├── antonyms.json │ │ │ ├── auto_categorization.json │ │ │ ├── auto_debugging.json │ │ │ ├── cause_and_effect.json │ │ │ ├── common_concept.json │ │ │ ├── diff.json │ │ │ ├── first_word_letter.json │ │ │ ├── informal_to_formal.json │ │ │ ├── larger_animal.json │ │ │ ├── letters_list.json │ │ │ ├── negation.json │ │ │ ├── num_to_verbal.json │ │ │ ├── object_counting.json │ │ │ ├── odd_one_out.json │ │ │ ├── orthography_starts_with.json │ │ │ ├── periodic_elements.json │ │ │ ├── rhymes.json │ │ │ ├── second_word_letter.json │ │ │ ├── sentence_similarity.json │ │ │ ├── sentiment.json │ │ │ ├── singular_to_plural.json │ │ │ ├── sum.json │ │ │ ├── synonyms.json │ │ │ ├── taxonomy_animal.json │ │ │ ├── translation_en-de.json │ │ │ ├── translation_en-es.json │ │ │ ├── translation_en-fr.json │ │ │ ├── word_in_context.json │ │ │ ├── word_sorting.json │ │ │ └── word_unscrambling.json │ │ └── induce/ │ │ ├── active_to_passive.json │ │ ├── antonyms.json │ │ ├── ascii_word_recognition.json │ │ ├── auto_categorization.json │ │ ├── auto_debugging.json │ │ ├── cause_and_effect.json │ │ ├── common_concept.json │ │ ├── cs_algorithms.json │ │ ├── diff.json │ │ ├── first_word_letter.json │ │ ├── informal_to_formal.json │ │ ├── larger_animal.json │ │ ├── letters_list.json │ │ ├── negation.json │ │ ├── num_to_verbal.json │ │ ├── object_counting.json │ │ ├── odd_one_out.json │ │ ├── orthography_starts_with.json │ │ ├── periodic_elements.json │ │ ├── rhymes.json │ │ ├── second_word_letter.json │ │ ├── sentence_similarity.json │ │ ├── sentiment.json │ │ ├── singular_to_plural.json │ │ ├── sum.json │ │ ├── synonyms.json │ │ ├── taxonomy_animal.json │ │ ├── translation_en-de.json │ │ ├── translation_en-es.json │ │ ├── translation_en-fr.json │ │ ├── word_in_context.json │ │ ├── word_sorting.json │ │ └── word_unscrambling.json │ ├── evaluation/ │ │ └── instruction_induction/ │ │ ├── __init__.py │ │ ├── exec_accuracy.py │ │ └── utility.py │ ├── instruction_coupled_kernel.py │ ├── misc.py │ ├── run_instructzero.py │ └── run_instructzero.sh ├── README.md └── requirements.txt
SYMBOL INDEX (120 symbols across 14 files)
FILE: InstructZero/experiments/args.py
function parse_args (line 3) | def parse_args():
FILE: InstructZero/experiments/automatic_prompt_engineer/ape.py
function get_simple_prompt_gen_template (line 5) | def get_simple_prompt_gen_template(prompt_gen_template, prompt_gen_mode):
function simple_ape (line 17) | def simple_ape(dataset,
function simple_eval (line 59) | def simple_eval(dataset,
function simple_estimate_cost (line 86) | def simple_estimate_cost(dataset,
function find_prompts (line 105) | def find_prompts(eval_template,
function evaluate_prompts (line 162) | def evaluate_prompts(prompts, eval_template, eval_data, demos_template, ...
function estimate_cost (line 193) | def estimate_cost(eval_template,
function get_generation_query (line 252) | def get_generation_query(eval_template,
function get_evaluation_query (line 277) | def get_evaluation_query(eval_template,
FILE: InstructZero/experiments/automatic_prompt_engineer/config.py
function update_config (line 5) | def update_config(config, base_config='configs/default.yaml'):
function simple_config (line 23) | def simple_config(eval_model, prompt_gen_model, prompt_gen_mode, num_pro...
FILE: InstructZero/experiments/automatic_prompt_engineer/data.py
function subsample_data (line 3) | def subsample_data(data, subsample_size):
function create_split (line 16) | def create_split(data, split_size):
FILE: InstructZero/experiments/automatic_prompt_engineer/evaluate.py
function get_eval_method (line 5) | def get_eval_method(eval_method):
function evaluate_prompts (line 25) | def evaluate_prompts(prompts, eval_template, eval_data, demos_template, ...
function demo_function (line 41) | def demo_function(eval_template, config):
class EvaluationResult (line 66) | class EvaluationResult(ABC):
method sorted (line 69) | def sorted(self, method='default'):
method in_place (line 75) | def in_place(self, method='default'):
FILE: InstructZero/experiments/automatic_prompt_engineer/generate.py
function get_query (line 4) | def get_query(prompt_gen_template, demos_template, subsampled_data):
function generate_prompts (line 19) | def generate_prompts(prompt_gen_template, demos_template, prompt_gen_dat...
FILE: InstructZero/experiments/automatic_prompt_engineer/llm.py
function dispatch_openai_requests (line 22) | async def dispatch_openai_requests(
function model_from_config (line 54) | def model_from_config(config, disable_tqdm=True):
class LLM (line 66) | class LLM(ABC):
method generate_text (line 70) | def generate_text(self, prompt):
method log_probs (line 80) | def log_probs(self, text, log_prob_range):
class Llama_Forward (line 91) | class Llama_Forward(LLM):
method __init__ (line 94) | def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
method auto_reduce_n (line 105) | def auto_reduce_n(self, fn, prompt, n):
method generate_text (line 114) | def generate_text(self, prompts, n):
method complete (line 125) | def complete(self, prompt, n):
method log_probs (line 141) | def log_probs(self, text, log_prob_range=None):
class Flan_T5 (line 171) | class Flan_T5(LLM):
method __init__ (line 174) | def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
method auto_reduce_n (line 184) | def auto_reduce_n(self, fn, prompt, n):
method generate_text (line 193) | def generate_text(self, prompts, n):
method complete (line 222) | def complete(self, prompt, n):
method log_probs (line 238) | def log_probs(self, text, log_prob_range=None):
class GPT_Forward (line 271) | class GPT_Forward(LLM):
method __init__ (line 274) | def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
method confirm_cost (line 280) | def confirm_cost(self, texts, n, max_tokens):
method auto_reduce_n (line 292) | def auto_reduce_n(self, fn, prompt, n):
method generate_text (line 301) | def generate_text(self, prompt, n):
method complete (line 321) | def complete(self, prompt, n):
method log_probs (line 337) | def log_probs(self, text, log_prob_range=None):
method __generate_text (line 366) | def __generate_text(self, prompt, n):
method __async_generate (line 402) | def __async_generate(self, prompt, n):
method __complete (line 441) | def __complete(self, prompt, n):
method __log_probs (line 461) | def __log_probs(self, text, log_prob_range=None):
method get_token_indices (line 512) | def get_token_indices(self, offsets, log_prob_range):
class Claude_Forward (line 531) | class Claude_Forward(LLM):
method __init__ (line 534) | def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
method confirm_cost (line 540) | def confirm_cost(self, texts, n, max_tokens):
method auto_reduce_n (line 552) | def auto_reduce_n(self, fn, prompt, n):
method generate_text (line 561) | def generate_text(self, prompt, n):
method complete (line 581) | def complete(self, prompt, n):
method log_probs (line 597) | def log_probs(self, text, log_prob_range=None):
method __generate_text (line 626) | def __generate_text(self, prompt, n):
method __async_generate (line 662) | def __async_generate(self, prompt, n):
method __complete (line 701) | def __complete(self, prompt, n):
method __log_probs (line 721) | def __log_probs(self, text, log_prob_range=None):
method get_token_indices (line 772) | def get_token_indices(self, offsets, log_prob_range):
class GPT_Insert (line 795) | class GPT_Insert(LLM):
method __init__ (line 797) | def __init__(self, config, needs_confirmation=False, disable_tqdm=True):
method confirm_cost (line 803) | def confirm_cost(self, texts, n, max_tokens):
method auto_reduce_n (line 815) | def auto_reduce_n(self, fn, prompt, n):
method generate_text (line 824) | def generate_text(self, prompt, n):
method log_probs (line 842) | def log_probs(self, text, log_prob_range=None):
method __generate_text (line 845) | def __generate_text(self, prompt, n):
function gpt_get_estimated_cost (line 868) | def gpt_get_estimated_cost(config, prompt, max_tokens):
class BatchSizeException (line 891) | class BatchSizeException(Exception):
FILE: InstructZero/experiments/automatic_prompt_engineer/template.py
class InitQATemplate (line 1) | class InitQATemplate:
method __init__ (line 11) | def __init__(self, template):
method fill (line 17) | def fill(self, full_demo=''):
class GenerationTemplate (line 24) | class GenerationTemplate:
method __init__ (line 34) | def __init__(self, template):
method fill (line 40) | def fill(self, full_demo='', input='', output=''):
class EvalTemplate (line 48) | class EvalTemplate:
method __init__ (line 58) | def __init__(self, template):
method fill (line 61) | def fill(self, prompt='', full_demo='', input='', output=''):
method convert_to_generation_template (line 68) | def convert_to_generation_template(self):
class DemosTemplate (line 75) | class DemosTemplate:
method __init__ (line 83) | def __init__(self, template, delimiter='\n\n'):
method fill (line 87) | def fill(self, data):
FILE: InstructZero/experiments/data/instruction_induction/load_data.py
function load_data (line 12) | def load_data(type, task):
FILE: InstructZero/experiments/evaluation/instruction_induction/exec_accuracy.py
function get_query (line 7) | def get_query(prompt, eval_template, input_, output_, demo_data, demos_t...
function get_query_for_test (line 16) | def get_query_for_test(prompt, eval_template, input_, output_):
function exec_accuracy_evaluator (line 25) | def exec_accuracy_evaluator(prompts, eval_template, eval_data, demos_tem...
class exec_evaluator (line 72) | class exec_evaluator(object):
method __init__ (line 73) | def __init__(self, api_model, config):
method evaluate (line 80) | def evaluate(self, prompts, eval_template, eval_data, demos_template, ...
method test (line 124) | def test(self, prompts, eval_template, eval_data, config):
class ExecAccuracyEvaluationResult (line 166) | class ExecAccuracyEvaluationResult(evaluate.EvaluationResult):
method __init__ (line 168) | def __init__(self, prompts, scores):
method _agg_scores (line 172) | def _agg_scores(self, method):
method sorted (line 189) | def sorted(self, method='default'):
method in_place (line 202) | def in_place(self, method='default'):
FILE: InstructZero/experiments/evaluation/instruction_induction/utility.py
function normalize_prediction (line 16) | def normalize_prediction(prediction, lowercase=True):
function get_f1_score (line 35) | def get_f1_score(prediction, ground_truth):
function get_em_score (line 50) | def get_em_score(prediction, ground_truth):
function get_exact_set_score (line 57) | def get_exact_set_score(prediction, ground_truth):
function get_contains_score (line 65) | def get_contains_score(prediction, ground_truth):
function get_multi_answer_em (line 73) | def get_multi_answer_em(prediction, answers):
function get_multi_answer_f1 (line 80) | def get_multi_answer_f1(prediction, answers):
function get_multi_answer_exact_set (line 87) | def get_multi_answer_exact_set(prediction, answers):
function get_multi_answer_contains (line 94) | def get_multi_answer_contains(prediction, answers):
FILE: InstructZero/experiments/instruction_coupled_kernel.py
class CombinedStringKernel (line 15) | class CombinedStringKernel(Kernel):
method __init__ (line 16) | def __init__(self, base_latent_kernel, instruction_kernel, latent_trai...
method forward (line 25) | def forward(self, z1, z2, **params):
function cma_es_concat (line 49) | def cma_es_concat(starting_point_for_cma, EI, tkwargs):
FILE: InstructZero/experiments/misc.py
function get_test_conf (line 31) | def get_test_conf(task, test_data):
function get_conf (line 58) | def get_conf(task, eval_data):
function set_all_seed (line 85) | def set_all_seed(seed):
FILE: InstructZero/experiments/run_instructzero.py
class LMForwardAPI (line 30) | class LMForwardAPI:
method __init__ (line 31) | def __init__(self, model_name=None, eval_data=None, init_prompt=None, ...
method eval (line 126) | def eval(self, prompt_embedding=None, test_data=None):
method return_best_prompt (line 212) | def return_best_prompt(self):
method return_prompts_set (line 215) | def return_prompts_set(self):
function run (line 218) | def run(args):
Copy disabled (too large)
Download .json
Condensed preview — 110 files, each showing path, character count, and a content snippet. Download the .json file for the full structured content (21,086K chars).
[
{
"path": ".gitignore",
"chars": 3209,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "InstructZero/experiments/args.py",
"chars": 1719,
"preview": "import argparse\n\ndef parse_args():\n parser = argparse.ArgumentParser(description=\"InstructZero pipeline\")\n parser."
},
{
"path": "InstructZero/experiments/automatic_prompt_engineer/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "InstructZero/experiments/automatic_prompt_engineer/ape.py",
"chars": 12956,
"preview": "import random\nfrom . import generate, evaluate, config, template, data, llm\n\n\ndef get_simple_prompt_gen_template(prompt_"
},
{
"path": "InstructZero/experiments/automatic_prompt_engineer/config.py",
"chars": 1740,
"preview": "import os\nimport yaml\n\n\ndef update_config(config, base_config='configs/default.yaml'):\n # Get default config from yam"
},
{
"path": "InstructZero/experiments/automatic_prompt_engineer/data.py",
"chars": 970,
"preview": "import random\nSEED=0\ndef subsample_data(data, subsample_size):\n \"\"\"\n Subsample data. Data is in the form of a tupl"
},
{
"path": "InstructZero/experiments/automatic_prompt_engineer/evaluate.py",
"chars": 2542,
"preview": "from abc import ABC, abstractmethod\nfrom . import llm\n\n\ndef get_eval_method(eval_method):\n \"\"\"\n Returns the evalua"
},
{
"path": "InstructZero/experiments/automatic_prompt_engineer/generate.py",
"chars": 1597,
"preview": "from . import data, llm\n\n\ndef get_query(prompt_gen_template, demos_template, subsampled_data):\n \"\"\"\n Returns a que"
},
{
"path": "InstructZero/experiments/automatic_prompt_engineer/llm.py",
"chars": 35986,
"preview": "\"\"\"Contains classes for querying large language models.\"\"\"\nimport os\nimport time\nfrom tqdm import tqdm\nfrom abc import A"
},
{
"path": "InstructZero/experiments/automatic_prompt_engineer/template.py",
"chars": 3331,
"preview": "class InitQATemplate:\n \"\"\"\n Takes a prompt template and provides methods for filling in blanks.\n The format is "
},
{
"path": "InstructZero/experiments/configs/instruction_induction.yaml",
"chars": 803,
"preview": "generation:\n num_subsamples: 3\n num_demos: 5\n num_prompts_per_subsample: 30\n model:\n name: GPT_forward\n batch_"
},
{
"path": "InstructZero/experiments/data/instruction_induction/README.md",
"chars": 622,
"preview": "# Data\nData for the instruction induction experiments. Followed We collect another 8 instruction induction tasks besides"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/active_to_passive.json",
"chars": 124408,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/antonyms.json",
"chars": 65650,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/cause_and_effect.json",
"chars": 159272,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/common_concept.json",
"chars": 103262,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/diff.json",
"chars": 55790,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/first_word_letter.json",
"chars": 56938,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/informal_to_formal.json",
"chars": 113138,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/larger_animal.json",
"chars": 72256,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/letters_list.json",
"chars": 69786,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/negation.json",
"chars": 136308,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/num_to_verbal.json",
"chars": 90712,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/orthography_starts_with.json",
"chars": 102848,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/rhymes.json",
"chars": 59592,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/second_word_letter.json",
"chars": 56938,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/sentence_similarity.json",
"chars": 168442,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/sentiment.json",
"chars": 99118,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/singular_to_plural.json",
"chars": 63268,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/sum.json",
"chars": 55790,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/synonyms.json",
"chars": 61714,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/taxonomy_animal.json",
"chars": 116250,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/translation_en-de.json",
"chars": 64136,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/translation_en-es.json",
"chars": 63194,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/translation_en-fr.json",
"chars": 63582,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/induction_input/word_in_context.json",
"chars": 158298,
"preview": "{\n \"examples\": {\n \"1\": {\n \"input\": \"I gave a friend an instruction and five inputs. The friend read the instruc"
},
{
"path": "InstructZero/experiments/data/instruction_induction/load_data.py",
"chars": 1646,
"preview": "import os\nimport json\nimport random\n\ninduce_data_path = os.path.join(os.path.dirname(__file__), 'raw/induce/')\neval_data"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/active_to_passive.json",
"chars": 16215,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"The professor mentioned the ar"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/antonyms.json",
"chars": 7439,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"fortunate\",\n \"output\": \"u"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/auto_categorization.json",
"chars": 13981,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Shaymin, C"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/auto_debugging.json",
"chars": 1200,
"preview": "{\n \"metadata\": {\n \"num_examples\": 8\n },\n \"examples\": {\n \"1\": {\n \"input\": \"```\\nclass M"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/cause_and_effect.json",
"chars": 2866,
"preview": "{\n \"metadata\": {\n \"num_examples\": 25\n },\n \"examples\": {\n \"1\": {\n \"cause\": \"The child hurt their knee.\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/common_concept.json",
"chars": 3552,
"preview": "{\n \"metadata\": {\n \"num_examples\": 16\n },\n \"examples\": {\n \"1\": {\n \"concept\": \"involve vertical movement.\",\n"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/diff.json",
"chars": 9495,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"47 0\",\n \"output\": \"47\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/first_word_letter.json",
"chars": 6555,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"place\",\n \"output\": \"p\"\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/informal_to_formal.json",
"chars": 1776,
"preview": "{\n \"metadata\": {\n \"num_examples\": 15\n },\n \"examples\": {\n \"1\": {\n \"input\": \"What's up?\",\n \"output\": \"H"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/larger_animal.json",
"chars": 13877,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"mirror carp, alligator\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/letters_list.json",
"chars": 20495,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"place\",\n \"output\": \"p l a"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/negation.json",
"chars": 19955,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Newton played as quarterback d"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/num_to_verbal.json",
"chars": 9972,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"94\",\n \"output\": \"ninety-f"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/object_counting.json",
"chars": 15940,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"I have two"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/odd_one_out.json",
"chars": 6008,
"preview": "{\n \"metadata\": {\n \"num_examples\": 50\n },\n \"examples\": {\n \"1\": {\n \"input\": \"twenty, sev"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/orthography_starts_with.json",
"chars": 28547,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Mary thinks that she is smart."
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/periodic_elements.json",
"chars": 4407,
"preview": "{\n \"metadata\": {\n \"num_examples\": 50\n },\n \"examples\": {\n \"1\": {\n \"input\": \"113\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/rhymes.json",
"chars": 26904,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"compete\",\n \"output\": \"nea"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/second_word_letter.json",
"chars": 6555,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"place\",\n \"output\": \"l\"\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/sentence_similarity.json",
"chars": 16848,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Sentence 1: the man is catchin"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/sentiment.json",
"chars": 11209,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"...routine, harmless diversion"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/singular_to_plural.json",
"chars": 9831,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"end\",\n \"output\": \"ends\",\n"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/sum.json",
"chars": 9449,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"0 47\",\n \"output\": \"47\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/synonyms.json",
"chars": 7006,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"procedure\",\n \"output\": \"p"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/taxonomy_animal.json",
"chars": 48522,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"kangaroo, beetle, bike, bird, "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/translation_en-de.json",
"chars": 18124,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"place\",\n \"output\": \"Platz"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/translation_en-es.json",
"chars": 16952,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"place\",\n \"output\": \"lugar"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/translation_en-fr.json",
"chars": 16630,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"place\",\n \"output\": \"place"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/word_in_context.json",
"chars": 26218,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Sentence 1: Hyenas scavenge. S"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/word_sorting.json",
"chars": 27611,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"\\nList: wa"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/execute/word_unscrambling.json",
"chars": 9167,
"preview": "{\n \"metadata\": {\n \"num_examples\": 100\n },\n \"examples\": {\n \"1\": {\n \"input\": \"atornco\",\n"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/active_to_passive.json",
"chars": 145629,
"preview": "{\n \"metadata\": {\n \"num_examples\": 900\n },\n \"examples\": {\n \"1\": {\n \"input\": \"The tourist supported the auth"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/antonyms.json",
"chars": 196169,
"preview": "{\n \"metadata\": {\n \"num_examples\": 2592\n },\n \"examples\": {\n \"1\": {\n \"input\": \"sane\",\n \"output\": \"insan"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/ascii_word_recognition.json",
"chars": 3962043,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3900\n },\n \"examples\": {\n \"1\": {\n \"input\": \".._..._.."
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/auto_categorization.json",
"chars": 24287,
"preview": "{\n \"metadata\": {\n \"num_examples\": 163\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Nigeria, R"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/auto_debugging.json",
"chars": 1770,
"preview": "{\n \"metadata\": {\n \"num_examples\": 10\n },\n \"examples\": {\n \"1\": {\n \"input\": \"```\\nd = {1"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/cause_and_effect.json",
"chars": 3008,
"preview": "{\n \"metadata\": {\n \"num_examples\": 26\n },\n \"examples\": {\n \"1\": {\n \"cause\": \"It started raining.\",\n \"ef"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/common_concept.json",
"chars": 3943,
"preview": "{\n \"metadata\": {\n \"num_examples\": 17\n },\n \"examples\": {\n \"1\": {\n \"concept\": \"involve oscillations.\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/cs_algorithms.json",
"chars": 19855,
"preview": "{\n \"metadata\": {\n \"num_examples\": 156\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Strings: Z"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/diff.json",
"chars": 477100,
"preview": "{\n \"metadata\": {\n \"num_examples\": 4950\n },\n \"examples\": {\n \"1\": {\n \"input\": \"0 0\",\n \"output\": \"0\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/first_word_letter.json",
"chars": 221130,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3306\n },\n \"examples\": {\n \"1\": {\n \"input\": \"time\",\n \"output\": \"t\"\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/informal_to_formal.json",
"chars": 1890,
"preview": "{\n \"metadata\": {\n \"num_examples\": 15\n },\n \"examples\": {\n \"1\": {\n \"input\": \"I think that this is interestin"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/larger_animal.json",
"chars": 407338,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3028\n },\n \"examples\": {\n \"1\": {\n \"input\": \"rabbit, snail\",\n \"output"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/letters_list.json",
"chars": 693975,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3306\n },\n \"examples\": {\n \"1\": {\n \"input\": \"time\",\n \"output\": \"t i m"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/negation.json",
"chars": 176692,
"preview": "{\n \"metadata\": {\n \"num_examples\": 904\n },\n \"examples\": {\n \"1\": {\n \"input\": \"To emphasize the 50th annivers"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/num_to_verbal.json",
"chars": 1000547,
"preview": "{\n \"metadata\": {\n \"num_examples\": 9900\n },\n \"examples\": {\n \"1\": {\n \"input\": \"0\",\n \"output\": \"zero\"\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/object_counting.json",
"chars": 110344,
"preview": "{\n \"metadata\": {\n \"num_examples\": 700\n },\n \"examples\": {\n \"1\": {\n \"input\": \"I have a d"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/odd_one_out.json",
"chars": 2393,
"preview": "{\n \"metadata\": {\n \"num_examples\": 19\n },\n \"examples\": {\n \"1\": {\n \"input\": \"listen, stu"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/orthography_starts_with.json",
"chars": 808765,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3000\n },\n \"examples\": {\n \"1\": {\n \"input\": \"I prefer for the girl to put "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/periodic_elements.json",
"chars": 4430,
"preview": "{\n \"metadata\": {\n \"num_examples\": 50\n },\n \"examples\": {\n \"1\": {\n \"input\": \"42\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/rhymes.json",
"chars": 1150468,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3000\n },\n \"examples\": {\n \"1\": {\n \"input\": \"pig\",\n \"output\": \"big\",\n"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/second_word_letter.json",
"chars": 221130,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3306\n },\n \"examples\": {\n \"1\": {\n \"input\": \"time\",\n \"output\": \"i\"\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/sentence_similarity.json",
"chars": 663508,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3716\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Sentence 1: A plane is taking"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/sentiment.json",
"chars": 127801,
"preview": "{\n \"metadata\": {\n \"num_examples\": 1167\n },\n \"examples\": {\n \"1\": {\n \"input\": \"unpretentious, charming, quir"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/singular_to_plural.json",
"chars": 194001,
"preview": "{\n \"metadata\": {\n \"num_examples\": 1943\n },\n \"examples\": {\n \"1\": {\n \"input\": \"game\",\n \"output\": \"games"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/sum.json",
"chars": 474621,
"preview": "{\n \"metadata\": {\n \"num_examples\": 4950\n },\n \"examples\": {\n \"1\": {\n \"input\": \"0 0\",\n \"output\": \"0\",\n "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/synonyms.json",
"chars": 158941,
"preview": "{\n \"metadata\": {\n \"num_examples\": 2222\n },\n \"examples\": {\n \"1\": {\n \"input\": \"pillow\",\n \"output\": \"res"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/taxonomy_animal.json",
"chars": 1450170,
"preview": "{\n \"metadata\": {\n \"num_examples\": 3000\n },\n \"examples\": {\n \"1\": {\n \"input\": \"jacket, tortoise, chicken, ta"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/translation_en-de.json",
"chars": 511620,
"preview": "{\n \"metadata\": {\n \"num_examples\": 2808\n },\n \"examples\": {\n \"1\": {\n \"input\": \"time\",\n \"output\": \"Zeit\""
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/translation_en-es.json",
"chars": 457420,
"preview": "{\n \"metadata\": {\n \"num_examples\": 2711\n },\n \"examples\": {\n \"1\": {\n \"input\": \"time\",\n \"output\": \"hora\""
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/translation_en-fr.json",
"chars": 468983,
"preview": "{\n \"metadata\": {\n \"num_examples\": 2811\n },\n \"examples\": {\n \"1\": {\n \"input\": \"time\",\n \"output\": \"temps"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/word_in_context.json",
"chars": 1044405,
"preview": "{\n \"metadata\": {\n \"num_examples\": 4084\n },\n \"examples\": {\n \"1\": {\n \"input\": \"Sentence 1: Approach a task. "
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/word_sorting.json",
"chars": 379171,
"preview": "{\n \"metadata\": {\n \"num_examples\": 1420\n },\n \"examples\": {\n \"1\": {\n \"input\": \"\\nList: s"
},
{
"path": "InstructZero/experiments/data/instruction_induction/raw/induce/word_unscrambling.json",
"chars": 653613,
"preview": "{\n \"metadata\": {\n \"num_examples\": 7034\n },\n \"examples\": {\n \"1\": {\n \"input\": \"ilpf\",\n "
},
{
"path": "InstructZero/experiments/evaluation/instruction_induction/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "InstructZero/experiments/evaluation/instruction_induction/exec_accuracy.py",
"chars": 7630,
"preview": "import numpy as np\n\nfrom automatic_prompt_engineer import data, llm, evaluate\nfrom evaluation.instruction_induction impo"
},
{
"path": "InstructZero/experiments/evaluation/instruction_induction/utility.py",
"chars": 3133,
"preview": "'''\nTaken from the Instruction Induction paper: https://arxiv.org/pdf/2205.10782.pdf\n'''\n\nimport re\nimport string\nfrom c"
},
{
"path": "InstructZero/experiments/instruction_coupled_kernel.py",
"chars": 3112,
"preview": "# gpytorch for substring kernel implemented in https://github.com/henrymoss/BOSS/tree/master/boss/code/kernels/string\nfr"
},
{
"path": "InstructZero/experiments/misc.py",
"chars": 2875,
"preview": "import torch\nimport random\nimport numpy as np\nfrom evaluation.instruction_induction.exec_accuracy import exec_accuracy_e"
},
{
"path": "InstructZero/experiments/run_instructzero.py",
"chars": 17768,
"preview": "import random\nimport torch\nimport numpy as np\nimport copy\nfrom automatic_prompt_engineer import ape, data\nfrom data.inst"
},
{
"path": "InstructZero/experiments/run_instructzero.sh",
"chars": 615,
"preview": "export CUDA_VISIBLE_DEVICES=0\nSFT=5\nRANDOM_PROJ='uniform'\nINTRINSIC_DIM=10\n# model_dir='lmsys/vicuna-13b-v1.3'\n# MODEL_N"
},
{
"path": "README.md",
"chars": 5197,
"preview": "# InstructZero: Efficient Instruction Optimization for Black-Box Large Language Models (ICML2024)\n\nLichang Chen*, Jiuhai"
},
{
"path": "requirements.txt",
"chars": 128,
"preview": "transformers==4.29.2\nopenai==0.27.2\nfire==0.5.0\ntqdm==4.65.0\ngradio==3.23.0\ncma==3.3.0\naccelerate==0.21.0\nsentencepiece="
}
]
About this extraction
This page contains the full source code of the Lichang-Chen/InstructZero GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 110 files (18.1 MB), approximately 4.7M tokens, and a symbol index with 120 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.