SYMBOL INDEX (24 symbols across 7 files)

FILE: extract_label.py
  function extract_label (line 5) | def extract_label(text, lang):

FILE: get_probability.py
  function create_model_tokenizer (line 28) | def create_model_tokenizer():
  function create_ipt (line 55) | def create_ipt(query, response, lang, model_base, rules=None):
  function get_probs (line 81) | def get_probs(scores, idx, lang, model_base):
  function generate (line 123) | def generate(datas, model, tokenizer, device, lang, model_base, batch_si...

FILE: infer_shieldlm.py
  function create_model_tokenizer (line 30) | def create_model_tokenizer():
  function create_ipt (line 57) | def create_ipt(query, response, lang, model_base, rules=None):
  function generate (line 83) | def generate(datas, model, tokenizer, device, lang, model_base, batch_si...

FILE: train_code/config.py
  function parse_args (line 3) | def parse_args():

FILE: train_code/data_helper.py
  class SafetyDatasetDecoderOnly (line 7) | class SafetyDatasetDecoderOnly(Dataset):
    method __init__ (line 11) | def __init__(self,
    method __len__ (line 27) | def __len__(self) -> int:
    method tokenize_text (line 30) | def tokenize_text(self, text: str, max_length, padding='max_length', a...
    method __getitem__ (line 40) | def __getitem__(self, idx: int) -> dict:

FILE: train_code/sft_data_process.py
  function add_qwen_format (line 6) | def add_qwen_format(path, outpath):
  function add_baichuan2_format (line 20) | def add_baichuan2_format(path, outpath):
  function add_llama3_format (line 34) | def add_llama3_format(path, outpath):
  function add_mistral_format (line 47) | def add_mistral_format(path, outpath):
  function add_chatglm3_format (line 61) | def add_chatglm3_format(path, outpath):
  function add_internlm2_format (line 75) | def add_internlm2_format(path, outpath):
  function change_label_first (line 79) | def change_label_first(path, outpath):
  function combine_rules (line 106) | def combine_rules(path, outpath):

FILE: train_code/train_decoderonly_hf.py
  function sft_collator (line 15) | def sft_collator(instances):
  function preprocess_logits_for_metrics (line 59) | def preprocess_logits_for_metrics(logits, labels):