SYMBOL INDEX (925 symbols across 95 files) FILE: train/dpo/dpo_trainer_2stages.py class DPOTrainer (line 60) | class DPOTrainer(Trainer): method __init__ (line 123) | def __init__( method _prepare_deepspeed (line 354) | def _prepare_deepspeed(self, model: PreTrainedModelWrapper): method concatenated_inputs (line 385) | def concatenated_inputs(self, batch: Dict[str, Union[List, torch.LongT... method dpo_loss (line 424) | def dpo_loss( method _get_batch_logps (line 467) | def _get_batch_logps( method _get_noisy_batch_logps (line 501) | def _get_noisy_batch_logps( method concatenated_forward (line 550) | def concatenated_forward( method get_batch_metrics (line 629) | def get_batch_metrics( method compute_loss (line 681) | def compute_loss( method get_batch_samples (line 702) | def get_batch_samples(self, model, batch: Dict[str, torch.LongTensor])... method prediction_step (line 739) | def prediction_step( method store_metrics (line 778) | def store_metrics(self, metrics: Dict[str, float], train_eval: Literal... method evaluation_loop (line 782) | def evaluation_loop( method log (line 832) | def log(self, logs: Dict[str, float]) -> None: FILE: train/dpo/llava/conversation.py class SeparatorStyle (line 9) | class SeparatorStyle(Enum): class Conversation (line 19) | class Conversation: method get_prompt (line 32) | def get_prompt(self): method append_message (line 109) | def append_message(self, role, message): method process_image (line 112) | def process_image(self, image, image_process_mode, return_pil=False, i... method get_images (line 152) | def get_images(self, return_pil=False): method to_gradio_chatbot (line 162) | def to_gradio_chatbot(self): method copy (line 180) | def copy(self): method dict (line 191) | def dict(self): FILE: train/dpo/llava/conversation_new.py class SeparatorStyle (line 6) | class SeparatorStyle(Enum): class Conversation (line 13) | class Conversation: method get_prompt (line 26) | def get_prompt(self): method append_message (line 51) | def append_message(self, role, message): method get_images (line 54) | def get_images(self, return_pil=False): method to_gradio_chatbot (line 103) | def to_gradio_chatbot(self): method copy (line 133) | def copy(self): method dict (line 143) | def dict(self): FILE: train/dpo/llava/eval/eval_gpt_review.py function get_eval (line 13) | def get_eval(content: str, max_tokens: int): function parse_score (line 39) | def parse_score(review): FILE: train/dpo/llava/eval/eval_gpt_review_bench.py function get_eval (line 11) | def get_eval(content: str, max_tokens: int): function parse_score (line 36) | def parse_score(review): FILE: train/dpo/llava/eval/eval_gpt_review_visual.py function get_eval (line 11) | def get_eval(content: str, max_tokens: int): function parse_score (line 36) | def parse_score(review): FILE: train/dpo/llava/eval/eval_pope.py function eval_pope (line 5) | def eval_pope(answers, label_file): FILE: train/dpo/llava/eval/eval_science_qa.py function get_args (line 8) | def get_args(): function convert_caps (line 19) | def convert_caps(results): function get_pred_idx (line 28) | def get_pred_idx(prediction, choices, options): FILE: train/dpo/llava/eval/eval_science_qa_gpt4.py function get_args (line 9) | def get_args(): function convert_caps (line 19) | def convert_caps(results): function get_pred_idx (line 28) | def get_pred_idx(prediction, choices, options): FILE: train/dpo/llava/eval/eval_science_qa_gpt4_requery.py function get_args (line 9) | def get_args(): function convert_caps (line 21) | def convert_caps(results): function get_pred_idx (line 30) | def get_pred_idx(prediction, choices, options): FILE: train/dpo/llava/eval/eval_textvqa.py function get_args (line 9) | def get_args(): function prompt_processor (line 17) | def prompt_processor(prompt): function eval_single (line 35) | def eval_single(annotation_file, result_file): FILE: train/dpo/llava/eval/generate_webpage_data_from_table.py function read_jsonl (line 10) | def read_jsonl(path: str, key: str=None): function trim_hanging_lines (line 23) | def trim_hanging_lines(s: str, n: int) -> str: FILE: train/dpo/llava/eval/m4c_evaluator.py class EvalAIAnswerProcessor (line 7) | class EvalAIAnswerProcessor: method __init__ (line 178) | def __init__(self, *args, **kwargs): method word_tokenize (line 181) | def word_tokenize(self, word): method process_punctuation (line 186) | def process_punctuation(self, in_text): method process_digit_article (line 198) | def process_digit_article(self, in_text): method __call__ (line 213) | def __call__(self, item): class TextVQAAccuracyEvaluator (line 221) | class TextVQAAccuracyEvaluator: method __init__ (line 222) | def __init__(self): method _compute_answer_scores (line 225) | def _compute_answer_scores(self, raw_answers): method eval_pred_list (line 248) | def eval_pred_list(self, pred_list): class STVQAAccuracyEvaluator (line 260) | class STVQAAccuracyEvaluator: method __init__ (line 261) | def __init__(self): method eval_pred_list (line 264) | def eval_pred_list(self, pred_list): class STVQAANLSEvaluator (line 276) | class STVQAANLSEvaluator: method __init__ (line 277) | def __init__(self): method get_anls (line 282) | def get_anls(self, s1, s2): method eval_pred_list (line 289) | def eval_pred_list(self, pred_list): class TextCapsBleu4Evaluator (line 301) | class TextCapsBleu4Evaluator: method __init__ (line 302) | def __init__(self): method eval_pred_list (line 321) | def eval_pred_list(self, pred_list): FILE: train/dpo/llava/eval/model_qa.py function eval_model (line 14) | def eval_model(model_name, questions_file, answers_file): FILE: train/dpo/llava/eval/model_vqa.py function split_list (line 18) | def split_list(lst, n): function get_chunk (line 24) | def get_chunk(lst, n, k): function eval_model (line 29) | def eval_model(args): FILE: train/dpo/llava/eval/model_vqa_loader.py function split_list (line 19) | def split_list(lst, n): function get_chunk (line 25) | def get_chunk(lst, n, k): class CustomDataset (line 31) | class CustomDataset(Dataset): method __init__ (line 32) | def __init__(self, questions, image_folder, tokenizer, image_processor... method __getitem__ (line 39) | def __getitem__(self, index): method __len__ (line 60) | def __len__(self): function collate_fn (line 64) | def collate_fn(batch): function create_data_loader (line 72) | def create_data_loader(questions, image_folder, tokenizer, image_process... function eval_model (line 79) | def eval_model(args): FILE: train/dpo/llava/eval/model_vqa_mmbench.py function split_list (line 22) | def split_list(lst, n): function get_chunk (line 28) | def get_chunk(lst, n, k): function is_none (line 33) | def is_none(value): function get_options (line 44) | def get_options(row, options): function eval_model (line 54) | def eval_model(args): FILE: train/dpo/llava/eval/model_vqa_science.py function split_list (line 18) | def split_list(lst, n): function get_chunk (line 24) | def get_chunk(lst, n, k): function eval_model (line 29) | def eval_model(args): FILE: train/dpo/llava/eval/qa_baseline_gpt35.py function get_answer (line 16) | def get_answer(question_id: int, question: str, max_tokens: int): FILE: train/dpo/llava/eval/run_llava.py function image_parser (line 28) | def image_parser(args): function load_image (line 33) | def load_image(image_file): function load_images (line 42) | def load_images(image_files): function eval_model (line 50) | def eval_model(args): FILE: train/dpo/llava/eval/summarize_gpt_review.py function parse_args (line 9) | def parse_args(): FILE: train/dpo/llava/mm_utils.py function select_best_resolution (line 12) | def select_best_resolution(original_size, possible_resolutions): function resize_and_pad_image (line 42) | def resize_and_pad_image(image, target_resolution): function divide_to_patches (line 77) | def divide_to_patches(image, patch_size): function get_anyres_image_grid_shape (line 99) | def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size): function process_anyres_image (line 119) | def process_anyres_image(image, processor, grid_pinpoints): function load_image_from_base64 (line 148) | def load_image_from_base64(image): function expand2square (line 152) | def expand2square(pil_img, background_color): function process_images (line 166) | def process_images(images, image_processor, model_cfg): function tokenizer_image_token (line 185) | def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOK... function get_model_name_from_path (line 207) | def get_model_name_from_path(model_path): class KeywordsStoppingCriteria (line 215) | class KeywordsStoppingCriteria(StoppingCriteria): method __init__ (line 216) | def __init__(self, keywords, tokenizer, input_ids): method call_for_batch (line 230) | def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.F... method __call__ (line 243) | def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTe... FILE: train/dpo/llava/model/apply_delta.py function apply_delta (line 13) | def apply_delta(base_model_path, target_model_path, delta_path): FILE: train/dpo/llava/model/builder.py function load_pretrained_model (line 26) | def load_pretrained_model(model_path, model_base, model_name, load_8bit=... FILE: train/dpo/llava/model/consolidate.py function consolidate_ckpt (line 13) | def consolidate_ckpt(src_path, dst_path): FILE: train/dpo/llava/model/language_model/llava_llama.py class LlavaConfig (line 30) | class LlavaConfig(LlamaConfig): class LlavaLlamaModel (line 34) | class LlavaLlamaModel(LlavaMetaModel, LlamaModel): method __init__ (line 37) | def __init__(self, config: LlamaConfig): class LlavaLlamaForCausalLM (line 41) | class LlavaLlamaForCausalLM(LlamaForCausalLM, LlavaMetaForCausalLM): method __init__ (line 44) | def __init__(self, config): method get_model (line 54) | def get_model(self): method forward (line 57) | def forward( method generate (line 105) | def generate( method prepare_inputs_for_generation (line 144) | def prepare_inputs_for_generation(self, input_ids, past_key_values=None, FILE: train/dpo/llava/model/language_model/llava_mistral.py class LlavaMistralConfig (line 31) | class LlavaMistralConfig(MistralConfig): class LlavaMistralModel (line 35) | class LlavaMistralModel(LlavaMetaModel, MistralModel): method __init__ (line 38) | def __init__(self, config: MistralConfig): class LlavaMistralForCausalLM (line 42) | class LlavaMistralForCausalLM(MistralForCausalLM, LlavaMetaForCausalLM): method __init__ (line 45) | def __init__(self, config): method get_model (line 54) | def get_model(self): method forward (line 57) | def forward( method generate (line 105) | def generate( method prepare_inputs_for_generation (line 144) | def prepare_inputs_for_generation(self, input_ids, past_key_values=None, FILE: train/dpo/llava/model/language_model/llava_mpt.py class LlavaMptConfig (line 25) | class LlavaMptConfig(MptConfig): class LlavaMptModel (line 29) | class LlavaMptModel(LlavaMetaModel, MptModel): method __init__ (line 32) | def __init__(self, config: MptConfig): method embed_tokens (line 36) | def embed_tokens(self, x): class LlavaMptForCausalLM (line 40) | class LlavaMptForCausalLM(MptForCausalLM, LlavaMetaForCausalLM): method __init__ (line 44) | def __init__(self, config): method get_model (line 53) | def get_model(self): method _set_gradient_checkpointing (line 56) | def _set_gradient_checkpointing(self, module, value=False): method forward (line 60) | def forward( method prepare_inputs_for_generation (line 87) | def prepare_inputs_for_generation(self, input_ids, past_key_values=Non... FILE: train/dpo/llava/model/llava_arch.py class LlavaMetaModel (line 29) | class LlavaMetaModel: method __init__ (line 31) | def __init__(self, config): method get_vision_tower (line 43) | def get_vision_tower(self): method initialize_vision_modules (line 49) | def initialize_vision_modules(self, model_args, fsdp=None): function unpad_image (line 100) | def unpad_image(tensor, original_size): class LlavaMetaForCausalLM (line 131) | class LlavaMetaForCausalLM(ABC): method get_model (line 134) | def get_model(self): method get_vision_tower (line 137) | def get_vision_tower(self): method encode_images (line 140) | def encode_images(self, images): method prepare_inputs_labels_for_multimodal (line 145) | def prepare_inputs_labels_for_multimodal( method initialize_vision_tokenizer (line 326) | def initialize_vision_tokenizer(self, model_args, tokenizer): FILE: train/dpo/llava/model/make_delta.py function make_delta (line 13) | def make_delta(base_model_path, target_model_path, delta_path, hub_repo_... FILE: train/dpo/llava/model/multimodal_encoder/builder.py function build_vision_tower (line 5) | def build_vision_tower(vision_tower_cfg, **kwargs): FILE: train/dpo/llava/model/multimodal_encoder/clip_encoder.py class CLIPVisionTower (line 7) | class CLIPVisionTower(nn.Module): method __init__ (line 8) | def __init__(self, vision_tower, args, delay_load=False): method load_model (line 24) | def load_model(self, device_map=None): method feature_select (line 35) | def feature_select(self, image_forward_outs): method forward (line 46) | def forward(self, images): method dummy_feature (line 60) | def dummy_feature(self): method dtype (line 64) | def dtype(self): method device (line 68) | def device(self): method config (line 72) | def config(self): method hidden_size (line 79) | def hidden_size(self): method num_patches_per_side (line 83) | def num_patches_per_side(self): method num_patches (line 87) | def num_patches(self): FILE: train/dpo/llava/model/multimodal_projector/builder.py class IdentityMap (line 6) | class IdentityMap(nn.Module): method __init__ (line 7) | def __init__(self): method forward (line 10) | def forward(self, x, *args, **kwargs): method config (line 14) | def config(self): class SimpleResBlock (line 18) | class SimpleResBlock(nn.Module): method __init__ (line 19) | def __init__(self, channels): method forward (line 28) | def forward(self, x): function build_vision_projector (line 33) | def build_vision_projector(config, delay_load=False, **kwargs): FILE: train/dpo/llava/model/utils.py function auto_upgrade (line 4) | def auto_upgrade(config): FILE: train/dpo/llava/serve/cli.py function load_image (line 18) | def load_image(image_file): function main (line 27) | def main(args): FILE: train/dpo/llava/serve/controller.py class DispatchMethod (line 28) | class DispatchMethod(Enum): method from_str (line 33) | def from_str(cls, name): class WorkerInfo (line 43) | class WorkerInfo: function heart_beat_controller (line 51) | def heart_beat_controller(controller): class Controller (line 57) | class Controller: method __init__ (line 58) | def __init__(self, dispatch_method: str): method register_worker (line 69) | def register_worker(self, worker_name: str, check_heart_beat: bool, method get_worker_status (line 88) | def get_worker_status(self, worker_name: str): method remove_worker (line 101) | def remove_worker(self, worker_name: str): method refresh_all_workers (line 104) | def refresh_all_workers(self): method list_models (line 112) | def list_models(self): method get_worker_address (line 120) | def get_worker_address(self, model_name: str): method receive_heart_beat (line 173) | def receive_heart_beat(self, worker_name: str, queue_length: int): method remove_stable_workers_by_expiration (line 183) | def remove_stable_workers_by_expiration(self): method worker_api_generate_stream (line 193) | def worker_api_generate_stream(self, params): method worker_api_get_status (line 220) | def worker_api_get_status(self): function register_worker (line 243) | async def register_worker(request: Request): function refresh_all_workers (line 251) | async def refresh_all_workers(): function list_models (line 256) | async def list_models(): function get_worker_address (line 262) | async def get_worker_address(request: Request): function receive_heart_beat (line 269) | async def receive_heart_beat(request: Request): function worker_api_generate_stream (line 277) | async def worker_api_generate_stream(request: Request): function worker_api_get_status (line 284) | async def worker_api_get_status(request: Request): FILE: train/dpo/llava/serve/gradio_web_server.py function get_conv_log_filename (line 32) | def get_conv_log_filename(): function get_model_list (line 38) | def get_model_list(): function load_demo (line 58) | def load_demo(url_params, request: gr.Request): function load_demo_refresh_model_list (line 71) | def load_demo_refresh_model_list(request: gr.Request): function vote_last_response (line 82) | def vote_last_response(state, vote_type, model_selector, request: gr.Req... function upvote_last_response (line 94) | def upvote_last_response(state, model_selector, request: gr.Request): function downvote_last_response (line 100) | def downvote_last_response(state, model_selector, request: gr.Request): function flag_last_response (line 106) | def flag_last_response(state, model_selector, request: gr.Request): function regenerate (line 112) | def regenerate(state, image_process_mode, request: gr.Request): function clear_history (line 122) | def clear_history(request: gr.Request): function add_text (line 128) | def add_text(state, text, image, image_process_mode, request: gr.Request): function http_bot (line 154) | def http_bot(state, model_selector, temperature, top_p, max_new_tokens, ... function build_demo (line 315) | def build_demo(embed_mode, cur_dir=None, concurrency_count=10): FILE: train/dpo/llava/serve/model_worker.py function heart_beat_worker (line 37) | def heart_beat_worker(controller): class ModelWorker (line 44) | class ModelWorker: method __init__ (line 45) | def __init__(self, controller_addr, worker_addr, method register_to_controller (line 75) | def register_to_controller(self): method send_heart_beat (line 87) | def send_heart_beat(self): method get_queue_length (line 108) | def get_queue_length(self): method get_status (line 115) | def get_status(self): method generate_stream (line 123) | def generate_stream(self, params): method generate_stream_gate (line 195) | def generate_stream_gate(self, params): function release_model_semaphore (line 225) | def release_model_semaphore(fn=None): function generate_stream (line 232) | async def generate_stream(request: Request): function get_status (line 248) | async def get_status(request: Request): FILE: train/dpo/llava/serve/sglang_worker.py function heart_beat_worker (line 38) | def heart_beat_worker(controller): function pipeline (line 45) | def pipeline(s, prompt, max_tokens): class ModelWorker (line 54) | class ModelWorker: method __init__ (line 55) | def __init__(self, controller_addr, worker_addr, sgl_endpoint, method register_to_controller (line 85) | def register_to_controller(self): method send_heart_beat (line 97) | def send_heart_beat(self): method get_queue_length (line 118) | def get_queue_length(self): method get_status (line 125) | def get_status(self): method generate_stream (line 132) | async def generate_stream(self, params): method generate_stream_gate (line 172) | async def generate_stream_gate(self, params): function release_model_semaphore (line 195) | def release_model_semaphore(fn=None): function generate_stream (line 202) | async def generate_stream(request: Request): function get_status (line 218) | async def get_status(request: Request): FILE: train/dpo/llava/serve/test_message.py function main (line 9) | def main(): FILE: train/dpo/llava/train/llama_flash_attn_monkey_patch.py function forward (line 16) | def forward( function _prepare_decoder_attention_mask (line 98) | def _prepare_decoder_attention_mask( function replace_llama_attn_with_flash_attn (line 105) | def replace_llama_attn_with_flash_attn(): FILE: train/dpo/llava/train/llama_xformers_attn_monkey_patch.py function replace_llama_attn_with_xformers_attn (line 19) | def replace_llama_attn_with_xformers_attn(): function xformers_forward (line 23) | def xformers_forward( FILE: train/dpo/llava/train/llava_trainer.py function maybe_zero_3 (line 18) | def maybe_zero_3(param, ignore_status=False, name=None): function get_mm_adapter_state_maybe_zero_3 (line 32) | def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match): function split_to_even_chunks (line 38) | def split_to_even_chunks(indices, lengths, num_chunks): function get_modality_length_grouped_indices (line 60) | def get_modality_length_grouped_indices(lengths, batch_size, world_size,... function get_length_grouped_indices (line 88) | def get_length_grouped_indices(lengths, batch_size, world_size, generato... class LengthGroupedSampler (line 99) | class LengthGroupedSampler(Sampler): method __init__ (line 105) | def __init__( method __len__ (line 122) | def __len__(self): method __iter__ (line 125) | def __iter__(self): class LLaVATrainer (line 133) | class LLaVATrainer(DPOTrainer): method _get_train_sampler (line 135) | def _get_train_sampler(self) -> Optional[torch.utils.data.Sampler]: method create_optimizer (line 150) | def create_optimizer(self): method _save_checkpoint (line 239) | def _save_checkpoint(self, model, trial, metrics=None): method _save (line 260) | def _save(self, output_dir: Optional[str] = None, state_dict=None): FILE: train/dpo/llava/train/train.py function rank0_print (line 44) | def rank0_print(*args): class ModelArguments (line 54) | class ModelArguments: class DataArguments (line 70) | class DataArguments: class TrainingArguments (line 80) | class TrainingArguments(transformers.TrainingArguments): function maybe_zero_3 (line 115) | def maybe_zero_3(param, ignore_status=False, name=None): function get_peft_state_maybe_zero_3 (line 130) | def get_peft_state_maybe_zero_3(named_params, bias): function get_peft_state_non_lora_maybe_zero_3 (line 155) | def get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only... function get_mm_adapter_state_maybe_zero_3 (line 163) | def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match): function find_all_linear_names (line 169) | def find_all_linear_names(model): function safe_save_model_for_hf_trainer (line 185) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, function smart_tokenizer_and_embedding_resize (line 224) | def smart_tokenizer_and_embedding_resize( function _tokenize_fn (line 249) | def _tokenize_fn(strings: Sequence[str], function _mask_targets (line 276) | def _mask_targets(target, tokenized_lens, speakers): function _add_speaker_and_signal (line 287) | def _add_speaker_and_signal(header, source, get_conversation=True): function preprocess_multimodal (line 308) | def preprocess_multimodal( function preprocess_llama_2 (line 332) | def preprocess_llama_2( function preprocess_v1 (line 414) | def preprocess_v1( function preprocess_mpt (line 500) | def preprocess_mpt( function preprocess_plain (line 588) | def preprocess_plain( function preprocess (line 610) | def preprocess( class LazySupervisedDataset (line 658) | class LazySupervisedDataset(Dataset): method __init__ (line 661) | def __init__(self, data_path: str, method __len__ (line 672) | def __len__(self): method lengths (line 676) | def lengths(self): method modality_lengths (line 684) | def modality_lengths(self): method __getitem__ (line 692) | def __getitem__(self, i) -> Dict[str, torch.Tensor]: class DataCollatorForSupervisedDataset (line 743) | class DataCollatorForSupervisedDataset(object): method __call__ (line 748) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: function make_supervised_data_module (line 776) | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokeni... function train (line 788) | def train(attn_implementation=None): FILE: train/dpo/llava/train/train_dpo.py function rank0_print (line 46) | def rank0_print(*args): function pil_to_tensor (line 50) | def pil_to_tensor(image): function tensor_to_pil (line 54) | def tensor_to_pil(tensor): function add_diffusion_noise (line 58) | def add_diffusion_noise(image_tensor, noise_step): class ModelArguments (line 86) | class ModelArguments: class DataArguments (line 102) | class DataArguments: class TrainingArguments (line 112) | class TrainingArguments(transformers.TrainingArguments): function maybe_zero_3 (line 147) | def maybe_zero_3(param, ignore_status=False, name=None): function get_peft_state_maybe_zero_3 (line 162) | def get_peft_state_maybe_zero_3(named_params, bias): function get_peft_state_non_lora_maybe_zero_3 (line 187) | def get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only... function get_mm_adapter_state_maybe_zero_3 (line 195) | def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match): function find_all_linear_names (line 201) | def find_all_linear_names(model): function safe_save_model_for_hf_trainer (line 217) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, function smart_tokenizer_and_embedding_resize (line 256) | def smart_tokenizer_and_embedding_resize( function _tokenize_fn (line 281) | def _tokenize_fn(strings: Sequence[str], function _mask_targets (line 308) | def _mask_targets(target, tokenized_lens, speakers): function _add_speaker_and_signal (line 319) | def _add_speaker_and_signal(header, source, get_conversation=True): function preprocess_multimodal (line 340) | def preprocess_multimodal( function preprocess_llama_2 (line 364) | def preprocess_llama_2( function preprocess_v1 (line 446) | def preprocess_v1( function preprocess_plain (line 544) | def preprocess_plain( function preprocess (line 566) | def preprocess( class LazySupervisedDataset (line 620) | class LazySupervisedDataset(Dataset): method __init__ (line 623) | def __init__(self, data_path: str, method __len__ (line 634) | def __len__(self): method lengths (line 638) | def lengths(self): method modality_lengths (line 646) | def modality_lengths(self): method __getitem__ (line 654) | def __getitem__(self, i) -> Dict[str, torch.Tensor]: class DataCollatorForSupervisedDataset (line 732) | class DataCollatorForSupervisedDataset(object): method __call__ (line 737) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: function make_supervised_data_module (line 794) | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokeni... function train (line 806) | def train(): FILE: train/dpo/llava/train/train_dpo_inherent.py function rank0_print (line 45) | def rank0_print(*args): function pil_to_tensor (line 49) | def pil_to_tensor(image): function tensor_to_pil (line 53) | def tensor_to_pil(tensor): function add_diffusion_noise (line 57) | def add_diffusion_noise(image_tensor, noise_step): class ModelArguments (line 85) | class ModelArguments: class DataArguments (line 100) | class DataArguments: class TrainingArguments (line 110) | class TrainingArguments(transformers.TrainingArguments): function maybe_zero_3 (line 145) | def maybe_zero_3(param, ignore_status=False, name=None): function get_peft_state_maybe_zero_3 (line 160) | def get_peft_state_maybe_zero_3(named_params, bias): function get_peft_state_non_lora_maybe_zero_3 (line 185) | def get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only... function get_mm_adapter_state_maybe_zero_3 (line 193) | def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match): function find_all_linear_names (line 199) | def find_all_linear_names(model): function safe_save_model_for_hf_trainer (line 215) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, function smart_tokenizer_and_embedding_resize (line 254) | def smart_tokenizer_and_embedding_resize( function _tokenize_fn (line 279) | def _tokenize_fn(strings: Sequence[str], function _mask_targets (line 306) | def _mask_targets(target, tokenized_lens, speakers): function _add_speaker_and_signal (line 317) | def _add_speaker_and_signal(header, source, get_conversation=True): function preprocess_multimodal (line 338) | def preprocess_multimodal( function preprocess_llama_2 (line 362) | def preprocess_llama_2( function preprocess_v1 (line 444) | def preprocess_v1( function preprocess_plain (line 542) | def preprocess_plain( function preprocess (line 564) | def preprocess( class LazySupervisedDataset (line 618) | class LazySupervisedDataset(Dataset): method __init__ (line 621) | def __init__(self, data_path: str, method __len__ (line 632) | def __len__(self): method lengths (line 636) | def lengths(self): method modality_lengths (line 644) | def modality_lengths(self): method __getitem__ (line 652) | def __getitem__(self, i) -> Dict[str, torch.Tensor]: class DataCollatorForSupervisedDataset (line 736) | class DataCollatorForSupervisedDataset(object): method __call__ (line 741) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: function make_supervised_data_module (line 804) | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokeni... function train (line 816) | def train(): FILE: train/dpo/llava/utils.py function build_logger (line 17) | def build_logger(logger_name, logger_filename): class StreamToLogger (line 60) | class StreamToLogger(object): method __init__ (line 64) | def __init__(self, logger, log_level=logging.INFO): method __getattr__ (line 70) | def __getattr__(self, attr): method write (line 73) | def write(self, buf): method flush (line 87) | def flush(self): function disable_torch_init (line 93) | def disable_torch_init(): function violates_moderation (line 102) | def violates_moderation(text): function pretty_print_semaphore (line 123) | def pretty_print_semaphore(semaphore): FILE: train/dpo/llava_trainer_2stages.py function maybe_zero_3 (line 18) | def maybe_zero_3(param, ignore_status=False, name=None): function get_mm_adapter_state_maybe_zero_3 (line 32) | def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match): function split_to_even_chunks (line 38) | def split_to_even_chunks(indices, lengths, num_chunks): function get_modality_length_grouped_indices (line 60) | def get_modality_length_grouped_indices(lengths, batch_size, world_size,... function get_length_grouped_indices (line 88) | def get_length_grouped_indices(lengths, batch_size, world_size, generato... class LengthGroupedSampler (line 99) | class LengthGroupedSampler(Sampler): method __init__ (line 105) | def __init__( method __len__ (line 122) | def __len__(self): method __iter__ (line 125) | def __iter__(self): class LLaVATrainer (line 133) | class LLaVATrainer(DPOTrainer): method _get_train_sampler (line 135) | def _get_train_sampler(self) -> Optional[torch.utils.data.Sampler]: method create_optimizer (line 150) | def create_optimizer(self): method _save_checkpoint (line 239) | def _save_checkpoint(self, model, trial, metrics=None): method _save (line 260) | def _save(self, output_dir: Optional[str] = None, state_dict=None): FILE: train/dpo/povid_infer.py function split_list (line 23) | def split_list(lst, n): function get_chunk (line 29) | def get_chunk(lst, n, k): function eval_model (line 34) | def eval_model(args): FILE: train/dpo/predict.py function download_json (line 54) | def download_json(url: str, dest: Path): function download_weights (line 62) | def download_weights(baseurl: str, basedest: str, files: list[str]): class Predictor (line 78) | class Predictor(BasePredictor): method setup (line 79) | def setup(self) -> None: method predict (line 87) | def predict( function load_image (line 148) | def load_image(image_file): FILE: train/dpo/scripts/convert_mmbench_for_submission.py function get_args (line 6) | def get_args(): FILE: train/dpo/scripts/convert_seed_for_submission.py function get_args (line 6) | def get_args(): function eval_single (line 14) | def eval_single(result_file, eval_only_type=None): FILE: train/dpo/scripts/convert_sqa_to_llava.py function convert_to_llava (line 8) | def convert_to_llava(base_dir, split, prompt_format="QCM-LEA"): function convert_to_jsonl (line 49) | def convert_to_jsonl(base_dir, split, prompt_format="QCM-LEPA"): function main (line 83) | def main(task, **kwargs): FILE: train/dpo/scripts/convert_sqa_to_llava_base_prompt.py function get_question_text (line 1) | def get_question_text(problem): function get_context_text (line 6) | def get_context_text(problem, use_caption): function get_choice_text (line 15) | def get_choice_text(probelm, options): function get_answer (line 25) | def get_answer(problem, options): function get_lecture_text (line 29) | def get_lecture_text(problem): function get_solution_text (line 35) | def get_solution_text(problem): function create_one_example_chatbot (line 41) | def create_one_example_chatbot(format, question, context, choice, answer... function create_one_example (line 106) | def create_one_example(format, question, context, choice, answer, lectur... function create_one_example_gpt4 (line 162) | def create_one_example_gpt4(format, question, context, choice, answer, l... function build_prompt_chatbot (line 221) | def build_prompt_chatbot(problems, shot_qids, prompt_format, use_caption... function build_prompt (line 244) | def build_prompt(problems, shot_qids, test_qid, args): function build_prompt_gpt4 (line 291) | def build_prompt_gpt4(problems, shot_qids, test_qid, args): FILE: train/dpo/scripts/convert_vizwiz_for_submission.py function parse_args (line 8) | def parse_args(): FILE: train/dpo/scripts/convert_vqav2_for_submission.py function parse_args (line 8) | def parse_args(): FILE: train/dpo/scripts/extract_mm_projector.py function parse_args (line 15) | def parse_args(): FILE: train/dpo/scripts/merge_lora_weights.py function merge_lora (line 6) | def merge_lora(args): FILE: train/dpo/tool/dpo_trainer.py class DPOTrainer (line 60) | class DPOTrainer(Trainer): method __init__ (line 123) | def __init__( method _prepare_deepspeed (line 354) | def _prepare_deepspeed(self, model: PreTrainedModelWrapper): method concatenated_inputs (line 385) | def concatenated_inputs(self, batch: Dict[str, Union[List, torch.LongT... method dpo_loss (line 424) | def dpo_loss( method _get_batch_logps (line 467) | def _get_batch_logps( method _get_noisy_batch_logps (line 501) | def _get_noisy_batch_logps( method concatenated_forward (line 550) | def concatenated_forward( method get_batch_metrics (line 629) | def get_batch_metrics( method compute_loss (line 681) | def compute_loss( method get_batch_samples (line 702) | def get_batch_samples(self, model, batch: Dict[str, torch.LongTensor])... method prediction_step (line 739) | def prediction_step( method store_metrics (line 778) | def store_metrics(self, metrics: Dict[str, float], train_eval: Literal... method evaluation_loop (line 782) | def evaluation_loop( method log (line 832) | def log(self, logs: Dict[str, float]) -> None: FILE: train/dpo/tool/dpo_trainer_inherent.py class DPOTrainer (line 56) | class DPOTrainer(Trainer): method __init__ (line 119) | def __init__( method _prepare_deepspeed (line 350) | def _prepare_deepspeed(self, model: PreTrainedModelWrapper): method concatenated_inputs (line 381) | def concatenated_inputs(self, batch: Dict[str, Union[List, torch.LongT... method dpo_loss (line 420) | def dpo_loss( method _get_batch_logps (line 463) | def _get_batch_logps( method _get_noisy_batch_logps (line 497) | def _get_noisy_batch_logps( method concatenated_multi_forward (line 546) | def concatenated_multi_forward( method get_batch_metrics (line 641) | def get_batch_metrics( method compute_loss (line 704) | def compute_loss( method get_batch_samples (line 725) | def get_batch_samples(self, model, batch: Dict[str, torch.LongTensor])... method prediction_step (line 762) | def prediction_step( method store_metrics (line 801) | def store_metrics(self, metrics: Dict[str, float], train_eval: Literal... method evaluation_loop (line 805) | def evaluation_loop( method log (line 855) | def log(self, logs: Dict[str, float]) -> None: FILE: train/dpo/train_dpo_2stages.py function rank0_print (line 72) | def rank0_print(*args): function pil_to_tensor (line 77) | def pil_to_tensor(image): function tensor_to_pil (line 82) | def tensor_to_pil(tensor): function add_gaussian_noise (line 86) | def add_gaussian_noise(image, mean=0.0, stddev=0.1): function generate_gaussian_noise (line 93) | def generate_gaussian_noise(image, mean=0.0, stddev=0.1): function add_diffusion_noise (line 99) | def add_diffusion_noise(image_tensor, noise_step): class ModelArguments (line 130) | class ModelArguments: class DataArguments (line 149) | class DataArguments: class TrainingArguments (line 160) | class TrainingArguments(transformers.TrainingArguments): function maybe_zero_3 (line 195) | def maybe_zero_3(param, ignore_status=False, name=None): function get_peft_state_maybe_zero_3 (line 213) | def get_peft_state_maybe_zero_3(named_params, bias): function get_peft_state_non_lora_maybe_zero_3 (line 238) | def get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only... function get_mm_adapter_state_maybe_zero_3 (line 248) | def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match): function find_all_linear_names (line 260) | def find_all_linear_names(model): function safe_save_model_for_hf_trainer (line 276) | def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, output... function smart_tokenizer_and_embedding_resize (line 318) | def smart_tokenizer_and_embedding_resize( function _tokenize_fn (line 345) | def _tokenize_fn( function _mask_targets (line 372) | def _mask_targets(target, tokenized_lens, speakers): function _add_speaker_and_signal (line 383) | def _add_speaker_and_signal(header, source, get_conversation=True): function preprocess_multimodal (line 405) | def preprocess_multimodal(sources: Sequence[str], data_args: DataArgumen... function preprocess_llama_2 (line 435) | def preprocess_llama_2( function preprocess_v1 (line 521) | def preprocess_v1( function preprocess_plain (line 634) | def preprocess_plain( function preprocess (line 663) | def preprocess( class LazySupervisedDataset (line 730) | class LazySupervisedDataset(Dataset): method __init__ (line 733) | def __init__( method __len__ (line 747) | def __len__(self): method lengths (line 751) | def lengths(self): method modality_lengths (line 762) | def modality_lengths(self): method __getitem__ (line 772) | def __getitem__(self, i) -> Dict[str, torch.Tensor]: class DataCollatorForSupervisedDataset (line 895) | class DataCollatorForSupervisedDataset(object): method __call__ (line 900) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: function make_supervised_data_module (line 967) | def make_supervised_data_module( function train (line 980) | def train(): FILE: train/open_clip/setup.py function _read_reqs (line 13) | def _read_reqs(relpath): FILE: train/open_clip/src/open_clip/big_vision.py function load_big_vision_weights (line 9) | def load_big_vision_weights(model: CustomTextCLIP, checkpoint_path: str): FILE: train/open_clip/src/open_clip/coca_model.py class MultimodalCfg (line 45) | class MultimodalCfg(CLIPTextCfg): function _build_text_decoder_tower (line 53) | def _build_text_decoder_tower( class CoCa (line 79) | class CoCa(nn.Module): method __init__ (line 80) | def __init__( method set_grad_checkpointing (line 134) | def set_grad_checkpointing(self, enable: bool = True): method _encode_image (line 139) | def _encode_image(self, images, normalize: bool = True): method _encode_text (line 144) | def _encode_text(self, text, normalize: bool = True): method encode_image (line 149) | def encode_image(self, images, normalize: bool = True): method encode_text (line 153) | def encode_text(self, text, normalize: bool = True): method forward (line 157) | def forward( method generate (line 187) | def generate( method _generate_beamsearch (line 310) | def _generate_beamsearch( function prepare_inputs_for_generation (line 459) | def prepare_inputs_for_generation(input_ids, image_inputs, past=None, **... FILE: train/open_clip/src/open_clip/factory.py function _natural_key (line 28) | def _natural_key(string_): function _rescan_model_configs (line 32) | def _rescan_model_configs(): function list_models (line 56) | def list_models(): function add_model_config (line 61) | def add_model_config(path): function get_model_config (line 69) | def get_model_config(model_name): function _get_hf_config (line 76) | def _get_hf_config(model_id, cache_dir=None): function get_tokenizer (line 83) | def get_tokenizer( function load_state_dict (line 127) | def load_state_dict(checkpoint_path: str, map_location='cpu'): function load_checkpoint (line 142) | def load_checkpoint(model, checkpoint_path, strict=True): function create_model (line 165) | def create_model( function create_loss (line 324) | def create_loss(args): function create_model_and_transforms (line 361) | def create_model_and_transforms( function create_model_from_pretrained (line 418) | def create_model_from_pretrained( FILE: train/open_clip/src/open_clip/hf_model.py class BaseModelOutput (line 20) | class BaseModelOutput: class PretrainedConfig (line 24) | class PretrainedConfig: function _camel2snake (line 31) | def _camel2snake(s): function register_pooler (line 39) | def register_pooler(cls): class MeanPooler (line 46) | class MeanPooler(nn.Module): method forward (line 49) | def forward(self, x: BaseModelOutput, attention_mask: TensorType): class MaxPooler (line 55) | class MaxPooler(nn.Module): method forward (line 58) | def forward(self, x: BaseModelOutput, attention_mask: TensorType): class ClsPooler (line 64) | class ClsPooler(nn.Module): method __init__ (line 67) | def __init__(self, use_pooler_output=True): method forward (line 72) | def forward(self, x: BaseModelOutput, attention_mask: TensorType): class ClsLastHiddenStatePooler (line 83) | class ClsLastHiddenStatePooler(nn.Module): method __init__ (line 88) | def __init__(self): method forward (line 92) | def forward(self, x: BaseModelOutput, attention_mask: TensorType): class HFTextEncoder (line 96) | class HFTextEncoder(nn.Module): method __init__ (line 100) | def __init__( method forward (line 154) | def forward(self, x: TensorType): method lock (line 171) | def lock(self, unlocked_layers: int = 0, freeze_layer_norm: bool = True): method set_grad_checkpointing (line 189) | def set_grad_checkpointing(self, enable=True): method init_parameters (line 192) | def init_parameters(self): FILE: train/open_clip/src/open_clip/loss.py function gather_features (line 19) | def gather_features( class ClipLoss (line 66) | class ClipLoss(nn.Module): method __init__ (line 68) | def __init__( method get_ground_truth (line 89) | def get_ground_truth(self, device, num_logits) -> torch.Tensor: method get_logits (line 102) | def get_logits(self, image_features, text_features, logit_scale): method forward (line 120) | def forward(self, image_features, text_features, logit_scale, output_d... class CoCaLoss (line 134) | class CoCaLoss(ClipLoss): method __init__ (line 135) | def __init__( method forward (line 160) | def forward(self, image_features, text_features, logits, labels, logit... class DistillClipLoss (line 180) | class DistillClipLoss(ClipLoss): method dist_loss (line 182) | def dist_loss(self, teacher_logits, student_logits): method forward (line 185) | def forward( function neighbour_exchange (line 219) | def neighbour_exchange(from_rank, to_rank, tensor, group=None): function neighbour_exchange_bidir (line 239) | def neighbour_exchange_bidir(left_rank, right_rank, tensor_to_left, tens... class NeighbourExchange (line 272) | class NeighbourExchange(torch.autograd.Function): method forward (line 274) | def forward(ctx, from_rank, to_rank, group, tensor): method backward (line 281) | def backward(ctx, grad_output): function neighbour_exchange_with_grad (line 285) | def neighbour_exchange_with_grad(from_rank, to_rank, tensor, group=None): class NeighbourExchangeBidir (line 289) | class NeighbourExchangeBidir(torch.autograd.Function): method forward (line 291) | def forward(ctx, left_rank, right_rank, group, tensor_to_left, tensor_... method backward (line 298) | def backward(ctx, *grad_outputs): function neighbour_exchange_bidir_with_grad (line 303) | def neighbour_exchange_bidir_with_grad(left_rank, right_rank, tensor_to_... class SigLipLoss (line 307) | class SigLipLoss(nn.Module): method __init__ (line 317) | def __init__( method get_ground_truth (line 337) | def get_ground_truth(self, device, dtype, num_logits, negative_only=Fa... method get_logits (line 343) | def get_logits(self, image_features, text_features, logit_scale, logit... method _loss (line 349) | def _loss(self, image_features, text_features, logit_scale, logit_bias... method forward (line 360) | def forward(self, image_features, text_features, logit_scale, logit_bi... FILE: train/open_clip/src/open_clip/model.py class CLIPVisionCfg (line 27) | class CLIPVisionCfg: class CLIPTextCfg (line 58) | class CLIPTextCfg: function get_cast_dtype (line 86) | def get_cast_dtype(precision: str): function get_input_dtype (line 95) | def get_input_dtype(precision: str): function _build_vision_tower (line 104) | def _build_vision_tower( function _build_text_tower (line 173) | def _build_text_tower( class CLIP (line 220) | class CLIP(nn.Module): method __init__ (line 223) | def __init__( method lock_image_tower (line 256) | def lock_image_tower(self, unlocked_groups=0, freeze_bn_stats=False): method set_grad_checkpointing (line 261) | def set_grad_checkpointing(self, enable=True): method encode_image (line 265) | def encode_image(self, image, normalize: bool = False): method encode_text (line 269) | def encode_text(self, text, normalize: bool = False): method get_logits (line 288) | def get_logits(self, image, text): method forward (line 297) | def forward( class CustomTextCLIP (line 320) | class CustomTextCLIP(nn.Module): method __init__ (line 323) | def __init__( method lock_image_tower (line 346) | def lock_image_tower(self, unlocked_groups=0, freeze_bn_stats=False): method lock_text_tower (line 350) | def lock_text_tower(self, unlocked_layers: int = 0, freeze_layer_norm:... method set_grad_checkpointing (line 354) | def set_grad_checkpointing(self, enable=True): method encode_image (line 358) | def encode_image(self, image, normalize: bool = False): method encode_text (line 362) | def encode_text(self, text, normalize: bool = False): method get_logits (line 366) | def get_logits(self, image, text): method forward (line 375) | def forward( function convert_weights_to_lp (line 398) | def convert_weights_to_lp(model: nn.Module, dtype=torch.float16): function convert_to_custom_text_state_dict (line 432) | def convert_to_custom_text_state_dict(state_dict: dict): function build_model_from_openai_state_dict (line 450) | def build_model_from_openai_state_dict( function trace_model (line 509) | def trace_model(model, batch_size=256, device=torch.device('cpu')): function resize_pos_embed (line 525) | def resize_pos_embed(state_dict, model, interpolation: str = 'bicubic', ... function resize_text_pos_embed (line 559) | def resize_text_pos_embed(state_dict, model, interpolation: str = 'linea... function get_model_preprocess_cfg (line 591) | def get_model_preprocess_cfg(model): function set_model_preprocess_cfg (line 608) | def set_model_preprocess_cfg(model, preprocess_cfg: Dict[str, Any]): function get_model_tokenize_cfg (line 615) | def get_model_tokenize_cfg(model): FILE: train/open_clip/src/open_clip/modified_resnet.py class Bottleneck (line 10) | class Bottleneck(nn.Module): method __init__ (line 13) | def __init__(self, inplanes, planes, stride=1): method forward (line 42) | def forward(self, x: torch.Tensor): class AttentionPool2d (line 58) | class AttentionPool2d(nn.Module): method __init__ (line 59) | def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, o... method forward (line 68) | def forward(self, x): class ModifiedResNet (line 95) | class ModifiedResNet(nn.Module): method __init__ (line 103) | def __init__(self, layers, output_dim, heads, image_size=224, width=64): method _make_layer (line 132) | def _make_layer(self, planes, blocks, stride=1): method init_parameters (line 141) | def init_parameters(self): method lock (line 154) | def lock(self, unlocked_groups=0, freeze_bn_stats=False): method set_grad_checkpointing (line 162) | def set_grad_checkpointing(self, enable=True): method stem (line 166) | def stem(self, x): method forward (line 173) | def forward(self, x): FILE: train/open_clip/src/open_clip/openai.py function list_openai_models (line 19) | def list_openai_models() -> List[str]: function load_openai_model (line 24) | def load_openai_model( FILE: train/open_clip/src/open_clip/pos_embed.py function get_2d_sincos_pos_embed (line 20) | def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False): function get_2d_sincos_pos_embed_from_grid (line 38) | def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): function get_1d_sincos_pos_embed_from_grid (line 49) | def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): function interpolate_pos_embed (line 75) | def interpolate_pos_embed(model, checkpoint_model): FILE: train/open_clip/src/open_clip/pretrained.py function _pcfg (line 29) | def _pcfg(url='', hf_hub='', **kwargs): function _slpcfg (line 42) | def _slpcfg(url='', hf_hub='', **kwargs): function _apcfg (line 55) | def _apcfg(url='', hf_hub='', **kwargs): function _clean_tag (line 445) | def _clean_tag(tag: str): function list_pretrained (line 450) | def list_pretrained(as_str: bool = False): function list_pretrained_models_by_tag (line 457) | def list_pretrained_models_by_tag(tag: str): function list_pretrained_tags_by_model (line 467) | def list_pretrained_tags_by_model(model: str): function is_pretrained_cfg (line 475) | def is_pretrained_cfg(model: str, tag: str): function get_pretrained_cfg (line 481) | def get_pretrained_cfg(model: str, tag: str): function get_pretrained_url (line 488) | def get_pretrained_url(model: str, tag: str): function download_pretrained_from_url (line 493) | def download_pretrained_from_url( function has_hf_hub (line 539) | def has_hf_hub(necessary=False): function download_pretrained_from_hf (line 547) | def download_pretrained_from_hf( function download_pretrained (line 558) | def download_pretrained( FILE: train/open_clip/src/open_clip/push_to_hf_hub.py function save_config_for_hf (line 40) | def save_config_for_hf( function save_for_hf (line 63) | def save_for_hf( function push_to_hf_hub (line 90) | def push_to_hf_hub( function push_pretrained_to_hf_hub (line 160) | def push_pretrained_to_hf_hub( function generate_readme (line 209) | def generate_readme(model_card: dict, model_name: str): FILE: train/open_clip/src/open_clip/timm_model.py class TimmModel (line 28) | class TimmModel(nn.Module): method __init__ (line 32) | def __init__( method lock (line 110) | def lock(self, unlocked_groups=0, freeze_bn_stats=False): method set_grad_checkpointing (line 143) | def set_grad_checkpointing(self, enable=True): method forward (line 149) | def forward(self, x): FILE: train/open_clip/src/open_clip/tokenizer.py function default_bpe (line 27) | def default_bpe(): function bytes_to_unicode (line 32) | def bytes_to_unicode(): function get_pairs (line 54) | def get_pairs(word): function basic_clean (line 66) | def basic_clean(text): function whitespace_clean (line 72) | def whitespace_clean(text): function _clean_canonicalize (line 78) | def _clean_canonicalize(x): function _clean_lower (line 83) | def _clean_lower(x): function _clean_whitespace (line 88) | def _clean_whitespace(x): function get_clean_fn (line 93) | def get_clean_fn(type: str): function canonicalize_text (line 104) | def canonicalize_text(text, *, keep_punctuation_exact_string=None): class SimpleTokenizer (line 127) | class SimpleTokenizer(object): method __init__ (line 128) | def __init__( method bpe (line 166) | def bpe(self, token): method encode (line 207) | def encode(self, text): method decode (line 215) | def decode(self, tokens): method __call__ (line 220) | def __call__(self, texts: Union[str, List[str]], context_length: Optio... function decode (line 265) | def decode(output_ids: torch.Tensor): function tokenize (line 270) | def tokenize(texts: Union[str, List[str]], context_length: int = DEFAULT... function random_mask_tokenize (line 274) | def random_mask_tokenize( function simple_mask_tokenize (line 303) | def simple_mask_tokenize( function syntax_mask_tokenize (line 325) | def syntax_mask_tokenize( function get_reduction_mask_fn (line 384) | def get_reduction_mask_fn(type: str): class HFTokenizer (line 397) | class HFTokenizer: method __init__ (line 400) | def __init__( method save_pretrained (line 419) | def save_pretrained(self, dest): method __call__ (line 422) | def __call__(self, texts: Union[str, List[str]], context_length: Optio... method set_language (line 449) | def set_language(self, src_lang): class SigLipTokenizer (line 456) | class SigLipTokenizer: method __init__ (line 466) | def __init__( method save_pretrained (line 490) | def save_pretrained(self, dest): method __call__ (line 493) | def __call__(self, texts: Union[str, List[str]], context_length: Optio... FILE: train/open_clip/src/open_clip/transform.py class PreprocessCfg (line 17) | class PreprocessCfg: method __post_init__ (line 26) | def __post_init__(self): method num_channels (line 30) | def num_channels(self): method input_size (line 34) | def input_size(self): function merge_preprocess_dict (line 40) | def merge_preprocess_dict( function merge_preprocess_kwargs (line 57) | def merge_preprocess_kwargs(base: PreprocessCfg, **kwargs): class AugmentationCfg (line 62) | class AugmentationCfg: function _setup_size (line 75) | def _setup_size(size, error_msg): class ResizeKeepRatio (line 88) | class ResizeKeepRatio: method __init__ (line 94) | def __init__( method get_params (line 116) | def get_params( method __call__ (line 144) | def __call__(self, img): method __repr__ (line 160) | def __repr__(self): function center_crop_or_pad (line 167) | def center_crop_or_pad(img: torch.Tensor, output_size: List[int], fill=0... class CenterCropOrPad (line 207) | class CenterCropOrPad(torch.nn.Module): method __init__ (line 219) | def __init__(self, size, fill=0): method forward (line 224) | def forward(self, img): method __repr__ (line 234) | def __repr__(self) -> str: function _convert_to_rgb (line 238) | def _convert_to_rgb(image): class color_jitter (line 242) | class color_jitter(object): method __init__ (line 246) | def __init__(self, brightness=0., contrast=0., saturation=0., hue=0., ... method __call__ (line 251) | def __call__(self, img): class gray_scale (line 258) | class gray_scale(object): method __init__ (line 262) | def __init__(self, p=0.2): method __call__ (line 267) | def __call__(self, img): function image_transform (line 274) | def image_transform( function image_transform_v2 (line 393) | def image_transform_v2( FILE: train/open_clip/src/open_clip/transformer.py class LayerNormFp32 (line 15) | class LayerNormFp32(nn.LayerNorm): method forward (line 18) | def forward(self, x: torch.Tensor): class LayerNorm (line 24) | class LayerNorm(nn.LayerNorm): method forward (line 27) | def forward(self, x: torch.Tensor): class QuickGELU (line 33) | class QuickGELU(nn.Module): method forward (line 35) | def forward(self, x: torch.Tensor): class LayerScale (line 39) | class LayerScale(nn.Module): method __init__ (line 40) | def __init__(self, dim, init_values=1e-5, inplace=False): method forward (line 45) | def forward(self, x): class PatchDropout (line 49) | class PatchDropout(nn.Module): method __init__ (line 54) | def __init__(self, prob, exclude_first_token=True): method forward (line 60) | def forward(self, x): class Attention (line 89) | class Attention(nn.Module): method __init__ (line 90) | def __init__( method forward (line 129) | def forward(self, x, attn_mask: Optional[torch.Tensor] = None): class AttentionalPooler (line 165) | class AttentionalPooler(nn.Module): method __init__ (line 166) | def __init__( method forward (line 180) | def forward(self, x: torch.Tensor): class ResidualAttentionBlock (line 188) | class ResidualAttentionBlock(nn.Module): method __init__ (line 189) | def __init__( method attention (line 216) | def attention( method forward (line 231) | def forward( class CustomResidualAttentionBlock (line 246) | class CustomResidualAttentionBlock(nn.Module): method __init__ (line 247) | def __init__( method forward (line 281) | def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] =... function _expand_token (line 287) | def _expand_token(token, batch_size: int): class Transformer (line 291) | class Transformer(nn.Module): method __init__ (line 292) | def __init__( method get_cast_dtype (line 313) | def get_cast_dtype(self) -> torch.dtype: method forward (line 318) | def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] =... class VisionTransformer (line 328) | class VisionTransformer(nn.Module): method __init__ (line 331) | def __init__( method lock (line 435) | def lock(self, unlocked_groups=0, freeze_bn_stats=False): method init_parameters (line 468) | def init_parameters(self): method set_grad_checkpointing (line 489) | def set_grad_checkpointing(self, enable=True): method _global_pool (line 492) | def _global_pool(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.T... method forward (line 502) | def forward(self, x: torch.Tensor): function text_global_pool (line 550) | def text_global_pool(x, text: Optional[torch.Tensor] = None, pool_type: ... class TextTransformer (line 565) | class TextTransformer(nn.Module): method __init__ (line 568) | def __init__( method init_parameters (line 628) | def init_parameters(self): method set_grad_checkpointing (line 652) | def set_grad_checkpointing(self, enable=True): method build_causal_mask (line 655) | def build_causal_mask(self): method build_cls_mask (line 663) | def build_cls_mask(self, text, cast_dtype: torch.dtype): method forward (line 672) | def forward(self, text): class MultimodalTransformer (line 711) | class MultimodalTransformer(Transformer): method __init__ (line 712) | def __init__( method init_parameters (line 753) | def init_parameters(self): method build_attention_mask (line 771) | def build_attention_mask(self): method forward (line 779) | def forward(self, image_embs, text_embs): method set_grad_checkpointing (line 802) | def set_grad_checkpointing(self, enable=True): FILE: train/open_clip/src/open_clip/utils.py function freeze_batch_norm_2d (line 9) | def freeze_batch_norm_2d(module, module_match={}, name=''): function _ntuple (line 49) | def _ntuple(n): function replace_linear (line 65) | def replace_linear(model, linear_replacement, include_modules=['c_fc', '... function convert_int8_model_to_inference_mode (line 84) | def convert_int8_model_to_inference_mode(model): FILE: train/open_clip/src/open_clip/zero_shot_classifier.py function batched (line 9) | def batched(iterable, n): function build_zero_shot_classifier (line 21) | def build_zero_shot_classifier( function build_zero_shot_classifier_legacy (line 71) | def build_zero_shot_classifier_legacy( FILE: train/open_clip/src/retrieve_clip_VQA.py function retrieve_topk_per_image (line 15) | def retrieve_topk_per_image(logits, val_k_list,retrieve_threshold=''): function get_logits (line 55) | def get_logits(image_features, text_features, logit_scale): function clean_data_info (line 62) | def clean_data_info(data_info): function split_and_clean_data_infos (line 75) | def split_and_clean_data_infos(batch_data_infos): function main (line 96) | def main(args): FILE: train/open_clip/src/retrieve_clip_report.py function retrieve_topk_per_image (line 16) | def retrieve_topk_per_image(logits, val_k_list,retrieve_threshold=''): function get_logits (line 56) | def get_logits(image_features, text_features, logit_scale): function clean_data_info (line 63) | def clean_data_info(data_info): function split_and_clean_data_infos (line 76) | def split_and_clean_data_infos(batch_data_infos): function main (line 97) | def main(args): FILE: train/open_clip/src/training/data.py class CsvDataset (line 41) | class CsvDataset(Dataset): method __init__ (line 42) | def __init__( method __len__ (line 55) | def __len__(self): method __getitem__ (line 58) | def __getitem__(self, idx): class IUXrayDataset (line 64) | class IUXrayDataset(Dataset): # TODO method __init__ (line 65) | def __init__( method __len__ (line 116) | def __len__(self): method __getitem__ (line 122) | def __getitem__(self, idx): class PubMedVisionDataset (line 138) | class PubMedVisionDataset(Dataset):#TODO method __init__ (line 139) | def __init__(self, img_root,json_file, transforms, tokenizer=None,load... method __len__ (line 177) | def __len__(self): method __getitem__ (line 183) | def __getitem__(self, idx): class RadiologyDataset (line 200) | class RadiologyDataset(Dataset): # TODO method __init__ (line 201) | def __init__( method __len__ (line 241) | def __len__(self): method __getitem__ (line 246) | def __getitem__(self, idx): class PathologyDataset (line 258) | class PathologyDataset(Dataset): # TODO method __init__ (line 259) | def __init__( method __len__ (line 298) | def __len__(self): method __getitem__ (line 303) | def __getitem__(self, idx): class MimicVQADataset (line 316) | class MimicVQADataset(Dataset): # method __init__ (line 317) | def __init__( method __len__ (line 357) | def __len__(self): method __getitem__ (line 363) | def __getitem__(self, idx): class IUXrayVQADataset (line 375) | class IUXrayVQADataset(Dataset): # TODO method __init__ (line 376) | def __init__( method __len__ (line 416) | def __len__(self): method __getitem__ (line 422) | def __getitem__(self, idx): class pmc_oa_VQADataset (line 435) | class pmc_oa_VQADataset(Dataset): # TODO method __init__ (line 436) | def __init__( method __len__ (line 478) | def __len__(self): method __getitem__ (line 484) | def __getitem__(self, idx): class HarvardVQADataset (line 496) | class HarvardVQADataset(Dataset): # TODO method __init__ (line 497) | def __init__( method __len__ (line 537) | def __len__(self): method __getitem__ (line 543) | def __getitem__(self, idx): class quilt_1m_VQADataset (line 555) | class quilt_1m_VQADataset(Dataset): # TODO method __init__ (line 556) | def __init__( method __len__ (line 596) | def __len__(self): method __getitem__ (line 602) | def __getitem__(self, idx): class quilt_1m_Dataset (line 615) | class quilt_1m_Dataset(Dataset): # TODO method __init__ (line 616) | def __init__( method __len__ (line 657) | def __len__(self): method __getitem__ (line 663) | def __getitem__(self, idx): class pmc_oa_Dataset (line 683) | class pmc_oa_Dataset(Dataset): # TODO method __init__ (line 684) | def __init__( method __len__ (line 729) | def __len__(self): method __getitem__ (line 735) | def __getitem__(self, idx): class IUXrayVQADataset_with_conf (line 756) | class IUXrayVQADataset_with_conf(Dataset): # TODO method __init__ (line 757) | def __init__( method __len__ (line 828) | def __len__(self): method __getitem__ (line 834) | def __getitem__(self, idx): method map_confidence_to_k (line 847) | def map_confidence_to_k(self, confidence): class MimicDataset (line 874) | class MimicDataset(Dataset): # TODO method __init__ (line 875) | def __init__( method __len__ (line 922) | def __len__(self): method __getitem__ (line 928) | def __getitem__(self, idx): class HarvardDataset (line 945) | class HarvardDataset(Dataset): # TODO method __init__ (line 946) | def __init__( method __len__ (line 997) | def __len__(self): method __getitem__ (line 1003) | def __getitem__(self, idx): class SharedEpoch (line 1021) | class SharedEpoch: method __init__ (line 1022) | def __init__(self, epoch: int = 0): method set_value (line 1025) | def set_value(self, epoch): method get_value (line 1028) | def get_value(self): class DataInfo (line 1033) | class DataInfo: method set_epoch (line 1038) | def set_epoch(self, epoch): function expand_urls (line 1045) | def expand_urls(urls, weights=None): function get_dataset_size (line 1068) | def get_dataset_size(shards): function get_imagenet (line 1090) | def get_imagenet(args, preprocess_fns, split): function count_samples (line 1137) | def count_samples(dataloader): function filter_no_caption_or_no_image (line 1147) | def filter_no_caption_or_no_image(sample): function log_and_continue (line 1155) | def log_and_continue(exn): function group_by_keys_nothrow (line 1161) | def group_by_keys_nothrow( function tarfile_to_samples_nothrow (line 1195) | def tarfile_to_samples_nothrow(src, handler=log_and_continue): function pytorch_worker_seed (line 1203) | def pytorch_worker_seed(increment=0): class detshuffle2 (line 1223) | class detshuffle2(wds.PipelineStage): method __init__ (line 1224) | def __init__( method run (line 1236) | def run(self, src): class ResampledShards2 (line 1255) | class ResampledShards2(IterableDataset): method __init__ (line 1258) | def __init__( method __iter__ (line 1286) | def __iter__(self): function get_wds_dataset (line 1312) | def get_wds_dataset( function get_csv_dataset (line 1451) | def get_csv_dataset(args, preprocess_fn, is_train, epoch=0, tokenizer=No... function get_pmc_oa_Dataset (line 1480) | def get_pmc_oa_Dataset( function get_quilt_1m_Dataset (line 1520) | def get_quilt_1m_Dataset( function get_HarvardDataset (line 1560) | def get_HarvardDataset( function get_IUXrayDataset (line 1601) | def get_IUXrayDataset( function get_MimicDataset (line 1642) | def get_MimicDataset( class SyntheticDataset (line 1683) | class SyntheticDataset(Dataset): method __init__ (line 1684) | def __init__( method __len__ (line 1700) | def __len__(self): method __getitem__ (line 1703) | def __getitem__(self, idx): function get_synthetic_dataset (line 1709) | def get_synthetic_dataset(args, preprocess_fn, is_train, epoch=0, tokeni... function get_radiology_Dataset (line 1735) | def get_radiology_Dataset( function get_pathology_Dataset (line 1775) | def get_pathology_Dataset( function get_dataset_fn (line 1816) | def get_dataset_fn(data_path, dataset_type): function get_data (line 1851) | def get_data( FILE: train/open_clip/src/training/distributed.py function is_global_master (line 12) | def is_global_master(args): function is_local_master (line 16) | def is_local_master(args): function is_master (line 20) | def is_master(args, local=False): function is_using_horovod (line 24) | def is_using_horovod(): function is_using_distributed (line 35) | def is_using_distributed(): function world_info_from_env (line 43) | def world_info_from_env(): function init_distributed_device (line 63) | def init_distributed_device(args): function broadcast_object (line 117) | def broadcast_object(args, obj, src=0): function all_gather_object (line 130) | def all_gather_object(args, obj, dst=0): FILE: train/open_clip/src/training/file_utils.py function remote_sync_s3 (line 10) | def remote_sync_s3(local_dir, remote_dir): function remote_sync_fsspec (line 20) | def remote_sync_fsspec(local_dir, remote_dir): function remote_sync (line 44) | def remote_sync(local_dir, remote_dir, protocol): function keep_running_remote_sync (line 54) | def keep_running_remote_sync(sync_every, local_dir, remote_dir, protocol): function start_sync_process (line 59) | def start_sync_process(sync_every, local_dir, remote_dir, protocol): function pt_save (line 64) | def pt_save(pt_obj, file_path): function pt_load (line 69) | def pt_load(file_path, map_location=None): function check_exists (line 77) | def check_exists(file_path): FILE: train/open_clip/src/training/logger.py function setup_logging (line 4) | def setup_logging(log_file, level, include_host=False): FILE: train/open_clip/src/training/main.py function random_seed (line 52) | def random_seed(seed=42, rank=0): function natural_key (line 58) | def natural_key(string_): function get_latest_checkpoint (line 63) | def get_latest_checkpoint(path: str, remote : bool): function main (line 79) | def main(args): function copy_codebase (line 519) | def copy_codebase(args): FILE: train/open_clip/src/training/main_feature-work.py function random_seed (line 54) | def random_seed(seed=42, rank=0): function natural_key (line 60) | def natural_key(string_): function get_latest_checkpoint (line 65) | def get_latest_checkpoint(path: str, remote : bool): function get_checkpoint_from_id (line 80) | def get_checkpoint_from_id(path: str, remote : bool,checkpoint_id:int): function main (line 101) | def main(args): FILE: train/open_clip/src/training/main_retrieve_report.py function random_seed (line 54) | def random_seed(seed=42, rank=0): function natural_key (line 60) | def natural_key(string_): function get_latest_checkpoint (line 65) | def get_latest_checkpoint(path: str, remote : bool): function get_checkpoint_from_id (line 80) | def get_checkpoint_from_id(path: str, remote : bool,checkpoint_id:int): function main (line 101) | def main(args): FILE: train/open_clip/src/training/main_retrieve_report_harvard.py function random_seed (line 54) | def random_seed(seed=42, rank=0): function natural_key (line 60) | def natural_key(string_): function get_latest_checkpoint (line 65) | def get_latest_checkpoint(path: str, remote : bool): function get_checkpoint_from_id (line 80) | def get_checkpoint_from_id(path: str, remote : bool,checkpoint_id:int): function main (line 101) | def main(args): FILE: train/open_clip/src/training/params.py function get_default_params (line 5) | def get_default_params(model_name): class ParseKwargs (line 14) | class ParseKwargs(argparse.Action): method __call__ (line 15) | def __call__(self, parser, namespace, values, option_string=None): function parse_args (line 26) | def parse_args(args): FILE: train/open_clip/src/training/precision.py function get_autocast (line 5) | def get_autocast(precision): FILE: train/open_clip/src/training/profiler.py function profile_fvcore (line 23) | def profile_fvcore( function profile_fvcore_text (line 44) | def profile_fvcore_text( function profile_fvcore_image (line 63) | def profile_fvcore_image( function profile_torch_image (line 82) | def profile_torch_image(model, image_input_size, batch_size=1, force_cpu... function profile_torch_text (line 96) | def profile_torch_text(model, text_input_size, batch_size=1, force_cpu=F... function profile_torch (line 110) | def profile_torch(model, text_input_size, image_input_size, batch_size=1... function count_params (line 125) | def count_params(model): function profile_model (line 128) | def profile_model(model_name, batch_size=1, profiler='torch'): function main (line 205) | def main(): FILE: train/open_clip/src/training/scheduler.py function assign_learning_rate (line 4) | def assign_learning_rate(optimizer, new_lr): function _warmup_lr (line 9) | def _warmup_lr(base_lr, warmup_length, step): function const_lr (line 13) | def const_lr(optimizer, base_lr, warmup_length, steps): function const_lr_cooldown (line 24) | def const_lr_cooldown(optimizer, base_lr, warmup_length, steps, cooldown... function cosine_lr (line 43) | def cosine_lr(optimizer, base_lr, warmup_length, steps): FILE: train/open_clip/src/training/train.py class AverageMeter (line 23) | class AverageMeter(object): method __init__ (line 26) | def __init__(self): method reset (line 29) | def reset(self): method update (line 35) | def update(self, val, n=1): function postprocess_clip_output (line 42) | def postprocess_clip_output(model_out): function unwrap_model (line 50) | def unwrap_model(model): function backward (line 57) | def backward(total_loss, scaler): function train_one_epoch (line 64) | def train_one_epoch(model, data, loss, epoch, optimizer, scaler, schedul... function evaluate (line 252) | def evaluate(model, data, epoch, args, tb_writer=None, tokenizer=None): function retrieve_report (line 362) | def retrieve_report(model, data, epoch, args, tb_writer=None, tokenizer=... function get_clip_metrics (line 579) | def get_clip_metrics(image_features, text_features, logit_scale): function get_logits (line 598) | def get_logits(image_features, text_features, logit_scale): function maybe_compute_generative_loss (line 611) | def maybe_compute_generative_loss(model_out): FILE: train/open_clip/src/training/zero_shot.py function accuracy (line 11) | def accuracy(output, target, topk=(1,)): function run (line 17) | def run(model, classifier, dataloader, args): function run_retrieve (line 43) | def run_retrieve(model, classifier, dataloader, args): function zero_shot_retrieve (line 69) | def zero_shot_retrieve(model, data, epoch, args, tokenizer=None): function zero_shot_eval (line 113) | def zero_shot_eval(model, data, epoch, args, tokenizer=None):