(0.3B in total) | [HuggingFace](https://huggingface.co/Haozhangcx/llava_grounding_gd_vp) |\n" }, { "path": "gradio_demo/LLaVA_G_Demo.py", "content": "\nimport gradio as gr\nimport os\nimport cv2\n\nimport torch\nimport numpy as np\nfrom llava.eval.LLaVA_G_Eval import Evaluator_MM_Inter\nfrom llava import conversation as conversation_lib\nfrom llava.mm_utils import tokenizer_image_token\nfrom llava.constants import DEFAULT_IMAGE_TOKEN\n\ndef get_image_name(dir_save=\"./gradio_demo/tmp_files\", prefix=\"click_img_\"):\n import os\n files = os.listdir(dir_save)\n file_orders = [int(file.split(\".\")[0][len(prefix):]) for file in files if file.endswith(\".jpg\") and file.startswith(prefix)]\n if len(file_orders) == 0:\n return os.path.join(dir_save, prefix + \"0.jpg\")\n else:\n return os.path.join(dir_save, prefix + str(max(file_orders) + 1) + \".jpg\")\ndef preprocess_multi_conv(\n sources,\n tokenizer,\n has_image = False\n):\n conv = conversation_lib.default_conversation.copy()\n roles = {\"human\": conv.roles[0], \"gpt\": conv.roles[1]}\n conv.messages = []\n for i, source in enumerate(sources):\n if roles[source[0][\"from\"]] != conv.roles[0]:\n # Skip the first one if it is not from human\n source = source[1:]\n\n for j, sentence in enumerate(source):\n role = roles[sentence[\"from\"]]\n assert role == conv.roles[j % 2], f\"{i}\"\n conv.append_message(role, sentence[\"value\"])\n conv_prompt = conv.get_prompt()\n conv_prompt = \"ASSISTANT: \".join(conv_prompt.split(\"ASSISTANT: \")[:-1]) + \"ASSISTANT:\"\n conv_prompt = conv_prompt.replace(\"\", \"\")\n conversations = [conv_prompt]\n print(\"Input Prompt: \", conv_prompt)\n\n # Tokenize conversations\n\n if has_image:\n input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)\n else:\n input_ids = tokenizer(\n conversations,\n return_tensors=\"pt\",\n padding=\"longest\",\n max_length=tokenizer.model_max_length,\n truncation=True,\n ).input_ids\n\n targets = input_ids.clone()\n\n assert conv.sep_style == conversation_lib.SeparatorStyle.TWO\n\n return dict(\n input_ids=input_ids,\n labels=targets,\n )\ndef filter_empty_box_mask(text, boxes_image, masks_image):\n def extract_text(sentence):\n # Use regular expression to find and extract the text and number\n import re\n pattern = r\"
\n
\n\n Who's GPT-4's favorite? Battles between State-of-the-Art Chatbots
\n\n \n\n
\n\n \n \n \n \n
\n \n \n \n
\n \n \n
\n \n \n \n
\n \n
\n\n \n \n \n
\n \n
\n \n ![\"other](\"\")
\n
\n \n
\n \n \n ![](\"figures/swords_FILL0_wght300_GRAD0_opsz48.svg\")
\n
\n \n \n
\n \n ![\"vicuna](\"figures/vicuna.jpeg\")
\n
\n \n \n \n
\n \n \n
\n\n \n \n
\n\n \n \n
\n \n
\n \n
\n \n
\n \n \n
\n \n \n
\n \n
\n \n \n
\n \n \n
\n \n
\n \n Assistant #2 (Vicuna, our model)\n
\n \n
\n \n \n
\n \n \n
\n GPT-4 Evaluation
\n \n
\n \n \n
\n \n \n
\n\n \n \n \n \n \n \n\n \n \n\n\n\n"
},
{
"path": "llava/eval/webpage/script.js",
"content": "// Description: Script for the evaluation webpage.\n\nlet currentQuestionIndex = 1;\n\n// Store the model name mapping for later use.\nmodelNameMapping = {\n \"gpt35\": \"ChatGPT-3.5\",\n \"gpt4\": \"GPT-4\",\n \"alpaca\": \"Alpaca-13b\",\n \"vicuna\": \"Vicuna-13b\",\n \"llama\": \"LLaMA-13b\",\n \"bard\": \"Bard\",\n};\n\nmodelFigureMapping = {\n \"vicuna\": \"figures/vicuna.jpeg\",\n // Image from: https://commons.wikimedia.org/wiki/File:ChatGPT_logo.svg\n \"gpt35\": \"figures/chatgpt.svg\",\n // Image from: https://www.reddit.com/r/logodesign/comments/1128aat/google_ai_bard_logo_design/\n \"bard\": \"figures/bard.jpg\",\n // Image from: https://crfm.stanford.edu/2023/03/13/alpaca.html\n \"alpaca\": \"figures/alpaca.png\",\n // Image adapted from https://commons.wikimedia.org/wiki/File:Llama_on_Machu_Picchu.jpg\n \"llama\": \"figures/llama.jpg\",\n}\n\n// Store the question data in a mapping for later use.\nquestionMapping = {};\n// Store the question ids in a mapping for later use.\ncategoryMapping = {};\n// Store the number of questions for later use.\nquestionsCount = 0;\n\n\nfunction text2Markdown(text) {\n // Normalize the text for markdown rendering.\n text = text.trim().replaceAll('\\n\\n', '\\n').replaceAll('\\n', '\\n\\n');\n return marked.parse(text);\n}\n\nfunction capitalizeFirstChar(str) {\n if (!str || str.length === 0) {\n return str;\n }\n return str.charAt(0).toUpperCase() + str.slice(1);\n}\n\nfunction updateQuestionSelect(question_id) {\n const select = document.getElementById('question-select');\n // Clear the question select.\n select.innerHTML = '';\n // Populate the question select.\n category = questionMapping[question_id].category;\n categoryMapping[category].forEach(question_id => {\n const question = questionMapping[question_id];\n const option = document.createElement('option');\n option.value = question_id;\n option.textContent = 'Q' + question_id.toString() + ': ' + question.question;\n select.appendChild(option);\n });\n select.value = question_id;\n}\n\nfunction updateModelSelect() {\n const select = document.getElementById('model-select');\n img_path = modelFigureMapping[select.value];\n document.getElementById('other-model-figure').src = img_path;\n}\n\nfunction populateModels(models) {\n const select = document.getElementById('model-select');\n models.forEach(model => {\n const option = document.createElement('option');\n option.value = model;\n option.textContent = modelNameMapping[model];\n select.appendChild(option);\n });\n updateModelSelect();\n}\n\nfunction populateQuestions(questions) {\n const category_select = document.getElementById('category-select');\n\n questionsCount = questions.length;\n questions.forEach(question => {\n const option = document.createElement('option');\n // Store the question data in a mapping for later use.\n questionMapping[question.id] = {\n category: question.category,\n question: question.question,\n answers: question.answers,\n evaluations: question.evaluations,\n scores: question.scores,\n };\n // Store the question id in the category mapping.\n if (question.category in categoryMapping) {\n categoryMapping[question.category].push(question.id);\n } else {\n categoryMapping[question.category] = [question.id];\n const category_option = document.createElement('option');\n category_option.value = question.category;\n category_option.textContent = capitalizeFirstChar(question.category);\n category_select.appendChild(category_option);\n }\n });\n // Set the default category.\n updateQuestionSelect(currentQuestionIndex);\n}\n\nfunction displayQuestion(index) {\n const question = questionMapping[index].question;\n document.getElementById('selected-question').innerHTML = text2Markdown('**Question:** ' + question);\n displayAnswers(index);\n}\n\nfunction displayAnswers(index) {\n const question = questionMapping[index];\n const otherModel = document.getElementById('model-select').value;\n // render the answers with markdown\n document.getElementById('other-model-answer').innerHTML = text2Markdown(question.answers[otherModel]);\n document.getElementById('our-model-answer').innerHTML = text2Markdown(question.answers.vicuna);\n\n // Display evaluation\n score = question.scores[otherModel];\n score_text = modelNameMapping[otherModel] + \" \" + score[0] + \"/10, Vicuna-13b \" + score[1] + \"/10\";\n document.getElementById('evaluation-header').textContent = \"GPT-4 Evaluation\" + \" (Score: \" + score_text + \")\";\n document.getElementById('evaluation-result').innerHTML = text2Markdown(question.evaluations[otherModel]);\n\n // Update model names\n let assistant1_title = \"Assistant #1\"; // (\" + modelNameMapping[otherModel] + \")\";\n let assistant2_title = \"Assistant #2 (Vicuna-13b, our model)\";\n // Update scores/labels.\n let assistant1_score_label = score[0].toString() + '/10';\n let assistant2_score_label = score[1].toString() + '/10';\n\n const colorRed ='#fa9'; // '#eb978d';\n // const colorGreen = '#c9f2c9';\n const colorBlue = '#8ef'; // '#71dbf9';\n const colorYellow = '#fe7'; // '#fada57';\n let otherModelHeaderColor = '';\n let ourModelHeaderColor = '';\n // Update the winner.\n if (score[0] == score[1]) {\n assistant1_title = 'π ' + assistant1_title;\n assistant1_score_label = 'π ' + assistant1_score_label;\n assistant2_title = 'π ' + assistant2_title;\n assistant2_score_label = 'π ' + assistant2_score_label;\n otherModelHeaderColor = colorYellow;\n ourModelHeaderColor = colorYellow;\n } else if (score[0] > score[1]) {\n assistant1_title = 'π ' + assistant1_title;\n assistant1_score_label = 'π ' + assistant1_score_label;\n otherModelHeaderColor = colorBlue;\n ourModelHeaderColor = colorRed;\n } else if (score[0] < score[1]) {\n assistant2_title = 'π ' + assistant2_title;\n assistant2_score_label = 'π ' + assistant2_score_label;\n otherModelHeaderColor = colorRed;\n ourModelHeaderColor = colorBlue;\n }\n\n document.getElementById('other-model-header-bg').style.backgroundColor = otherModelHeaderColor;\n document.getElementById('our-model-header').style.backgroundColor = ourModelHeaderColor;\n\n document.getElementById('other-model-header').textContent = assistant1_title;\n document.getElementById('our-model-header').textContent = assistant2_title;\n\n document.getElementById('other-score-label').textContent = assistant1_score_label;\n document.getElementById('our-score-label').textContent = assistant2_score_label;\n\n // Update expand buttons visibility for both cards after displaying answers\n // Reset the expanded state and update expand buttons visibility for both cards after displaying answers\n document.querySelectorAll('.expandable-card').forEach(card => {\n card.classList.remove('expanded');\n updateExpandButtonVisibility(card);\n const expandBtn = card.querySelector('.expand-btn');\n expandBtn.innerHTML = 'keyboard_arrow_down Show more'; // .textContent = 'Show more';\n });\n}\n\ndocument.getElementById('question-select').addEventListener('change', e => {\n currentQuestionIndex = parseInt(e.target.value);\n displayQuestion(currentQuestionIndex);\n});\n\ndocument.getElementById('category-select').addEventListener('change', e => {\n let currentCategory = e.target.value;\n const questionIds = categoryMapping[currentCategory];\n currentQuestionIndex = questionIds[0];\n updateQuestionSelect(currentQuestionIndex);\n displayQuestion(currentQuestionIndex);\n});\n\n// Update expand buttons whenever the model is changed\ndocument.getElementById('model-select').addEventListener('change', () => {\n displayAnswers(currentQuestionIndex);\n document.querySelectorAll('.expandable-card').forEach(card => {\n updateExpandButtonVisibility(card);\n });\n updateModelSelect();\n});\n\nfunction switchQuestionAndCategory() {\n document.getElementById('question-select').value = currentQuestionIndex;\n old_category = document.getElementById('category-select').value;\n new_category = questionMapping[currentQuestionIndex].category;\n if (old_category != new_category) {\n document.getElementById('category-select').value = new_category;\n updateQuestionSelect(currentQuestionIndex);\n }\n displayQuestion(currentQuestionIndex);\n}\n\ndocument.getElementById('prev-question').addEventListener('click', () => {\n // Question index starts from 1.\n currentQuestionIndex = Math.max(1, currentQuestionIndex - 1);\n switchQuestionAndCategory();\n});\n\ndocument.getElementById('next-question').addEventListener('click', () => {\n // Question index starts from 1.\n currentQuestionIndex = Math.min(questionsCount, currentQuestionIndex + 1);\n switchQuestionAndCategory();\n});\n\nfunction updateExpandButtonVisibility(card) {\n const cardTextContainer = card.querySelector('.card-text-container');\n const expandBtn = card.querySelector('.expand-btn');\n if (cardTextContainer.scrollHeight > cardTextContainer.offsetHeight) {\n expandBtn.style.display = 'flex';\n } else {\n expandBtn.style.display = 'none';\n card.classList.add('expanded');\n }\n}\n\ndocument.querySelectorAll('.expand-btn').forEach(btn => {\n btn.addEventListener('click', e => {\n const card = e.target.closest('.expandable-card');\n card.classList.toggle('expanded');\n const more = 'keyboard_arrow_down Show more';\n const less = 'keyboard_arrow_up Show less';\n e.target.innerHTML = card.classList.contains('expanded') ? less : more;\n });\n});\n"
},
{
"path": "llava/eval/webpage/styles.css",
"content": "body {\n font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;\n background-color: #f8f9fa;\n}\n\n.navbar-dark .navbar-nav .nav-link {\n color: #f1cf68;\n font-size: 1.1rem;\n padding: 0.5rem 0.6rem;\n}\n\n.card-header {\n font-weight: bold;\n}\n\n.card {\n box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);\n transition: 0.3s;\n}\n\n.card:hover {\n box-shadow: 0 8px 16px rgba(0, 0, 0, 0.2);\n}\n\nbutton {\n transition: background-color 0.3s;\n}\n\nbutton:hover {\n background-color: #007bff;\n}\n\n@media (max-width: 767px) {\n .form-row .form-group {\n margin-bottom: 10px;\n }\n}\n\n/* Extra styles */\n\n.expandable-card .card-text-container {\n max-height: 200px;\n overflow-y: hidden;\n position: relative;\n}\n\n.expandable-card.expanded .card-text-container {\n max-height: none;\n}\n\n.expand-btn {\n position: relative;\n display: none;\n background-color: rgba(255, 255, 255, 0.8);\n color: #510c75;\n border-color: transparent;\n}\n\n.expand-btn:hover {\n background-color: rgba(200, 200, 200, 0.8);\n text-decoration: none;\n border-color: transparent;\n color: #510c75;\n}\n\n.expand-btn:focus {\n outline: none;\n text-decoration: none;\n}\n\n.expandable-card:not(.expanded) .card-text-container:after {\n content: \"\";\n position: absolute;\n bottom: 0;\n left: 0;\n width: 100%;\n height: 90px;\n background: linear-gradient(rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 1));\n}\n\n.expandable-card:not(.expanded) .expand-btn {\n margin-top: -40px;\n}\n\n.card-body {\n padding-bottom: 5px;\n}\n\n.vertical-flex-layout {\n justify-content: center;\n align-items: center;\n height: 100%;\n display: flex;\n flex-direction: column;\n gap: 5px;\n}\n\n.figure-img {\n max-width: 100%;\n height: auto;\n}\n\n.adjustable-font-size {\n font-size: calc(0.5rem + 2vw);\n}\n"
},
{
"path": "llava/mm_utils.py",
"content": "from PIL import Image\nfrom io import BytesIO\nimport base64\n\nimport torch\nfrom transformers import StoppingCriteria\nfrom llava.constants import IMAGE_TOKEN_INDEX\n\n\ndef load_image_from_base64(image):\n return Image.open(BytesIO(base64.b64decode(image)))\n\n\ndef process_images(images, image_processor, model_cfg):\n return image_processor(images, return_tensors='pt')['pixel_values']\n\n\ndef tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):\n prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('\n This website is co-authored with GPT-4.\n
\n