Showing preview only (3,081K chars total). Download the full file or copy to clipboard to get everything.
Repository: milvus-io/milvus-lite
Branch: main
Commit: 9ac839f1608d
Files: 185
Total size: 2.9 MB
Directory structure:
gitextract_0fobvlb0/
├── .clang-format
├── .gitignore
├── .gitmodules
├── .pylintrc
├── CMakeLists.txt
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── ci/
│ ├── e2e.groovy
│ ├── entrypoint.sh
│ ├── get_author_email.sh
│ ├── nightly.groovy
│ ├── rte-build.yaml
│ ├── set_docker_mirror.sh
│ └── test.sh
├── cmake/
│ ├── Findantlr4-runtime.cmake
│ └── milvus-storage.cmake
├── conanfile.py
├── examples/
│ ├── bfloat16_example.py
│ ├── binary_example.py
│ ├── bm25.py
│ ├── customize_schema.py
│ ├── customize_schema_auto_id.py
│ ├── dynamic_field.py
│ ├── example_group_by.py
│ ├── float16_example.py
│ ├── fuzzy_match.py
│ ├── hello_hybrid_bm25.py
│ ├── hello_milvus.py
│ ├── hello_milvus_array.py
│ ├── hello_milvus_delete.py
│ ├── index.py
│ ├── non_ascii_encode.py
│ ├── simple.py
│ ├── simple_auto_id.py
│ └── sparse.py
├── python/
│ ├── pyproject.toml
│ ├── requirements.txt
│ ├── setup.py
│ └── src/
│ └── milvus_lite/
│ ├── __init__.py
│ ├── cmdline.py
│ ├── server.py
│ └── server_manager.py
├── scripts/
│ ├── Dockerfile.manylinux.aarch64
│ ├── Dockerfile.manylinux.x86_64
│ ├── README.md
│ ├── build.sh
│ └── build_milvus_lite.sh
├── src/
│ ├── CMakeLists.txt
│ ├── common.h
│ ├── create_collection_task.cpp
│ ├── create_collection_task.h
│ ├── create_index_task.cpp
│ ├── create_index_task.h
│ ├── delete_task.cpp
│ ├── delete_task.h
│ ├── function/
│ │ ├── bm25_function.cpp
│ │ ├── bm25_function.h
│ │ ├── function.h
│ │ ├── function_executor.cpp
│ │ ├── function_executor.h
│ │ └── function_util.h
│ ├── hybrid_search_task.cpp
│ ├── hybrid_search_task.h
│ ├── index.cpp
│ ├── index.h
│ ├── insert_task.cpp
│ ├── insert_task.h
│ ├── milvus_id.hpp
│ ├── milvus_local.cpp
│ ├── milvus_local.h
│ ├── milvus_proxy.cpp
│ ├── milvus_proxy.h
│ ├── milvus_service_impl.cpp
│ ├── milvus_service_impl.h
│ ├── parser/
│ │ ├── Plan.g4
│ │ ├── antlr/
│ │ │ ├── PlanBaseVisitor.cpp
│ │ │ ├── PlanBaseVisitor.h
│ │ │ ├── PlanLexer.cpp
│ │ │ ├── PlanLexer.h
│ │ │ ├── PlanParser.cpp
│ │ │ ├── PlanParser.h
│ │ │ ├── PlanVisitor.cpp
│ │ │ └── PlanVisitor.h
│ │ ├── parser.cc
│ │ ├── parser.h
│ │ └── utils.h
│ ├── query_task.cpp
│ ├── query_task.h
│ ├── re_scorer.cpp
│ ├── re_scorer.h
│ ├── retrieve_result.h
│ ├── schema_util.cpp
│ ├── schema_util.h
│ ├── search_result.h
│ ├── search_task.cpp
│ ├── search_task.h
│ ├── segcore_wrapper.cpp
│ ├── segcore_wrapper.h
│ ├── server.cpp
│ ├── status.h
│ ├── storage/
│ │ ├── bm25_stats.cpp
│ │ ├── bm25_stats.h
│ │ ├── collection_data.cpp
│ │ ├── collection_data.h
│ │ ├── collection_meta.cpp
│ │ ├── collection_meta.h
│ │ ├── storage.cpp
│ │ └── storage.h
│ ├── string_util.hpp
│ ├── timer.cpp
│ ├── timer.h
│ ├── type.h
│ ├── unittest/
│ │ ├── CMakeLists.txt
│ │ ├── bm25_function_test.cpp
│ │ ├── grpc_server_test.cpp
│ │ ├── milvus_local_test.cpp
│ │ ├── milvus_proxy_test.cpp
│ │ ├── run_examples.py
│ │ ├── storage_test.cpp
│ │ ├── test_util.cpp
│ │ └── test_util.h
│ ├── upsert_task.cpp
│ └── upsert_task.h
├── tests/
│ ├── base/
│ │ ├── client_base.py
│ │ ├── collection_wrapper.py
│ │ ├── connections_wrapper.py
│ │ ├── database_wrapper.py
│ │ ├── high_level_api_wrapper.py
│ │ ├── index_wrapper.py
│ │ ├── partition_wrapper.py
│ │ ├── schema_wrapper.py
│ │ └── utility_wrapper.py
│ ├── check/
│ │ ├── func_check.py
│ │ └── param_check.py
│ ├── common/
│ │ ├── bulk_insert_data.py
│ │ ├── code_mapping.py
│ │ ├── common_func.py
│ │ ├── common_type.py
│ │ ├── constants.py
│ │ ├── cus_resource_opts.py
│ │ ├── milvus_sys.py
│ │ └── minio_comm.py
│ ├── config/
│ │ └── log_config.py
│ ├── conftest.py
│ ├── customize/
│ │ ├── README.md
│ │ ├── milvus_operator.py
│ │ ├── template/
│ │ │ ├── default.yaml
│ │ │ └── minimum.yaml
│ │ ├── test_customize_segment_size.py
│ │ └── test_simd_compat.py
│ ├── data/
│ │ └── train_embeddings.csv
│ ├── milvus_lite/
│ │ ├── test_milvus_lite_close.py
│ │ ├── test_milvus_lite_collection.py
│ │ ├── test_milvus_lite_delete.py
│ │ ├── test_milvus_lite_index.py
│ │ ├── test_milvus_lite_init.py
│ │ ├── test_milvus_lite_insert.py
│ │ ├── test_milvus_lite_migrate.py
│ │ ├── test_milvus_lite_query.py
│ │ ├── test_milvus_lite_search.py
│ │ ├── test_milvus_lite_sparse.py
│ │ └── test_milvus_lite_stability.py
│ ├── pytest.ini
│ ├── requirements.txt
│ ├── run.sh
│ ├── test_bm25.py
│ ├── test_delete.py
│ ├── test_dump_tool.py
│ ├── test_hybrid_search.py
│ ├── test_query.py
│ ├── test_schema.py
│ ├── test_search.py
│ ├── test_sparse.py
│ ├── testcases/
│ │ └── test_collection.py
│ └── utils/
│ ├── api_request.py
│ ├── thread_util.py
│ ├── util_common.py
│ ├── util_k8s.py
│ ├── util_log.py
│ ├── util_pymilvus.py
│ └── wrapper.py
└── thirdparty/
├── knowhere-android.patch
├── knowhere.patch
└── milvus.patch
================================================
FILE CONTENTS
================================================
================================================
FILE: .clang-format
================================================
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
BasedOnStyle: Google
DerivePointerAlignment: false
ColumnLimit: 80
IndentWidth: 4
AccessModifierOffset: -3
AlwaysBreakAfterReturnType: All
AllowShortBlocksOnASingleLine: false
AllowShortFunctionsOnASingleLine: false
AllowShortIfStatementsOnASingleLine: false
AlignTrailingComments: true
# Appended Options
SortIncludes: false
Standard: Latest
AlignAfterOpenBracket: Align
BinPackParameters: false
BinPackArguments: false
ReflowComments: false
================================================
FILE: .gitignore
================================================
.DS_Store
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
*.db
compile_commands.json
# CLion
.idea/
cmake-build-debug/
# Claude Code
CLAUDE.md
================================================
FILE: .gitmodules
================================================
[submodule "thirdparty/milvus-proto"]
path = thirdparty/milvus-proto
url = https://github.com/milvus-io/milvus-proto.git
[submodule "thirdparty/milvus"]
path = thirdparty/milvus
url = https://github.com/milvus-io/milvus.git
================================================
FILE: .pylintrc
================================================
# This Pylint rcfile contains a best-effort configuration to uphold the
# best-practices and style described in the Google Python style guide:
# https://google.github.io/styleguide/pyguide.html
#
# Its canonical open-source location is:
# https://google.github.io/styleguide/pylintrc
[MASTER]
# Files or directories to be skipped. They should be base names, not paths.
ignore=third_party
# Files or directories matching the regex patterns are skipped. The regex
# matches against base names, not paths.
ignore-patterns=
# Pickle collected data for later comparisons.
persistent=no
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=4
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
confidence=
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once). See also the "--disable" option for examples.
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=abstract-method,
apply-builtin,
arguments-differ,
attribute-defined-outside-init,
backtick,
bad-option-value,
basestring-builtin,
buffer-builtin,
c-extension-no-member,
consider-using-enumerate,
cmp-builtin,
cmp-method,
coerce-builtin,
coerce-method,
delslice-method,
div-method,
duplicate-code,
eq-without-hash,
execfile-builtin,
file-builtin,
filter-builtin-not-iterating,
fixme,
getslice-method,
global-statement,
hex-method,
idiv-method,
implicit-str-concat,
import-error,
import-self,
import-star-module-level,
inconsistent-return-statements,
input-builtin,
intern-builtin,
invalid-str-codec,
locally-disabled,
long-builtin,
long-suffix,
map-builtin-not-iterating,
misplaced-comparison-constant,
missing-function-docstring,
metaclass-assignment,
next-method-called,
next-method-defined,
no-absolute-import,
no-else-break,
no-else-continue,
no-else-raise,
no-else-return,
no-init, # added
no-member,
no-name-in-module,
no-self-use,
nonzero-method,
oct-method,
old-division,
old-ne-operator,
old-octal-literal,
old-raise-syntax,
parameter-unpacking,
print-statement,
raising-string,
range-builtin-not-iterating,
raw_input-builtin,
rdiv-method,
reduce-builtin,
relative-import,
reload-builtin,
round-builtin,
setslice-method,
signature-differs,
standarderror-builtin,
suppressed-message,
sys-max-int,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
too-many-boolean-expressions,
too-many-branches,
too-many-instance-attributes,
too-many-locals,
too-many-nested-blocks,
too-many-public-methods,
too-many-return-statements,
too-many-statements,
trailing-newlines,
unichr-builtin,
unicode-builtin,
unnecessary-pass,
unpacking-in-except,
useless-else-on-loop,
useless-object-inheritance,
useless-suppression,
using-cmp-argument,
wrong-import-order,
xrange-builtin,
zip-builtin-not-iterating,
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html. You can also give a reporter class, eg
# mypackage.mymodule.MyReporterClass.
output-format=text
# Tells whether to display a full report or only the messages
reports=no
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details
#msg-template=
[BASIC]
# Good variable names which should always be accepted, separated by a comma
good-names=main,_
# Bad variable names which should always be refused, separated by a comma
bad-names=
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Include a hint for the correct naming format with invalid-name
include-naming-hint=no
# List of decorators that produce properties, such as abc.abstractproperty. Add
# to this list to register other decorators that produce valid properties.
property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl
# Regular expression matching correct function names
function-rgx=^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
# Regular expression matching correct variable names
variable-rgx=^[a-z][a-z0-9_]*$
# Regular expression matching correct constant names
const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
# Regular expression matching correct attribute names
attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
# Regular expression matching correct argument names
argument-rgx=^[a-z][a-z0-9_]*$
# Regular expression matching correct class attribute names
class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
# Regular expression matching correct inline iteration names
inlinevar-rgx=^[a-z][a-z0-9_]*$
# Regular expression matching correct class names
class-rgx=^_?[A-Z][a-zA-Z0-9]*$
# Regular expression matching correct module names
module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$
# Regular expression matching correct method names
method-rgx=(?x)^(?:(?P<exempt>_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=10
[TYPECHECK]
# List of decorators that produce context managers, such as
# contextlib.contextmanager. Add to this list to register other decorators that
# produce valid context managers.
contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=optparse.Values,thread._local,_thread._local
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=120
# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt
# lines made too long by directives to pytype.
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=(?x)(
^\s*(\#\ )?<?https?://\S+>?$|
^\s*(from\s+\S+\s+)?import\s+.+$)
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=yes
# Maximum number of lines in a module
max-module-lines=99999
# String used as indentation unit. The internal Google style guide mandates 2
# spaces. Google's externaly-published style guide says 4, consistent with
# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google
# projects (like TensorFlow).
# milvus use 4 spaces
indent-string=' '
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=TODO
[STRING]
# This flag controls whether inconsistent-quotes generates a warning when the
# character used as a quote delimiter is used inconsistently within a module.
check-quote-consistency=yes
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_)
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
# List of qualified module names which can have objects that can redefine
# builtins.
redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format
logging-modules=logging,absl.logging,tensorflow.io.logging
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[SPELLING]
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package.
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,
TERMIOS,
Bastion,
rexec,
sets
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
# Force import order to recognize a module as part of the standard
# compatibility libraries.
known-standard-library=
# Force import order to recognize a module as part of a third party library.
known-third-party=enchant, absl
# Analyse import fallback blocks. This can be used to support both Python 2 and
# 3 compatible code, which means that the block might have code that exists
# only in one or another interpreter, leading to false positives when analysed.
analyse-fallback-blocks=no
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,
__new__,
setUp
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,
_fields,
_replace,
_source,
_make
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls,
class_
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcs
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=builtins.StandardError,
builtins.Exception,
builtins.BaseException
================================================
FILE: CMakeLists.txt
================================================
# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License
cmake_minimum_required(VERSION 3.20)
project(milvus-lite)
option(ENABLE_UNIT_TESTS "Enable unit tests" ON)
message(STATUS "Enable testing: ${ENABLE_UNIT_TESTS}")
option(USE_SYSTEM_DEPS "Build with system dependencies rather than Conan" OFF)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/ ${CMAKE_SOURCE_DIR}/cmake)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED True)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(COMPILER_SUPPORTS_CXX17 True)
if(LINUX)
# When building for Linux, tell the dynamic loader to load shared libraries
# from executable's directory. This also prevents CMake from leaking build
# paths into the distributed executable.
set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
set(CMAKE_INSTALL_RPATH "$ORIGIN")
endif()
if(APPLE)
set(CMAKE_OSX_DEPLOYMENT_TARGET 11.0)
endif()
set(CMAKE_VERBOSE_MAKEFILE ON)
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CMAKE_CXX_FLAGS "-g -Wall -fPIC ${CMAKE_CXX_FLAGS}")
else()
set(CMAKE_CXX_FLAGS "-O3 -Wall -fPIC ${CMAKE_CXX_FLAGS}")
endif()
if(CMAKE_LITE_BUILD_TYPE STREQUAL "SHARED")
set(LITE_BUILD_TYPE "SHARED")
else()
set(LITE_BUILD_TYPE "STATIC")
endif()
if(APPLE)
if(DEFINED ENV{HOMEBREW_PREFIX})
message(STATUS "Homebrew prefix from environment: $ENV{HOMEBREW_PREFIX}")
set(OMP_INCLUDE $ENV{HOMEBREW_PREFIX}/opt/libomp/include)
include_directories($ENV{HOMEBREW_PREFIX}/opt/libomp/include)
link_directories($ENV{HOMEBREW_PREFIX}/opt/libomp/lib)
else()
message(STATUS "Homebrew prefix from environment not found, use default")
set(OMP_INCLUDE /opt/homebrew/opt/libomp/include)
include_directories(/opt/homebrew/opt/libomp/include)
link_directories(/opt/homebrew/opt/libomp/lib)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lomp")
set(OpenMP_C "${CMAKE_C_COMPILER}")
set(OpenMP_C_FLAGS "-Xclang -fopenmp -I${OMP_INCLUDE}")
set(OpenMP_C_LIB_NAMES "libomp")
set(OpenMP_libomp_LIBRARY "omp")
set(OpenMP_CXX "${CMAKE_CXX_COMPILER}")
set(OpenMP_CXX_FLAGS "-Xclang -fopenmp -I${OMP_INCLUDE}")
set(OpenMP_CXX_LIB_NAMES "libomp")
set(OpenMP_libomp_LIBRARY "omp")
endif()
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(MILVUS_LITE True)
include(FetchContent)
set(CONAN_LIBS "${CONAN_LIBS};marisa::marisa;")
find_package(SQLiteCpp REQUIRED)
include_directories(${SQLiteCpp_INCLUDE_DIRS})
find_package(antlr4-runtime REQUIRED)
include_directories(${antlr4-cppruntime_INCLUDES})
# note: gRPC needs to be included before protobuf when using system dependencies.
find_package(gRPC REQUIRED)
include_directories(${gRPC_INCLUDE_DIRS})
find_package(Protobuf REQUIRED)
if(USE_SYSTEM_DEPS)
set(PROTOBUF_LIBRARIES protobuf::libprotobuf)
else()
set(PROTOBUF_LIBRARIES protobuf::protobuf)
include_directories(${protobuf_INCLUDE_DIRS})
endif()
find_package(TBB REQUIRED)
include_directories(${TBB_tbb_INCLUDE_DIRS})
find_package(nlohmann_json REQUIRED)
include_directories(${nlohmann_json_INCLUDE_DIRS})
find_package(Boost REQUIRED)
include_directories(${Boost_INCLUDE_DIRS})
find_package(Libevent REQUIRED)
include_directories(${libevent_INCLUDE_DIRS})
find_package(folly REQUIRED)
include_directories(${Folly_INCLUDE_DIRS})
find_package(fmt REQUIRED)
include_directories(${fmt_INCLUDE_DIRS})
find_package(glog REQUIRED)
include_directories(${glog_INCLUDE_DIRS})
find_package(gflags REQUIRED)
include_directories(${gflags_INCLUDE_DIRS})
find_package(Arrow REQUIRED)
if(USE_SYSTEM_DEPS)
find_package(Parquet REQUIRED)
set(ARROW_LIBRARIES Arrow::arrow_shared Parquet::parquet_shared)
else()
set(ARROW_LIBRARIES arrow::arrow)
include_directories(${arrow_INCLUDE_DIRS})
endif()
find_package(re2 REQUIRED)
include_directories(${re2_INCLUDE_DIRS})
link_directories(${re2_LIB_DIRS})
find_package(double-conversion REQUIRED)
include_directories(${double-conversion_INCLUDE_DIRS})
find_package(prometheus-cpp REQUIRED)
if(USE_SYSTEM_DEPS)
set(PROMETHEUS_CPP_TARGETS prometheus-cpp::pull)
else()
set(PROMETHEUS_CPP_TARGETS prometheus-cpp::prometheus-cpp)
include_directories(${prometheus-cpp_INCLUDE_DIRS})
endif()
if(USE_SYSTEM_DEPS)
find_package(Marisa REQUIRED)
set(MARISA_LIBRARIES Marisa::marisa)
else()
find_package(marisa REQUIRED)
set(MARISA_LIBRARIES marisa::marisa)
include_directories(${marisa_INCLUDE_DIRS})
endif()
find_package(yaml-cpp REQUIRED)
include_directories(${yaml-cpp_INCLUDE_DIRS})
find_package(RapidJSON REQUIRED)
include_directories(${RapidJSON_INCLUDE_DIRS})
find_package(roaring REQUIRED)
include_directories(${roaring_INCLUDE_DIRS})
find_package(prometheus-cpp REQUIRED)
include_directories(${prometheus-cpp_INCLUDE_DIRS})
if(NOT USE_SYSTEM_DEPS)
include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
endif()
FetchContent_Declare(
simdjson
GIT_REPOSITORY https://github.com/simdjson/simdjson.git
GIT_TAG v3.1.7)
FetchContent_MakeAvailable(simdjson)
include_directories(${simdjson_SOURCE_DIR}/include)
add_definitions(-DANTLR4CPP_STATIC)
add_definitions(-DHAVE_CPP_STDLIB)
add_definitions(-DMILVUS_LITE)
add_definitions(-DFMT_HEADER_ONLY)
if(APPLE)
add_definitions("-D_GNU_SOURCE")
endif()
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/pb)
add_library(
milvus_proto STATIC
"${CMAKE_SOURCE_DIR}/src/proto/plan.proto"
"${CMAKE_SOURCE_DIR}/src/proto/schema.proto"
"${CMAKE_SOURCE_DIR}/src/proto/common.proto"
"${CMAKE_SOURCE_DIR}/src/proto/segcore.proto"
"${CMAKE_SOURCE_DIR}/src/proto/milvus.proto"
"${CMAKE_SOURCE_DIR}/src/proto/msg.proto"
"${CMAKE_SOURCE_DIR}/src/proto/feder.proto"
"${CMAKE_SOURCE_DIR}/src/proto/cgo_msg.proto"
"${CMAKE_SOURCE_DIR}/src/proto/index_cgo_msg.proto"
"${CMAKE_SOURCE_DIR}/src/proto/rg.proto")
set(PROTO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/pb")
protobuf_generate(
TARGET milvus_proto IMPORT_DIRS "${CMAKE_SOURCE_DIR}/src/proto"
PROTOC_OUT_DIR "${PROTO_BINARY_DIR}")
add_library(milvus_grpc_service STATIC
"${CMAKE_SOURCE_DIR}/src/proto/milvus.proto")
target_link_libraries(milvus_grpc_service gRPC::grpc++ milvus_proto)
protobuf_generate(
TARGET
milvus_grpc_service
# OUT_VAR PROTO_GENERATED_FILES
LANGUAGE
grpc
GENERATE_EXTENSIONS
.grpc.pb.h
.grpc.pb.cc
PLUGIN
"protoc-gen-grpc=\$<TARGET_FILE:gRPC::grpc_cpp_plugin>"
IMPORT_DIRS
"${CMAKE_SOURCE_DIR}/src/proto"
PROTOC_OUT_DIR
"${PROTO_BINARY_DIR}")
include_directories("${CMAKE_SOURCE_DIR}/src/parser/")
include_directories("${CMAKE_SOURCE_DIR}/src/parser/antlr/")
include_directories("${CMAKE_BINARY_DIR}/pb")
include_directories("${CMAKE_BINARY_DIR}")
include_directories(${PROTO_BINARY_DIR})
add_library(
parser STATIC
"${CMAKE_SOURCE_DIR}/src/parser/parser.cc"
"${CMAKE_SOURCE_DIR}/src/parser/antlr/PlanBaseVisitor.cpp"
"${CMAKE_SOURCE_DIR}/src/parser/antlr/PlanLexer.cpp"
"${CMAKE_SOURCE_DIR}/src/parser/antlr/PlanParser.cpp"
"${CMAKE_SOURCE_DIR}/src/parser/antlr/PlanVisitor.cpp")
target_link_libraries(parser milvus_proto ${antlr4-cppruntime_LIBRARIES})
include(thirdparty/milvus/internal/core/cmake/BuildUtils.cmake)
include(thirdparty/milvus/internal/core/cmake/Utils.cmake)
set(WITH_LIGHT ON CACHE BOOL "" FORCE)
add_subdirectory(thirdparty/milvus/internal/core/thirdparty/knowhere)
include_directories(
"${CMAKE_SOURCE_DIR}/thirdparty/milvus/internal/core/thirdparty/")
include_directories(
"${CMAKE_SOURCE_DIR}/thirdparty/milvus/internal/core/thirdparty/tantivy/tantivy-binding/include/"
)
include_directories(
"${CMAKE_SOURCE_DIR}/thirdparty/milvus/internal/core/thirdparty/tantivy/")
include_directories("${knowhere_SOURCE_DIR}/include")
#include(cmake/milvus-storage.cmake)
include_directories("${CMAKE_SOURCE_DIR}/thirdparty/milvus/internal/core/src/")
add_library(
boost_bitset_ext
thirdparty/milvus/internal/core/thirdparty/boost_ext/dynamic_bitset_ext.cpp)
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CARGO_CMD cargo build)
else()
set(CARGO_CMD cargo build --release)
endif()
if(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR STREQUAL
"aarch64")
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
set(CARGO_CMD cargo build --target aarch64-linux-android)
else()
set(CARGO_CMD cargo build --target aarch64-linux-android --release)
endif()
endif()
set(HOME_VAR $ENV{HOME})
add_custom_command(
OUTPUT ls_cargo
COMMENT "ls cargo"
COMMAND ls ${HOME_VAR}/.cargo/bin/)
add_custom_target(ls_cargo_target DEPENDS ls_cargo)
add_custom_command(
OUTPUT compile_tantivy
COMMENT "Compiling tantivy binding"
COMMAND CARGO_TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR}
MACOSX_DEPLOYMENT_TARGET=11.0 ${CARGO_CMD}
WORKING_DIRECTORY
${CMAKE_SOURCE_DIR}/thirdparty/milvus/internal/core/thirdparty/tantivy/tantivy-binding
DEPENDS ls_cargo_target)
add_custom_target(tantivy_binding_target DEPENDS compile_tantivy)
add_library(tantivy_binding STATIC IMPORTED)
string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
set(IMPORT_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE_LOWER}/libtantivy_binding.a")
if(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR STREQUAL
"aarch64")
set(IMPORT_LOCATION
"${CMAKE_CURRENT_BINARY_DIR}/aarch64-linux-android/${CMAKE_BUILD_TYPE_LOWER}/libtantivy_binding.a"
)
endif()
set_target_properties(
tantivy_binding
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION "${IMPORT_LOCATION}"
INTERFACE_INCLUDE_DIRECTORIES
"${CMAKE_SOURCE_DIR}/thirdparty/milvus/internal/core/thirdparty/tantivy/tantivy-binding/include/"
)
add_dependencies(tantivy_binding tantivy_binding_target)
execute_process(
COMMAND git diff --quiet
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/thirdparty/milvus
RESULT_VARIABLE CHECK_RESULT)
if(${CHECK_RESULT} EQUAL 0)
message("Apply milvus patch...")
execute_process(
COMMAND git apply ${CMAKE_SOURCE_DIR}/thirdparty/milvus.patch
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/thirdparty/milvus
OUTPUT_VARIABLE result)
else()
message("Milvus not need applying patch...")
endif()
set(MILVUS_ENGINE_SRC "${CMAKE_SOURCE_DIR}/thirdparty/milvus/internal/core/src")
add_subdirectory(thirdparty/milvus/internal/core/src/log)
add_subdirectory(thirdparty/milvus/internal/core/src/config)
add_subdirectory(thirdparty/milvus/internal/core/src/common)
add_subdirectory(thirdparty/milvus/internal/core/src/storage)
add_subdirectory(thirdparty/milvus/internal/core/src/query)
add_subdirectory(thirdparty/milvus/internal/core/src/exec)
add_subdirectory(thirdparty/milvus/internal/core/src/index)
add_subdirectory(thirdparty/milvus/internal/core/src/segcore)
add_subdirectory(thirdparty/milvus/internal/core/src/bitset)
add_subdirectory(thirdparty/milvus/internal/core/src/indexbuilder)
add_subdirectory(thirdparty/milvus/internal/core/src/futures)
add_subdirectory(thirdparty/milvus/internal/core/src/monitor)
add_library(milvus_core STATIC
$<TARGET_OBJECTS:milvus_log>
$<TARGET_OBJECTS:milvus_config>
$<TARGET_OBJECTS:milvus_common>
$<TARGET_OBJECTS:milvus_storage>
$<TARGET_OBJECTS:milvus_index>
$<TARGET_OBJECTS:milvus_query>
$<TARGET_OBJECTS:milvus_segcore>
$<TARGET_OBJECTS:milvus_indexbuilder>
$<TARGET_OBJECTS:milvus_exec>
$<TARGET_OBJECTS:milvus_bitset>
$<TARGET_OBJECTS:milvus_futures>
$<TARGET_OBJECTS:milvus_monitor>
)
target_link_libraries(
milvus_core
PUBLIC
milvus_proto
${PROMETHEUS_CPP_TARGET}
knowhere
)
if(ENABLE_UNIT_TESTS)
include(CTest)
enable_testing()
endif()
add_subdirectory(src)
find_program(MEMORYCHECK_COMMAND NAMES valgrind)
set(MEMORYCHECK_COMMAND_OPTIONS
"--trace-children=yes --track-origins=yes --leak-check=full --show-leak-kinds=all"
)
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing Guide
Welcome contributors! This guide will help you get started with contributing to Milvus-lite.
Please always find the latest version of this guide at [CONTRIBUTING.md:main](https://github.com/milvus-io/milvus-lite/blob/main/CONTRIBUTING.md)
## How to set up the development environment
The Milvus-lite project is written in Python. To set up the development environment, you need to install Python 3.8 or later. We recommend that you use a virtual environment to install the dependencies, although the Milvus-lite project requires a very small number of external packages.
The main dependencies for build Milvus-lite is to install dependencies of milvis, so generally you could refer to Milvus's [install_deps.sh](https://github.com/milvus-io/milvus/blob/master/scripts/install_deps.sh) as a reference. Please note, you should follow the related branch of Milvus. For example, if you want to build Milvus-lite with Milvus 2.4.0, you should checkout the branch of Milvus 2.4.0.
For python3 build wheel, we use the build module and requires newer version of setuptools. So you should install the latest version of setuptools and build.
## Build Milvus-lite
```bash
python3 setup.py bdist_wheel
```
After build, you shoud have wheel package under dist folder.
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
<div align="center">
<img src="https://raw.githubusercontent.com/milvus-io/milvus-lite/refs/heads/main/milvus_lite_logo.png#gh-light-mode-only" width="60%"/>
</div>
<h3 align="center">
<p style="text-align: center;"> <span style="font-weight: bold; font: Arial, sans-serif;"></span>A lightweight version of Milvus</p>
</h3>
<div class="column" align="middle">
<a href="https://www.apache.org/licenses/LICENSE-2.0">
<img src="https://img.shields.io/badge/license-apache2.0-green?style=flat" alt="license"/>
</a>
<a href="https://pypi.org/project/milvus-lite/">
<img src="https://img.shields.io/pypi/v/milvus-lite?label=Release&color&logo=Python" alt="github actions">
</a>
<a href="https://pypi.org/project/milvus-lite/">
<img src="https://img.shields.io/pypi/dm/milvus-lite.svg?color=bright-green&logo=Pypi" alt="github actions">
</a>
</div>
# Introduction
Milvus Lite is the lightweight version of [Milvus](https://github.com/milvus-io/milvus), a high-performance vector database that powers AI applications with vector similarity search. This repo contains the core components of Milvus Lite.
With Milvus Lite, you can start building an AI application with vector similarity search within minutes! Milvus Lite is good for running in the following environment:
- Jupyter Notebook / Google Colab
- Laptops
- Edge Devices
Milvus Lite can be imported into your Python application, providing the core vector search functionality of Milvus. Milvus Lite is already included in the [Python SDK of Milvus](https://github.com/milvus-io/pymilvus). To use it, you just need `pip install pymilvus`.
Milvus Lite uses the same API as Milvus Standalone and Distributed, providing a consistent experience across environments. Develop your GenAI applications once and run them anywhere: on a laptop or Jupyter Notebook with Milvus Lite, in a Docker container with Milvus Standalone, or on a K8s cluster with Milvus Distributed for large-scale production.
<div align="center">
<img src="https://milvus.io/docs/v2.5.x/assets/select-deployment-option.png" width="80%"/>
</div>
Milvus Lite is only suitable for small scale prototyping (usually less than a million vectors) or edge devices. For large scale production, we recommend using [Milvus Standalone](https://milvus.io/docs/install-overview.md#Milvus-Standalone) or [Milvus Distributed](https://milvus.io/docs/install-overview.md#Milvus-Distributed). You can also consider the fully-managed Milvus on [Zilliz Cloud](https://zilliz.com/cloud).
# Requirements
Milvus Lite currently supports the following environments:
- Ubuntu >= 20.04 (x86_64 and arm64)
- MacOS >= 11.0 (Apple Silicon M1/M2 and x86_64)
***Note:*** Windows is not yet supported.
# Installation
```shell
pip install -U pymilvus[milvus-lite]
```
We recommend using `pymilvus`. You can `pip install` with `-U` to force update to the latest version and `milvus-lite` will be automatically installed.
If you want to explicitly install `milvus-lite` package, or you have installed an older version of `milvus-lite` and would like to update it, you can do `pip install -U milvus-lite`.
# Usage
In `pymilvus`, specify a local file name as uri parameter of MilvusClient will use Milvus Lite.
```python
from pymilvus import MilvusClient
client = MilvusClient("./milvus_demo.db")
```
> **_NOTE:_** Note that the same API also applies to Milvus Standalone, Milvus Distributed and Zilliz Cloud, the only difference is to replace local file name to remote server endpoint and credentials, e.g.
`client = MilvusClient(uri="http://localhost:19530", token="username:password")` for self-hosted Milvus server.
# Examples
Following is a simple demo showing how to use Milvus Lite for text search. There are more comprehensive [examples](https://github.com/milvus-io/bootcamp/tree/master/bootcamp/tutorials) for using Milvus Lite to build applications
such as [RAG](https://github.com/milvus-io/bootcamp/blob/master/bootcamp/tutorials/quickstart/build_RAG_with_milvus.ipynb), [image search](https://github.com/milvus-io/bootcamp/blob/master/bootcamp/tutorials/quickstart/image_search_with_milvus.ipynb), and using Milvus Lite in popular RAG framework such as [LangChain](https://github.com/milvus-io/bootcamp/blob/master/bootcamp/tutorials/integration/rag_with_milvus_and_langchain.ipynb) and [LlamaIndex](https://github.com/milvus-io/bootcamp/blob/master/bootcamp/tutorials/integration/rag_with_milvus_and_llamaindex.ipynb)!
```python
from pymilvus import MilvusClient
import numpy as np
client = MilvusClient("./milvus_demo.db")
client.create_collection(
collection_name="demo_collection",
dimension=384 # The vectors we will use in this demo has 384 dimensions
)
# Text strings to search from.
docs = [
"Artificial intelligence was founded as an academic discipline in 1956.",
"Alan Turing was the first person to conduct substantial research in AI.",
"Born in Maida Vale, London, Turing was raised in southern England.",
]
# For illustration, here we use fake vectors with random numbers (384 dimension).
vectors = [[ np.random.uniform(-1, 1) for _ in range(384) ] for _ in range(len(docs)) ]
data = [ {"id": i, "vector": vectors[i], "text": docs[i], "subject": "history"} for i in range(len(vectors)) ]
res = client.insert(
collection_name="demo_collection",
data=data
)
# This will exclude any text in "history" subject despite close to the query vector.
res = client.search(
collection_name="demo_collection",
data=[vectors[0]],
filter="subject == 'history'",
limit=2,
output_fields=["text", "subject"],
)
print(res)
# a query that retrieves all entities matching filter expressions.
res = client.query(
collection_name="demo_collection",
filter="subject == 'history'",
output_fields=["text", "subject"],
)
print(res)
# delete
res = client.delete(
collection_name="demo_collection",
filter="subject == 'history'",
)
print(res)
```
# Supported Features
## Functionality
Milvus Lite shares the same API as Milvus Standalone, Milvus Distributed and Zilliz Cloud, offering core features including:
- Insert/upsert operations
- Vector data persistence and collection management
- Dense, sparse, and hybrid vector search
- Metadata filtering
- Multi-vector support
## Index Type
Milvus Lite has limited index type support compared to other Milvus deployments:
- Prior to version 2.4.11:
- Only supports the [FLAT](https://milvus.io/docs/index.md?tab=floating#FLAT) index type
- Uses FLAT index regardless of the specified index type in collection creation
- Version 2.4.11 and later:
- Supports both [FLAT](https://milvus.io/docs/index.md?tab=floating#FLAT) and [IVF_FLAT](https://milvus.io/docs/index.md?tab=floating#IVFFLAT) index types
- For IVF_FLAT indexes:
- Data size < 100,000: Automatically uses FLAT index internally for better performance
- Data size ≥ 100,000: Constructs and uses IVF_FLAT index as specified
# Known Limitations
Milvus Lite does not support partitions, users/roles/RBAC, alias. To use those features, please choose other Milvus deployment types such as [Standalone](https://milvus.io/docs/install-overview.md#Milvus-Standalone), [Distributed](https://milvus.io/docs/install-overview.md#Milvus-Distributed) or [Zilliz Cloud](https://zilliz.com/cloud) (fully-managed Milvus).
# Migrating data from Milvus Lite
All data stored in Milvus Lite can be easily exported and loaded into other types of Milvus deployment, such as Milvus Standalone on Docker, Milvus Distributed on K8s, or fully-managed Milvus on [Zilliz Cloud](https://zilliz.com/cloud).
Milvus Lite provides a command line tool that can dump data into a json file, which can be imported into self-hosted [Milvus](https://github.com/milvus-io/milvus) or fully-managed Milvus on [Zilliz Cloud](https://zilliz.com/cloud).
```shell
pip install -U "pymilvus[bulk_writer]"
# The `milvus-lite` command line tool is already included in `milvus-lite` package which is part of "pymilvus", but it also needs some dependencies from `pymilvus[bulk_writer]` for dumping data.
milvus-lite dump -h
usage: milvus-lite dump [-h] [-d DB_FILE] [-c COLLECTION] [-p PATH]
optional arguments:
-h, --help show this help message and exit
-d DB_FILE, --db-file DB_FILE
milvus lite db file
-c COLLECTION, --collection COLLECTION
collection that need to be dumped
-p PATH, --path PATH dump file storage dir
```
The following example dumps all data from `demo_collection` collection that's stored in `./milvus_demo.db` (a user specified local file that persists data for Milvus Lite)
To export data:
```shell
milvus-lite dump -d ./milvus_demo.db -c demo_collection -p ./data_dir
# ./milvus_demo.db: milvus lite db file
# demo_collection: collection that need to be dumped
#./data_dir : dump file storage dir
```
You can use the dump file as input to upload data to Zilliz Cloud via [Data Import](https://docs.zilliz.com/docs/data-import), or a self-hosted Milvus server via [Bulk Insert](https://milvus.io/docs/import-data.md).
# Contributing
If you want to contribute to Milvus Lite, please read the [Contributing Guide](https://github.com/milvus-io/milvus-lite/blob/main/CONTRIBUTING.md) first.
# Report a bug
For any bug or feature request, please report it by submitting an issue in [milvus-lite](https://github.com/milvus-io/milvus-lite/issues/new/choose) repo.
# License
Milvus Lite is under the Apache 2.0 license. See the [LICENSE](https://github.com/milvus-io/milvus-lite/blob/main/LICENSE) file for details.
================================================
FILE: ci/e2e.groovy
================================================
#!/usr/bin/env groovy
int total_timeout_minutes = 60 * 5
pipeline {
options {
timestamps()
timeout(time: total_timeout_minutes, unit: 'MINUTES')
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
parallelsAlwaysFailFast()
preserveStashes(buildCount: 5)
disableConcurrentBuilds(abortPrevious: true)
}
agent {
kubernetes {
cloud '4am'
inheritFrom 'milvus-e2e-4am'
defaultContainer 'main'
yamlFile 'ci/rte-build.yaml'
customWorkspace '/home/jenkins/agent/workspace'
}
}
environment {
DOCKER_BUILDKIT = 1
}
stages {
stage('Build') {
steps {
container('main') {
script {
sh '''
MIRROR_URL="https://docker-nexus-ci.zilliz.cc" ./ci/set_docker_mirror.sh
'''
sh '''
docker run --net=host \
-e CONAN_USER_HOME=/root/ -v \$PWD:/root/milvus-lite -v /root/.conan:/root/.conan -w /root/milvus-lite \
milvusdb/milvus-env:lite-main bash ci/entrypoint.sh
'''
}
}
}
}
stage('arhive Artifacts ') {
steps {
container('main') {
archiveArtifacts artifacts: 'python/dist/*.whl',
allowEmptyArchive: true,
fingerprint: true,
onlyIfSuccessful: true
}
}
}
stage('install wheel') {
steps {
container('pytest') {
sh '''
pip install ./python/dist/*.whl
'''
}
}
}
stage('Test') {
steps {
container('pytest') {
sh '''
bash ci/test.sh -m e2e
'''
}
}
}
}
post {
unsuccessful {
container('jnlp') {
dir('ci') {
script {
def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ')
emailext subject: '$DEFAULT_SUBJECT',
body: '$DEFAULT_CONTENT',
recipientProviders: [developers(), culprits()],
replyTo: '$DEFAULT_REPLYTO',
to: "${authorEmail},devops@zilliz.com"
}
}
}
}
}
}
================================================
FILE: ci/entrypoint.sh
================================================
#!/bin/bash
git config --global --add safe.directory /root/milvus-lite/thirdparty/milvus;
cd python
python3 -m build --wheel
================================================
FILE: ci/get_author_email.sh
================================================
#!/bin/bash
# author emaill will be jenkins's email, when the pr branch is not fast forward compared with master branch
# Exit immediately for non zero status
set -e
function get_author_email(){
email=$(git --no-pager show -s --format=\'%ae\' HEAD )
# Get last commit author when Jenkins submit a merge commit
if [[ "${email}" == \'nobody@nowhere\' ]]; then
email=$(git --no-pager show -s --format=\'%ae\' HEAD^ )
fi
echo ${email} | sed $'s/\'//g'
}
get_author_email
================================================
FILE: ci/nightly.groovy
================================================
#!/usr/bin/env groovy
int total_timeout_minutes = 60 * 5
// When scheduling a job that gets automatically triggered by changes,
// you need to include a [cronjob] tag within the commit message.
String cron_timezone = 'TZ=Asia/Shanghai'
String cron_string = BRANCH_NAME == "2.4" ? "50 3 * * * " : ""
pipeline {
triggers {
cron """${cron_timezone}
${cron_string}"""
}
options {
timestamps()
timeout(time: total_timeout_minutes, unit: 'MINUTES')
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
parallelsAlwaysFailFast()
preserveStashes(buildCount: 5)
disableConcurrentBuilds(abortPrevious: true)
}
agent {
kubernetes {
cloud '4am'
inheritFrom 'milvus-e2e-4am'
defaultContainer 'main'
yamlFile 'ci/rte-build.yaml'
customWorkspace '/home/jenkins/agent/workspace'
}
}
environment {
DOCKER_BUILDKIT = 1
}
stages {
stage('Build') {
steps {
container('main') {
script {
sh '''
MIRROR_URL="https://docker-nexus-ci.zilliz.cc" ./ci/set_docker_mirror.sh
'''
sh '''
docker run --net=host \
-e CONAN_USER_HOME=/root/ -v \$PWD:/root/milvus-lite -v /root/.conan:/root/.conan -w /root/milvus-lite \
milvusdb/milvus-env:lite-manylinux2014 bash ci/entrypoint.sh
# the image milvusdb/milvus-env:lite-manylinux2014 is from scripts/Dockerfile
'''
}
}
}
}
stage('arhive Artifacts ') {
steps {
container('main') {
archiveArtifacts artifacts: 'python/dist/*.whl',
allowEmptyArchive: true,
fingerprint: true,
onlyIfSuccessful: true
}
}
}
stage('install wheel') {
steps {
container('pytest') {
sh '''
pip install ./python/dist/*.whl
'''
}
}
}
stage('Test') {
steps {
container('pytest') {
sh '''
bash ci/test.sh -m nightly
'''
}
}
}
}
post {
unsuccessful {
container('jnlp') {
dir('ci') {
script {
def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ')
emailext subject: '$DEFAULT_SUBJECT',
body: '$DEFAULT_CONTENT',
recipientProviders: [developers(), culprits()],
replyTo: '$DEFAULT_REPLYTO',
to: "${authorEmail},devops@zilliz.com"
}
}
}
}
}
}
================================================
FILE: ci/rte-build.yaml
================================================
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus-e2e
task: milvus-e2e-build
namespace: milvus-ci
spec:
enableServiceLinks: false
nodeSelector:
kubernetes.io/hostname: 4am-node16
containers:
- name: main
image: milvusdb/krte:20211213-dcc15e9
env:
- name: DOCKER_IN_DOCKER_ENABLED
value: "true"
- name: DOCKER_VOLUME_DIRECTORY
value: "/mnt/disk/.docker"
- name: DOCKER_CLIENT_TIMEOUT
value: 240
- name: COMPOSE_HTTP_TIMEOUT
value: 240
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
tty: true
securityContext:
privileged: true
args: ["cat"]
resources:
limits:
cpu: "6"
memory: 12Gi
requests:
cpu: "0.5"
memory: 2Gi
volumeMounts:
- mountPath: /docker-graph
name: docker-graph
- mountPath: /var/lib/docker
name: docker-root
- mountPath: /lib/modules
name: modules
readOnly: true
- mountPath: /sys/fs/cgroup
name: cgroup
- mountPath: /root/.conan
name: build-cache
- mountPath: /tmp/ci_logs
name: ci-logs
- name: pytest
image: harbor.milvus.io/dockerhub/milvusdb/pytest:20240516-9536e79
resources:
limits:
cpu: "6"
memory: 12Gi
requests:
cpu: "0.5"
memory: 5Gi
volumeMounts:
- mountPath: /tmp/ci_logs
name: ci-logs
volumes:
- emptyDir: {}
name: docker-graph
- emptyDir: {}
name: docker-root
- hostPath:
path: /root/.conan
type: DirectoryOrCreate
name: build-cache
- hostPath:
path: /lib/modules
type: Directory
name: modules
- hostPath:
path: /sys/fs/cgroup
type: Directory
name: cgroup
- name: ci-logs
nfs:
path: /volume1/ci-logs
# path: /volume1/4am-logs
server: 172.16.70.249
================================================
FILE: ci/set_docker_mirror.sh
================================================
#!/usr/bin/env bash
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Use Internal docker mirror to solve https://www.docker.com/increase-rate-limits
# Exit immediately for non zero status
set -e
# Use nexus as docker mirror registry
MIRROR_URL="${MIRROR_URL:-http://nexus-nexus-repository-manager-docker-5000.nexus:5000}"
#MIRROR_URL="http://nexus-nexus-repository-manager-docker-5000.nexus:5000"
# Add registry mirror config into docker daemon
set_daemon_json_file(){
DOCKER_DAEMON_JSON_FILE="/etc/docker/daemon.json"
mkdir -p "/etc/docker"
echo "{\"registry-mirrors\": [\"${MIRROR_URL}\"],\"insecure-registries\":[\"${MIRROR_URL}\"]}" | tee ${DOCKER_DAEMON_JSON_FILE}
}
restart_docker () {
echo "set-mirror.sh] service docker start"
docker ps -aq | xargs -r docker rm -f || true
service docker stop || true
service docker start
while true; do
# docker ps -q should only work if the daemon is ready
docker info > /dev/null 2>&1 && break
if [[ ${WAIT_N} -lt 5 ]]; then
WAIT_N=$((WAIT_N+1))
echo " set-mirror.sh] Waiting for Docker to be ready, sleeping for ${WAIT_N} seconds ..."
sleep ${WAIT_N}
else
echo "set-mirror.sh] [SETUP] Reached maximum attempts, not waiting any longer ..."
break
fi
done
echo "Show Docker Info With mirror url"
docker info
}
set_mirror(){
set_daemon_json_file
restart_docker
echo "Success."
exit 0
}
set_mirror
================================================
FILE: ci/test.sh
================================================
#!/bin/bash
# Function to display usage
usage() {
echo "Usage: $0 -m {nightly|e2e}"
exit 1
}
# Check if the correct number of arguments is provided
if [ $# -ne 2 ]; then
usage
fi
# Initialize variables
MODE=""
# Parse the command-line arguments
while getopts ":m:" opt; do
case ${opt} in
m )
MODE=$OPTARG
;;
\? )
usage
;;
esac
done
# Validate the MODE value
if [ "$MODE" != "nightly" ] && [ "$MODE" != "e2e" ]; then
usage
fi
# Install dependencies
export PIP_TRUSTED_HOST="nexus-ci.zilliz.cc"
export PIP_INDEX_URL="https://nexus-ci.zilliz.cc/repository/pypi-all/simple"
export PIP_INDEX="https://nexus-ci.zilliz.cc/repository/pypi-all/pypi"
export PIP_FIND_LINKS="https://nexus-ci.zilliz.cc/repository/pypi-all/pypi"
python3 -m pip install --no-cache-dir -r tests/requirements.txt --timeout 300 --retries 6
cd tests/milvus_lite
# Main logic based on the MODE
if [ "$MODE" == "nightly" ]; then
echo "Running in nightly mode"
pytest -v -m 'not L3' --enable_milvus_local_api ./lite-nighlty.db # nighlty
elif [ "$MODE" == "e2e" ]; then
echo "Running in e2e mode"
pytest -s -v --tags L0 --enable_milvus_local_api lite-e2e.db # pr 合并时
fi
================================================
FILE: cmake/Findantlr4-runtime.cmake
================================================
FIND_PATH(antlr4-cppruntime_INCLUDES antlr4-runtime.h PATH_SUFFIXES antlr4-runtime REQUIRED)
FIND_LIBRARY(antlr4-cppruntime_LIBRARIES antlr4-runtime REQUIRED)
================================================
FILE: cmake/milvus-storage.cmake
================================================
# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License
file(GLOB_RECURSE SRC_FILES thirdparty/milvus-storage/cpp/src/*.cpp thirdparty/milvus-storage/cpp/src/*.cc)
add_library(milvus-storage ${SRC_FILES})
target_include_directories(milvus-storage PUBLIC BEFORE thirdparty/milvus-storage/cpp/include/milvus-storage thirdparty/milvus-storage/cpp/src)
target_link_libraries(milvus-storage PUBLIC
arrow::arrow
Boost::boost
protobuf::protobuf
glog::glog
)
================================================
FILE: conanfile.py
================================================
from conans import ConanFile
from conan.tools.cmake import CMake
class MilvusLiteConan(ConanFile):
settings = "os", "compiler", "build_type", "arch"
requires = (
# gtest
"gtest/1.13.0",
# glog
"xz_utils/5.4.0#a6d90890193dc851fa0d470163271c7a",
"zlib/1.2.13#df233e6bed99052f285331b9f54d9070",
"glog/0.6.0#d22ebf9111fed68de86b0fa6bf6f9c3f",
# protobuf
"protobuf/3.21.4#fd372371d994b8585742ca42c12337f9",
# folly
"fmt/9.1.0#95259249fb7ef8c6b5674a40b00abba3",
"folly/2023.10.30.08@milvus/dev#81d7729cd4013a1b708af3340a3b04d9",
# antlr
"antlr4-cppruntime/4.13.1",
# sqlite
"sqlitecpp/3.3.1",
"onetbb/2021.9.0#4a223ff1b4025d02f31b65aedf5e7f4a",
"nlohmann_json/3.11.2#ffb9e9236619f1c883e36662f944345d",
"boost/1.82.0#744a17160ebb5838e9115eab4d6d0c06",
"openssl/3.1.2#02594c4c0a6e2b4feb3cd15119993597",
"libcurl/7.86.0#bbc887fae3341b3cb776c601f814df05",
"grpc/1.50.1@milvus/dev#75103960d1cac300cf425ccfccceac08",
"prometheus-cpp/1.1.0#ea9b101cb785943adb40ad82eda7856c",
"re2/20230301#f8efaf45f98d0193cd0b2ea08b6b4060",
# "simdjson/3.7.0",
"arrow/15.0.0#0456d916ff25d509e0724c5b219b4c45",
"double-conversion/3.2.1#640e35791a4bac95b0545e2f54b7aceb",
"marisa/0.2.6#68446854f5a420672d21f21191f8e5af",
"zstd/1.5.4#308b8b048f9a3823ce248f9c150cc889",
"yaml-cpp/0.7.0#9c87b3998de893cf2e5a08ad09a7a6e0",
"libdwarf/20191104#7f56c6c7ccda5fadf5f28351d35d7c01",
"rapidjson/cci.20230929#624c0094d741e6a3749d2e44d834b96c",
"roaring/3.0.0#25a703f80eda0764a31ef939229e202d",
"libevent/2.1.12#4fd19d10d3bed63b3a8952c923454bc0",
)
generators = {"cmake", "cmake_find_package"}
default_options = {
"glog:with_gflags": True,
"glog:shared": True,
"gtest:build_gmock": False,
"onetbb:tbbmalloc": False,
"onetbb:tbbproxy": False,
"boost:without_locale": True,
"boost:without_test": True,
"boost:without_stacktrace": True,
"fmt:header_only": True,
"prometheus-cpp:with_pull": False,
"double-conversion:shared": True,
"arrow:filesystem_layer": True,
"arrow:parquet": True,
"arrow:compute": True,
"arrow:with_re2": True,
"arrow:with_zstd": False,
"arrow:with_boost": True,
"arrow:with_thrift": True,
"arrow:with_jemalloc": True,
"arrow:shared": False,
"arrow:with_s3": False,
"libcurl:with_ssl": False,
}
def configure(self):
if self.settings.os in ["Macos", "Android"]:
self.options["arrow"].with_jemalloc = False
if self.settings.compiler == "gcc":
if self.settings.compiler.libcxx == "libstdc++":
raise Exception("This package is only compatible with libstdc++11")
def config_options(self):
if self.settings.os != "Macos":
self.options["onetbb"].tbbbind = False
def requirements(self):
if self.settings.os not in ["Macos", "Android"]:
self.requires("libunwind/1.7.2")
if self.settings.os == "Android":
self.requires("openblas/0.3.27")
def imports(self):
self.copy("*.so*", "./lib", "lib")
self.copy("*.dylib", "./lib", "lib")
def build(self):
target = "11.0"
self.run("export MACOSX_DEPLOYMENT_TARGET={}".format(target))
================================================
FILE: examples/bfloat16_example.py
================================================
import random
import tensorflow as tf
from pymilvus import (
connections,
utility,
FieldSchema, CollectionSchema, DataType,
Collection,
)
bf16_index_types = ["FLAT"]
default_bf16_index_params = [{"nlist": 128}]
def gen_bf16_vectors(num, dim):
raw_vectors = []
bf16_vectors = []
for _ in range(num):
raw_vector = [random.random() for _ in range(dim)]
raw_vectors.append(raw_vector)
bf16_vector = tf.cast(raw_vector, dtype=tf.bfloat16).numpy()
bf16_vectors.append(bf16_vector)
return raw_vectors, bf16_vectors
def bf16_vector_search():
connections.connect(uri="./local_test.db")
int64_field = FieldSchema(name="int64", dtype=DataType.INT64, is_primary=True, auto_id=True)
dim = 128
nb = 3000
vector_field_name = "bfloat16_vector"
bf16_vector = FieldSchema(name=vector_field_name, dtype=DataType.BFLOAT16_VECTOR, dim=dim)
schema = CollectionSchema(fields=[int64_field, bf16_vector])
if utility.has_collection("hello_milvus_fp16"):
utility.drop_collection("hello_milvus_fp16")
hello_milvus = Collection("hello_milvus_fp16", schema, consistency_level="Strong")
_, vectors = gen_bf16_vectors(nb, dim)
hello_milvus.insert([vectors[:6]])
rows = [
{vector_field_name: vectors[6]},
{vector_field_name: vectors[7]},
{vector_field_name: vectors[8]},
{vector_field_name: vectors[9]},
{vector_field_name: vectors[10]},
{vector_field_name: vectors[11]},
]
hello_milvus.insert(rows)
hello_milvus.flush()
for i, index_type in enumerate(bf16_index_types):
index_params = default_bf16_index_params[i]
hello_milvus.create_index(vector_field_name,
index_params={"index_type": index_type, "params": index_params, "metric_type": "L2"})
hello_milvus.load()
print("index_type = ", index_type)
res = hello_milvus.search(vectors[0:10], vector_field_name, {"metric_type": "L2"}, limit=1)
print(res)
hello_milvus.release()
hello_milvus.drop_index()
hello_milvus.drop()
if __name__ == "__main__":
bf16_vector_search()
================================================
FILE: examples/binary_example.py
================================================
import random
import numpy as np
from pymilvus import (
connections,
utility,
FieldSchema, CollectionSchema, DataType,
Collection,
)
bin_index_types = ["BIN_FLAT"]
default_bin_index_params = [{"nlist": 128}, {"nlist": 128}]
def gen_binary_vectors(num, dim):
raw_vectors = []
binary_vectors = []
for _ in range(num):
raw_vector = [random.randint(0, 1) for _ in range(dim)]
raw_vectors.append(raw_vector)
# packs a binary-valued array into bits in a unit8 array, and bytes array_of_ints
binary_vectors.append(bytes(np.packbits(raw_vector, axis=-1).tolist()))
return raw_vectors, binary_vectors
def binary_vector_search():
connections.connect(uri='./local_test.db')
int64_field = FieldSchema(name="int64", dtype=DataType.INT64, is_primary=True, auto_id=True)
dim = 128
nb = 3000
vector_field_name = "binary_vector"
binary_vector = FieldSchema(name=vector_field_name, dtype=DataType.BINARY_VECTOR, dim=dim)
schema = CollectionSchema(fields=[int64_field, binary_vector], enable_dynamic_field=True)
has = utility.has_collection("hello_milvus_bin")
if has:
hello_milvus = Collection("hello_milvus_bin")
hello_milvus.drop()
else:
hello_milvus = Collection("hello_milvus_bin", schema)
_, vectors = gen_binary_vectors(nb, dim)
rows = [
{vector_field_name: vectors[0]},
{vector_field_name: vectors[1]},
{vector_field_name: vectors[2]},
{vector_field_name: vectors[3]},
{vector_field_name: vectors[4]},
{vector_field_name: vectors[5]},
]
hello_milvus.insert(rows)
hello_milvus.flush()
for i, index_type in enumerate(bin_index_types):
index_params = default_bin_index_params[i]
hello_milvus.create_index(vector_field_name,
index_params={"index_type": index_type, "params": index_params, "metric_type": "HAMMING"})
hello_milvus.load()
print("index_type = ", index_type)
res = hello_milvus.search(vectors[:1], vector_field_name, {"metric_type": "HAMMING"}, limit=1)
print("res = ", res)
hello_milvus.release()
hello_milvus.drop_index()
hello_milvus.drop()
if __name__ == "__main__":
binary_vector_search()
================================================
FILE: examples/bm25.py
================================================
from pymilvus import (
MilvusClient,
Function,
FunctionType,
DataType,
)
fmt = "\n=== {:30} ===\n"
collection_name = "doc_in_doc_out"
milvus_client = MilvusClient(uri="./bm25.db")
has_collection = milvus_client.has_collection(collection_name, timeout=5)
if has_collection:
milvus_client.drop_collection(collection_name)
schema = milvus_client.create_schema()
schema.add_field("id", DataType.INT64, is_primary=True, auto_id=False)
schema.add_field(
"document_content", DataType.VARCHAR, max_length=9000, enable_analyzer=True
)
schema.add_field("sparse_vector", DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
name="bm25_fn",
input_field_names=["document_content"],
output_field_names="sparse_vector",
function_type=FunctionType.BM25,
)
schema.add_function(bm25_function)
index_params = milvus_client.prepare_index_params()
index_params.add_index(
field_name="sparse_vector",
index_name="sparse_inverted_index",
index_type="SPARSE_INVERTED_INDEX",
metric_type="BM25",
params={"bm25_k1": 1.2, "bm25_b": 0.75},
)
ret = milvus_client.create_collection(
collection_name,
schema=schema,
index_params=index_params,
consistency_level="Strong",
)
print("----")
print(ret)
print(fmt.format(" all collections "))
print(milvus_client.list_collections())
print(fmt.format(f"schema of collection {collection_name}"))
print(milvus_client.describe_collection(collection_name))
rows = [
{"id": 1, "document_content": "hello world"},
{"id": 2, "document_content": "hello milvus"},
{"id": 3, "document_content": "hello zilliz"},
]
print(fmt.format("Start inserting entities"))
insert_result = milvus_client.insert(collection_name, rows, progress_bar=True)
print(fmt.format("Inserting entities done"))
print(insert_result)
texts_to_search = ["hello"]
# search_params = {"metric_type": "BM25", "params": {"bm25_avgdl": 10.0}}
search_params = {"metric_type": "BM25"}
print(fmt.format(f"Start search with retrieve several fields."))
result = milvus_client.search(
collection_name,
texts_to_search,
limit=3,
output_fields=["document_content"],
search_params=search_params,
)
print(result)
for hits in result:
for hit in hits:
print(f"hit: {hit}")
print(fmt.format("Start query by specifying primary keys"))
query_results = milvus_client.query(collection_name, ids=[3])
print(query_results[0])
upsert_ret = milvus_client.upsert(
collection_name, {"id": 2, "document_content": "hello milvus again"}
)
print(upsert_ret)
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(
collection_name, filter="document_content == 'hello milvus again'"
)
for ret in query_results:
print(ret)
print(f"start to delete by specifying filter in collection {collection_name}")
delete_result = milvus_client.delete(collection_name, ids=[3])
print(delete_result)
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(
collection_name, filter="document_content == 'hello zilliz'"
)
print(f"Query results after deletion: {query_results}")
milvus_client.drop_collection(collection_name)
================================================
FILE: examples/customize_schema.py
================================================
import time
import numpy as np
from pymilvus import (
MilvusClient,
DataType
)
fmt = "\n=== {:30} ===\n"
dim = 8
collection_name = "hello_milvus"
milvus_client = MilvusClient("./local_test.db")
has_collection = milvus_client.has_collection(collection_name, timeout=5)
if has_collection:
milvus_client.drop_collection(collection_name)
schema = milvus_client.create_schema(enable_dynamic_field=True)
schema.add_field("id", DataType.INT64, is_primary=True)
schema.add_field("embeddings", DataType.FLOAT_VECTOR, dim=dim)
schema.add_field("title", DataType.VARCHAR, max_length=64)
index_params = milvus_client.prepare_index_params()
index_params.add_index(field_name = "embeddings", metric_type="L2")
milvus_client.create_collection(collection_name, schema=schema, index_params=index_params, consistency_level="Strong")
print(fmt.format(" all collections "))
print(milvus_client.list_collections())
print(fmt.format(f"schema of collection {collection_name}"))
print(milvus_client.describe_collection(collection_name))
rng = np.random.default_rng(seed=19530)
rows = [
{"id": 1, "embeddings": rng.random((1, dim))[0], "a": 100, "title": "t1"},
{"id": 2, "embeddings": rng.random((1, dim))[0], "b": 200, "title": "t2"},
{"id": 3, "embeddings": rng.random((1, dim))[0], "c": 300, "title": "t3"},
{"id": 4, "embeddings": rng.random((1, dim))[0], "d": 400, "title": "t4"},
{"id": 5, "embeddings": rng.random((1, dim))[0], "e": 500, "title": "t5"},
{"id": 6, "embeddings": rng.random((1, dim))[0], "f": 600, "title": "t6"},
]
print(fmt.format("Start inserting entities"))
insert_result = milvus_client.insert(collection_name, rows)
print(fmt.format("Inserting entities done"))
print(insert_result)
print(fmt.format("Start load collection "))
milvus_client.load_collection(collection_name)
print(fmt.format("Start query by specifying primary keys"))
query_results = milvus_client.query(collection_name, ids=[2])
print(query_results[0])
print(fmt.format("Start get by specifying primary keys"))
get_results = milvus_client.get(collection_name, ids=[2])
print(get_results[0])
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(collection_name, filter= "f == 600 or title == 't2'")
for ret in query_results:
print(ret)
rng = np.random.default_rng(seed=19530)
vectors_to_search = rng.random((1, dim))
print(fmt.format(f"Start search with retrieve serveral fields."))
result = milvus_client.search(collection_name, vectors_to_search, limit=3, output_fields=["pk", "a", "b"])
for hits in result:
for hit in hits:
print(f"hit: {hit}")
milvus_client.drop_collection(collection_name)
================================================
FILE: examples/customize_schema_auto_id.py
================================================
import numpy as np
from pymilvus import (
MilvusClient,
DataType
)
fmt = "\n=== {:30} ===\n"
dim = 8
collection_name = "hello_milvus"
milvus_client = MilvusClient(uri='./local_test.db')
has_collection = milvus_client.has_collection(collection_name, timeout=5)
if has_collection:
milvus_client.drop_collection(collection_name)
schema = milvus_client.create_schema(enable_dynamic_field=True, auto_id=True)
schema.add_field("id", DataType.INT64, is_primary=True)
schema.add_field("embeddings", DataType.FLOAT_VECTOR, dim=dim)
schema.add_field("title", DataType.VARCHAR, max_length=64)
index_params = milvus_client.prepare_index_params()
index_params.add_index(field_name = "embeddings", metric_type="L2")
milvus_client.create_collection(collection_name, schema=schema, index_params=index_params, consistency_level="Strong")
print(fmt.format(" all collections "))
print(milvus_client.list_collections())
print(fmt.format(f"schema of collection {collection_name}"))
print(milvus_client.describe_collection(collection_name))
rng = np.random.default_rng(seed=19530)
rows = [
{"embeddings": rng.random((1, dim))[0], "a": 100, "title": "t1"},
{"embeddings": rng.random((1, dim))[0], "b": 200, "title": "t2"},
{"embeddings": rng.random((1, dim))[0], "c": 300, "title": "t3"},
{"embeddings": rng.random((1, dim))[0], "d": 400, "title": "t4"},
{"embeddings": rng.random((1, dim))[0], "e": 500, "title": "t5"},
{"embeddings": rng.random((1, dim))[0], "f": 600, "title": "t6"},
]
print(fmt.format("Start inserting entities"))
insert_result = milvus_client.insert(collection_name, rows)
print(fmt.format("Inserting entities done"))
print(insert_result)
print(fmt.format("Start load collection "))
milvus_client.load_collection(collection_name)
print(fmt.format("Start query by specifying primary keys"))
query_results = milvus_client.query(collection_name, ids=insert_result['ids'][0])
print(query_results[0])
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(collection_name, filter= "f == 600 or title == 't2'")
for ret in query_results:
print(ret)
rng = np.random.default_rng(seed=19530)
vectors_to_search = rng.random((1, dim))
print(fmt.format(f"Start search with retrieve serveral fields."))
result = milvus_client.search(collection_name, vectors_to_search, limit=3, output_fields=["pk", "a", "b"])
for hits in result:
for hit in hits:
print(f"hit: {hit}")
milvus_client.drop_collection(collection_name)
================================================
FILE: examples/dynamic_field.py
================================================
import time
import numpy as np
from pymilvus import (
connections,
utility,
FieldSchema, CollectionSchema, DataType,
Collection,
)
fmt = "\n=== {:30} ===\n"
dim = 8
print(fmt.format("start connecting to Milvus"))
connections.connect("default", uri='./local_test.db')
has = utility.has_collection("hello_milvus")
print(f"Does collection hello_milvus exist in Milvus: {has}")
if has:
utility.drop_collection("hello_milvus")
fields = [
FieldSchema(name="pk", dtype=DataType.VARCHAR, is_primary=True, auto_id=False, max_length=100),
FieldSchema(name="random", dtype=DataType.DOUBLE),
FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=dim)
]
schema = CollectionSchema(fields, "hello_milvus is the simplest demo to introduce the APIs", enable_dynamic_field=True)
print(fmt.format("Create collection `hello_milvus`"))
hello_milvus = Collection("hello_milvus", schema, consistency_level="Strong")
################################################################################
# 3. insert data
hello_milvus2 = Collection("hello_milvus")
print(fmt.format("Start inserting entities"))
rng = np.random.default_rng(seed=19530)
rows = [
{"pk": "1", "random": 1.0, "embeddings": rng.random((1, dim))[0], "a": 1},
{"pk": "2", "random": 1.0, "embeddings": rng.random((1, dim))[0], "b": 1},
{"pk": "3", "random": 1.0, "embeddings": rng.random((1, dim))[0], "c": 1},
{"pk": "4", "random": 1.0, "embeddings": rng.random((1, dim))[0], "d": 1},
{"pk": "5", "random": 1.0, "embeddings": rng.random((1, dim))[0], "e": 1},
{"pk": "6", "random": 1.0, "embeddings": rng.random((1, dim))[0], "f": 1},
]
insert_result = hello_milvus.insert(rows)
hello_milvus.insert({"pk": "7", "random": 1.0, "embeddings": rng.random((1, dim))[0], "g": 1})
hello_milvus.flush()
print(f"Number of entities in Milvus: {hello_milvus.num_entities}") # check the num_entites
# 4. create index
print(fmt.format("Start Creating index IVF_FLAT"))
index = {
"index_type": "IVF_FLAT",
"metric_type": "L2",
"params": {"nlist": 128},
}
hello_milvus.create_index("embeddings", index)
print(fmt.format("Start loading"))
hello_milvus.load()
# -----------------------------------------------------------------------------
# search based on vector similarity
print(fmt.format("Start searching based on vector similarity"))
rng = np.random.default_rng(seed=19530)
vectors_to_search = rng.random((1, dim))
search_params = {
"metric_type": "L2",
"params": {"nprobe": 10},
}
start_time = time.time()
result = hello_milvus.search(vectors_to_search, "embeddings", search_params, limit=3, output_fields=["pk", "embeddings"])
end_time = time.time()
for hits in result:
for hit in hits:
print(f"hit: {hit}")
result = hello_milvus.search(vectors_to_search, "embeddings", search_params, limit=3, output_fields=["pk", "embeddings", "$meta"])
for hits in result:
for hit in hits:
print(f"hit: {hit}")
expr = f'pk in ["1" , "2"] || g == 1'
print(fmt.format(f"Start query with expr `{expr}`"))
result = hello_milvus.query(expr=expr, output_fields=["random", "a", "g"])
for hit in result:
print("hit:", hit)
###############################################################################
# 7. drop collection
print(fmt.format("Drop collection `hello_milvus`"))
utility.drop_collection("hello_milvus")
================================================
FILE: examples/example_group_by.py
================================================
from pymilvus import CollectionSchema, FieldSchema, Collection, connections, DataType, Partition, utility
import random
import numpy as np
import secrets
def generate_random_hex_string(length):
return secrets.token_hex(length // 2)
connections.connect("default", uri="milvus.db")
dim = 128
clean_exist = False
prepare_data = True
fields = [
FieldSchema(name="pk", dtype=DataType.INT64, is_primary=True),
FieldSchema(name="int64", dtype=DataType.INT64),
FieldSchema(name="float", dtype=DataType.FLOAT),
FieldSchema(name="float_vector", dtype=DataType.FLOAT_VECTOR, dim=dim),
FieldSchema(name="bool", dtype=DataType.BOOL),
FieldSchema(name="string", dtype=DataType.VARCHAR, max_length=512)
]
schema = CollectionSchema(fields=fields)
collection_name = 'test_group_by_' + generate_random_hex_string(24)
if clean_exist and utility.has_collection(collection_name):
utility.drop_collection(collection_name)
collection = Collection(collection_name, schema=schema)
nb = 1500
batch_num = 3
vectors = [[random.random() for _ in range(dim)] for _ in range(nb)]
if prepare_data:
for i in range(batch_num):
data = [
[i for i in range(nb * i, nb * (i + 1))],
[i % 33 for i in range(nb)],
[np.float32(i) for i in range(nb)],
vectors,
[bool(random.randrange(2)) for i in range(nb)],
[str(i % 44) for i in range(nb * i, nb * (i + 1))],
]
collection.insert(data)
print("insert data done")
collection.flush()
collection.create_index("float_vector", {"metric_type": "COSINE"})
# create collection and load
collection.load()
batch_size = 100
search_params = {"metric_type": "COSINE"}
result = collection.search(vectors[:3], "float_vector", search_params, limit=batch_size, timeout=600,
output_fields=["int64", "string"], group_by_field="string") #set up group_by_field
for i in range(len(result)):
resultI = result[i]
print(f"---result{i}_size:{len(result[i])}-------------------------")
for j in range(len(resultI)):
print(resultI[j])
print("----------------------------")
================================================
FILE: examples/float16_example.py
================================================
import random
import numpy as np
from pymilvus import (
connections,
utility,
FieldSchema, CollectionSchema, DataType,
Collection,
)
from milvus_lite.server_manager import server_manager_instance
fp16_index_types = ["FLAT"]
default_fp16_index_params = [{"nlist": 128}]
def gen_fp16_vectors(num, dim):
raw_vectors = []
fp16_vectors = []
for _ in range(num):
raw_vector = [random.random() for _ in range(dim)]
raw_vectors.append(raw_vector)
fp16_vector = np.array(raw_vector, dtype=np.float16)
fp16_vectors.append(fp16_vector)
return raw_vectors, fp16_vectors
def fp16_vector_search():
connections.connect(uri='./local_test.db')
int64_field = FieldSchema(name="int64", dtype=DataType.INT64, is_primary=True, auto_id=True)
dim = 128
nb = 3000
vector_field_name = "float16_vector"
fp16_vector = FieldSchema(name=vector_field_name, dtype=DataType.FLOAT16_VECTOR, dim=dim)
schema = CollectionSchema(fields=[int64_field, fp16_vector])
if utility.has_collection("hello_milvus_fp16"):
utility.drop_collection("hello_milvus_fp16")
hello_milvus = Collection("hello_milvus_fp16", schema)
_, vectors = gen_fp16_vectors(nb, dim)
hello_milvus.insert([vectors[:6]])
rows = [
{vector_field_name: vectors[6]},
{vector_field_name: vectors[7]},
{vector_field_name: vectors[8]},
{vector_field_name: vectors[9]},
{vector_field_name: vectors[10]},
{vector_field_name: vectors[11]},
]
hello_milvus.insert(rows)
hello_milvus.flush()
for i, index_type in enumerate(fp16_index_types):
index_params = default_fp16_index_params[i]
hello_milvus.create_index(vector_field_name,
index_params={"index_type": index_type, "params": index_params, "metric_type": "L2"})
hello_milvus.load()
print("index_type = ", index_type)
res = hello_milvus.search(vectors[0:10], vector_field_name, {"metric_type": "L2"}, limit=1)
print(res)
hello_milvus.release()
hello_milvus.drop_index()
hello_milvus.drop()
if __name__ == "__main__":
fp16_vector_search()
================================================
FILE: examples/fuzzy_match.py
================================================
from pymilvus import (
connections,
FieldSchema, CollectionSchema, DataType,
Collection,
)
DIMENSION = 8
COLLECTION_NAME = "books2"
connections.connect(uri="./local_test.db")
fields = [
FieldSchema(name='id', dtype=DataType.INT64, is_primary=True),
FieldSchema(name='title', dtype=DataType.VARCHAR, max_length=200),
FieldSchema(name='release_year', dtype=DataType.INT64),
FieldSchema(name='embeddings', dtype=DataType.FLOAT_VECTOR, dim=DIMENSION),
]
schema = CollectionSchema(fields=fields, enable_dynamic_field=True)
collection = Collection(name=COLLECTION_NAME, schema=schema)
data_rows = [
{
"id": 1,
"title": "Lord of the Flies",
"release_year": 1954,
"embeddings": [0.64, 0.44, 0.13, 0.47, 0.74, 0.03, 0.32, 0.6],
},
{
"id": 2,
"title": "The Great Gatsby",
"release_year": 1925,
"embeddings": [0.9, 0.45, 0.18, 0.43, 0.4, 0.4, 0.7, 0.24],
},
{
"id": 3,
"title": "The Catcher in the Rye",
"release_year": 1951,
"embeddings": [0.43, 0.57, 0.43, 0.88, 0.84, 0.69, 0.27, 0.98],
},
{
"id": 4,
"title": "Flipped",
"release_year": 2010,
"embeddings": [0.84, 0.69, 0.27, 0.43, 0.57, 0.43, 0.88, 0.98],
},
]
collection.insert(data_rows)
collection.create_index(
"embeddings", {"index_type": "FLAT", "metric_type": "L2"})
collection.load()
# prefix match.
res = collection.query(expr='title like "The%"', output_fields=["id", "title"])
print(res)
# infix match.
res = collection.query(expr='title like "%the%"', output_fields=["id", "title"])
print(res)
# postfix match.
res = collection.query(expr='title like "%Rye"', output_fields=["id", "title"])
print(res)
# _ match any one and only one character.
res = collection.query(expr='title like "Flip_ed"', output_fields=["id", "title"])
print(res)
# you can create inverted index to accelerate the fuzzy match.
collection.release()
collection.create_index(
"title", {"index_type": "INVERTED"})
collection.load()
# _ match any one and only one character.
res = collection.query(expr='title like "Flip_ed"', output_fields=["id", "title"])
print(res)
================================================
FILE: examples/hello_hybrid_bm25.py
================================================
# A demo showing hybrid semantic search with dense and full text search with BM25
# using Milvus.
#
# You can optionally choose to use the BGE-M3 model to embed the text as dense
# vectors, or simply use random generated vectors as an example.
#
# You can also use the BGE CrossEncoder model to rerank the search results.
#
# Note that the full text search feature is only available in Milvus 2.4.0 or
# higher version. Make sure you follow https://milvus.io/docs/install_standalone-docker.md
# to set up the latest version of Milvus in your local environment.
# To connect to Milvus server, you need the python client library called pymilvus.
# To use BGE-M3 model, you need to install the optional `model` module in pymilvus.
# You can get them by simply running the following commands:
#
# pip install pymilvus
# pip install pymilvus[model]
# If true, use BGE-M3 model to generate dense vectors.
# If false, use random numbers to compose dense vectors.
use_bge_m3 = False
# If true, the search result will be reranked using BGE CrossEncoder model.
use_reranker = False
# The overall steps are as follows:
# 1. embed the text as dense and sparse vectors
# 2. setup a Milvus collection to store the dense and sparse vectors
# 3. insert the data to Milvus
# 4. search and inspect the result!
import random
import string
import numpy as np
from pymilvus import (
utility,
FieldSchema,
CollectionSchema,
DataType,
Collection,
AnnSearchRequest,
RRFRanker,
connections,
Function,
FunctionType,
)
# 1. prepare a small corpus to search
docs = [
"Artificial intelligence was founded as an academic discipline in 1956.",
"Alan Turing was the first person to conduct substantial research in AI.",
"Born in Maida Vale, London, Turing was raised in southern England.",
]
# add some randomly generated texts
docs.extend(
[
" ".join(
"".join(random.choice(string.ascii_lowercase) for _ in range(random.randint(1, 8)))
for _ in range(10)
)
for _ in range(1000)
]
)
query = "Who started AI research?"
def random_embedding(texts):
rng = np.random.default_rng()
return {
"dense": np.random.rand(len(texts), 768),
}
dense_dim = 768
ef = random_embedding
if use_bge_m3:
# BGE-M3 model is included in the optional `model` module in pymilvus, to
# install it, simply run "pip install pymilvus[model]".
from pymilvus.model.hybrid import BGEM3EmbeddingFunction
ef = BGEM3EmbeddingFunction(use_fp16=False, device="cpu")
dense_dim = ef.dim["dense"]
docs_embeddings = ef(docs)
query_embeddings = ef([query])
# 2. setup Milvus collection and index
connections.connect("default", uri="milvus.db")
# Specify the data schema for the new Collection.
fields = [
# Use auto generated id as primary key
FieldSchema(name="pk", dtype=DataType.VARCHAR, is_primary=True, auto_id=True, max_length=100),
# Store the original text to retrieve based on semantically distance
FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=512, enable_analyzer=True),
# We need a sparse vector field to perform full text search with BM25,
# but you don't need to provide data for it when inserting data.
FieldSchema(name="sparse_vector", dtype=DataType.SPARSE_FLOAT_VECTOR),
FieldSchema(name="dense_vector", dtype=DataType.FLOAT_VECTOR, dim=dense_dim),
]
functions = [
Function(
name="bm25",
function_type=FunctionType.BM25,
input_field_names=["text"],
output_field_names="sparse_vector",
)
]
schema = CollectionSchema(fields, "", functions=functions)
col_name = "hybrid_bm25_demo"
# Now we can create the new collection with above name and schema.
col = Collection(col_name, schema, consistency_level="Strong")
# We need to create indices for the vector fields. The indices will be loaded
# into memory for efficient search.
sparse_index = {"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "BM25"}
col.create_index("sparse_vector", sparse_index)
dense_index = {"index_type": "FLAT", "metric_type": "IP"}
col.create_index("dense_vector", dense_index)
col.load()
# 3. insert text and sparse/dense vector representations into the collection
entities = [docs, docs_embeddings["dense"]]
col.insert(entities)
col.flush()
# 4. search and inspect the result!
k = 2 # we want to get the top 2 docs closest to the query
# Prepare the search requests for both full text search and dense vector search
full_text_search_params = {"metric_type": "BM25"}
# provide raw text query for full text search, while use the sparse vector as
# ANNS field
full_text_search_req = AnnSearchRequest([query], "sparse_vector", full_text_search_params, limit=k)
dense_search_params = {"metric_type": "IP"}
dense_req = AnnSearchRequest(
query_embeddings["dense"], "dense_vector", dense_search_params, limit=k
)
# Search topK docs based on dense and sparse vectors and rerank with RRF.
res = col.hybrid_search(
[full_text_search_req, dense_req], rerank=RRFRanker(), limit=k, output_fields=["text"]
)
# Currently Milvus only support 1 query in the same hybrid search request, so
# we inspect res[0] directly. In future release Milvus will accept batch
# hybrid search queries in the same call.
res = res[0]
if use_reranker:
result_texts = [hit.fields["text"] for hit in res]
from pymilvus.model.reranker import BGERerankFunction
bge_rf = BGERerankFunction(device="cpu")
# rerank the results using BGE CrossEncoder model
results = bge_rf(query, result_texts, top_k=2)
for hit in results:
print(f"text: {hit.text} distance {hit.score}")
else:
for hit in res:
print(f'text: {hit.fields["text"]} distance {hit.distance}')
# If you used both BGE-M3 and the reranker, you should see the following:
# text: Alan Turing was the first person to conduct substantial research in AI. distance 0.9306981017573297
# text: Artificial intelligence was founded as an academic discipline in 1956. distance 0.03217001154515051
#
# If you used only BGE-M3, you should see the following:
# text: Alan Turing was the first person to conduct substantial research in AI. distance 0.032786883413791656
# text: Artificial intelligence was founded as an academic discipline in 1956. distance 0.016129031777381897
# In this simple example the reranker yields the same result as the embedding based hybrid search, but in more complex
# scenarios the reranker can provide more accurate results.
# If you used random vectors, the result will be different each time you run the script.
# Drop the collection to clean up the data.
utility.drop_collection(col_name)
================================================
FILE: examples/hello_milvus.py
================================================
# hello_milvus.py demonstrates the basic operations of PyMilvus, a Python SDK of Milvus.
# 1. connect to Milvus
# 2. create collection
# 3. insert data
# 4. create index
# 5. search, query, and hybrid search on entities
# 6. delete entities by PK
# 7. drop collection
import time
import numpy as np
from pymilvus import (
connections,
utility,
FieldSchema, CollectionSchema, DataType,
Collection,
)
from pymilvus.client.types import LoadState
fmt = "\n=== {:30} ===\n"
search_latency_fmt = "search latency = {:.4f}s"
num_entities, dim = 3000, 8
#################################################################################
# 1. connect to Milvus
# Add a new connection alias `default` for Milvus server in `localhost:19530`
# Actually the "default" alias is a buildin in PyMilvus.
# If the address of Milvus is the same as `localhost:19530`, you can omit all
# parameters and call the method as: `connections.connect()`.
#
# Note: the `using` parameter of the following methods is default to "default".
print(fmt.format("start connecting to Milvus"))
connections.connect("default", uri="./local_test.db")
# connections.connect("default", host="localhost", port="19530")
has = utility.has_collection("hello_milvus")
print(f"Does collection hello_milvus exist in Milvus: {has}")
#################################################################################
# 2. create collection
# We're going to create a collection with 3 fields.
# +-+------------+------------+------------------+------------------------------+
# | | field name | field type | other attributes | field description |
# +-+------------+------------+------------------+------------------------------+
# |1| "pk" | VarChar | is_primary=True | "primary field" |
# | | | | auto_id=False | |
# +-+------------+------------+------------------+------------------------------+
# |2| "random" | Double | | "a double field" |
# +-+------------+------------+------------------+------------------------------+
# |3|"embeddings"| FloatVector| dim=8 | "float vector with dim 8" |
# +-+------------+------------+------------------+------------------------------+
fields = [
FieldSchema(name="random", dtype=DataType.DOUBLE),
FieldSchema(name="pk", dtype=DataType.VARCHAR, is_primary=True, auto_id=False, max_length=100),
FieldSchema(name="embeddings", dtype=DataType.FLOAT_VECTOR, dim=dim)
]
schema = CollectionSchema(fields, "hello_milvus is the simplest demo to introduce the APIs")
print(fmt.format("Create collection `hello_milvus`"))
hello_milvus = Collection("hello_milvus", schema, consistency_level="Strong")
# works ok
hello_milvus = Collection("hello_milvus", schema, consistency_level="Strong")
################################################################################
# 3. insert data
# We are going to insert 3000 rows of data into `hello_milvus`
# Data to be inserted must be organized in fields.
#
# The insert() method returns:
# - either automatically generated primary keys by Milvus if auto_id=True in the schema;
# - or the existing primary key field from the entities if auto_id=False in the schema.
print(fmt.format("Start inserting entities"))
rng = np.random.default_rng(seed=19530)
entities = [
# provide the pk field because `auto_id` is set to False
rng.random(num_entities).tolist(), # field random, only supports list
[str(i) for i in range(num_entities)],
rng.random((num_entities, dim)), # field embeddings, supports numpy.ndarray and list
]
insert_result = hello_milvus.insert(entities)
# hello_milvus.flush()
# print(f"Number of entities in Milvus: {hello_milvus.num_entities}") # check the num_entities
################################################################################
# 4. create index
# We are going to create an IVF_FLAT index for hello_milvus collection.
# create_index() can only be applied to `FloatVector` and `BinaryVector` fields.
print(fmt.format("Start Creating index FLAT"))
index = {
"index_type": "FLAT",
"metric_type": "L2",
"params": {"nlist": 128},
}
hello_milvus.create_index("embeddings", index)
################################################################################
# 5. search, query, and hybrid search
# After data were inserted into Milvus and indexed, you can perform:
# - search based on vector similarity
# - query based on scalar filtering(boolean, int, etc.)
# - hybrid search based on vector similarity and scalar filtering.
#
# Before conducting a search or a query, you need to load the data in `hello_milvus` into memory.
print(fmt.format("Start loading"))
hello_milvus.load()
assert utility.load_state("hello_milvus") == LoadState.Loaded
# -----------------------------------------------------------------------------
# search based on vector similarity
print(fmt.format("Start searching based on vector similarity"))
vectors_to_search = entities[-1][-2:]
search_params = {
"metric_type": "L2",
"params": {"nprobe": 10},
}
start_time = time.time()
result = hello_milvus.search(vectors_to_search, "embeddings", search_params, limit=3, output_fields=["random"])
end_time = time.time()
for hits in result:
for hit in hits:
print(f"hit: {hit}, random field: {hit.entity.get('random')}")
print(search_latency_fmt.format(end_time - start_time))
# -----------------------------------------------------------------------------
# query based on scalar filtering(boolean, int, etc.)
print(fmt.format("Start querying with `random > 0.5`"))
start_time = time.time()
result = hello_milvus.query(expr="random > 0.5", output_fields=["random", "embeddings"])
end_time = time.time()
print(f"query result:\n-{result[0]}")
print(search_latency_fmt.format(end_time - start_time))
# -----------------------------------------------------------------------------
# pagination
r1 = hello_milvus.query(expr="random > 0.5", limit=4, output_fields=["random"])
r2 = hello_milvus.query(expr="random > 0.5", offset=1, limit=3, output_fields=["random"])
print(f"query pagination(limit=4):\n\t{r1}")
print(f"query pagination(offset=1, limit=3):\n\t{r2}")
# -----------------------------------------------------------------------------
# hybrid search
print(fmt.format("Start hybrid searching with `random > 0.5`"))
start_time = time.time()
result = hello_milvus.search(vectors_to_search, "embeddings", search_params, limit=3, expr="random > 0.5", output_fields=["random"])
end_time = time.time()
for hits in result:
for hit in hits:
print(f"hit: {hit}, random field: {hit.entity.get('random')}")
print(search_latency_fmt.format(end_time - start_time))
###############################################################################
# 6. delete entities by PK
# You can delete entities by their PK values using boolean expressions.
ids = insert_result.primary_keys
expr = f'pk in ["{ids[0]}" , "{ids[1]}"]'
print(fmt.format(f"Start deleting with expr `{expr}`"))
result = hello_milvus.query(expr=expr, output_fields=["random", "embeddings"])
print(f"query before delete by expr=`{expr}` -> result: \n-{result[0]}\n-{result[1]}\n")
hello_milvus.delete(expr)
result = hello_milvus.query(expr=expr, output_fields=["random", "embeddings"])
print(f"query after delete by expr=`{expr}` -> result: {result}\n")
###############################################################################
# 7. drop collection
# Finally, drop the hello_milvus collection
print(fmt.format("Drop collection `hello_milvus`"))
utility.drop_collection("hello_milvus")
================================================
FILE: examples/hello_milvus_array.py
================================================
from pymilvus import CollectionSchema, FieldSchema, Collection, connections, DataType, Partition, utility
import numpy as np
import random
import pandas as pd
# connections.connect()
from milvus_lite.server_manager import server_manager_instance
connections.connect(uri="./local_test.db")
dim = 128
collection_name = "test_array"
arr_len = 100
nb = 10
if utility.has_collection(collection_name):
utility.drop_collection(collection_name)
# create collection
pk_field = FieldSchema(name="int64", dtype=DataType.INT64, is_primary=True, description='pk')
vector_field = FieldSchema(name="float_vector", dtype=DataType.FLOAT_VECTOR, dim=dim)
int8_array = FieldSchema(name="int8_array", dtype=DataType.ARRAY, element_type=DataType.INT8, max_capacity=arr_len)
int16_array = FieldSchema(name="int16_array", dtype=DataType.ARRAY, element_type=DataType.INT16, max_capacity=arr_len)
int32_array = FieldSchema(name="int32_array", dtype=DataType.ARRAY, element_type=DataType.INT32, max_capacity=arr_len)
int64_array = FieldSchema(name="int64_array", dtype=DataType.ARRAY, element_type=DataType.INT64, max_capacity=arr_len)
bool_array = FieldSchema(name="bool_array", dtype=DataType.ARRAY, element_type=DataType.BOOL, max_capacity=arr_len)
float_array = FieldSchema(name="float_array", dtype=DataType.ARRAY, element_type=DataType.FLOAT, max_capacity=arr_len)
double_array = FieldSchema(name="double_array", dtype=DataType.ARRAY, element_type=DataType.DOUBLE, max_capacity=arr_len)
string_array = FieldSchema(name="string_array", dtype=DataType.ARRAY, element_type=DataType.VARCHAR, max_capacity=arr_len,
max_length=100)
fields = [pk_field, vector_field, int8_array, int16_array, int32_array, int64_array,
bool_array, float_array, double_array, string_array]
schema = CollectionSchema(fields=fields)
collection = Collection(collection_name, schema=schema)
# insert data
pk_value = [i for i in range(nb)]
vector_value = [[random.random() for _ in range(dim)] for i in range(nb)]
int8_value = [[np.int8(j) for j in range(arr_len)] for i in range(nb)]
int16_value = [[np.int16(j) for j in range(arr_len)] for i in range(nb)]
int32_value = [[np.int32(j) for j in range(arr_len)] for i in range(nb)]
int64_value = [[np.int64(j) for j in range(arr_len)] for i in range(nb)]
bool_value = [[np.bool_(j) for j in range(arr_len)] for i in range(nb)]
float_value = [[np.float32(j) for j in range(arr_len)] for i in range(nb)]
double_value = [[np.double(j) for j in range(arr_len)] for i in range(nb)]
string_value = [[str(j) for j in range(arr_len)] for i in range(nb)]
data = [pk_value, vector_value,
int8_value,int16_value, int32_value, int64_value,
bool_value,
float_value,
double_value,
string_value
]
#collection.insert(data)
data = pd.DataFrame({
'int64': pk_value,
'float_vector': vector_value,
"int8_array": int8_value,
"int16_array": int16_value,
"int32_array": int32_value,
"int64_array": int64_value,
"bool_array": bool_value,
"float_array": float_value,
"double_array": double_value,
"string_array": string_value
})
collection.insert(data)
index = {
"index_type": "FLAT",
"metric_type": "L2",
"params": {"nlist": 128},
}
collection.create_index("float_vector", index)
collection.load()
res = collection.query("int64 >= 0", output_fields=["int8_array"])
for hits in res:
print(hits)
================================================
FILE: examples/hello_milvus_delete.py
================================================
import time
import numpy as np
from pymilvus import (
MilvusClient,
exceptions
)
fmt = "\n=== {:30} ===\n"
dim = 8
collection_name = "hello_milvus"
milvus_client = MilvusClient("./local_test.db")
milvus_client.drop_collection(collection_name)
milvus_client.create_collection(collection_name, dim, consistency_level="Strong", metric_type="L2")
print("collections:", milvus_client.list_collections())
print(f"{collection_name} :", milvus_client.describe_collection(collection_name))
rng = np.random.default_rng(seed=19530)
rows = [
{"id": 1, "vector": rng.random((1, dim))[0], "a": 1},
{"id": 2, "vector": rng.random((1, dim))[0], "b": 2},
{"id": 3, "vector": rng.random((1, dim))[0], "c": 3},
{"id": 4, "vector": rng.random((1, dim))[0], "d": 4},
{"id": 5, "vector": rng.random((1, dim))[0], "e": 5},
{"id": 6, "vector": rng.random((1, dim))[0], "f": 6},
]
print(fmt.format("Start inserting entities"))
pks = milvus_client.insert(collection_name, rows, progress_bar=True)['ids']
pks2 = milvus_client.insert(collection_name, {"id": 7, "vector": rng.random((1, dim))[0], "g": 1})['ids']
pks.extend(pks2)
def fetch_data_by_pk(pk):
print(f"get primary key {pk} from {collection_name}")
pk_data = milvus_client.get(collection_name, pk)
if pk_data:
print(f"data of primary key {pk} is", pk_data[0])
else:
print(f"data of primary key {pk} is empty")
fetch_data_by_pk(pks[2])
print(f"start to delete primary key {pks[2]} in collection {collection_name}")
milvus_client.delete(collection_name, pks = pks[2])
fetch_data_by_pk(pks[2])
fetch_data_by_pk(pks[4])
filter = "e == 5 or f == 6"
print(f"start to delete by expr {filter} in collection {collection_name}")
milvus_client.delete(collection_name, filter=filter)
fetch_data_by_pk(pks[4])
print(f"start to delete by expr '{filter}' or by primary 4 in collection {collection_name}, expect get exception")
try:
milvus_client.delete(collection_name, pks = 4, filter=filter)
except Exception as e:
assert isinstance(e, exceptions.ParamError)
print("catch exception", e)
print(f"start to delete without specify any expr '{filter}' or any primary key in collection {collection_name}, expect get exception")
try:
milvus_client.delete(collection_name)
except Exception as e:
print("catch exception", e)
result = milvus_client.query(collection_name, "", output_fields = ["count(*)"])
print(f"final entities in {collection_name} is {result[0]['count(*)']}")
milvus_client.drop_collection(collection_name)
================================================
FILE: examples/index.py
================================================
from milvus_lite.server_manager import server_manager_instance
import time
import numpy as np
from pymilvus import (
MilvusClient,
DataType
)
fmt = "\n=== {:30} ===\n"
dim = 8
collection_name = "hello_milvus"
milvus_client = MilvusClient("./local_test.db")
has_collection = milvus_client.has_collection(collection_name, timeout=5)
if has_collection:
milvus_client.drop_collection(collection_name)
schema = milvus_client.create_schema(enable_dynamic_field=True)
schema.add_field("id", DataType.INT64, is_primary=True)
schema.add_field("embeddings", DataType.FLOAT_VECTOR, dim=dim)
schema.add_field("title", DataType.VARCHAR, max_length=64)
# collection is not loaded after creation
milvus_client.create_collection(collection_name, schema=schema, consistency_level="Strong")
rng = np.random.default_rng(seed=19530)
rows = [
{"id": 1, "embeddings": rng.random((1, dim))[0], "a": 100, "title": "t1"},
{"id": 2, "embeddings": rng.random((1, dim))[0], "b": 200, "title": "t2"},
{"id": 3, "embeddings": rng.random((1, dim))[0], "c": 300, "title": "t3"},
{"id": 4, "embeddings": rng.random((1, dim))[0], "d": 400, "title": "t4"},
{"id": 5, "embeddings": rng.random((1, dim))[0], "e": 500, "title": "t5"},
{"id": 6, "embeddings": rng.random((1, dim))[0], "f": 600, "title": "t6"},
]
print(fmt.format("Start inserting entities"))
insert_result = milvus_client.insert(collection_name, rows)
print(fmt.format("Inserting entities done"))
print(insert_result)
index_params = milvus_client.prepare_index_params()
index_params.add_index(field_name = "embeddings", metric_type="L2")
index_params.add_index(field_name = "title", index_type = "Trie", index_name="my_trie")
print(fmt.format("Start create index"))
milvus_client.create_index(collection_name, index_params)
index_names = milvus_client.list_indexes(collection_name)
print(f"index names for {collection_name}:", index_names)
for index_name in index_names:
index_info = milvus_client.describe_index(collection_name, index_name=index_name)
print(f"index info for index {index_name} is:", index_info)
print(fmt.format("Start load collection"))
milvus_client.load_collection(collection_name)
print(fmt.format("Start query by specifying primary keys"))
query_results = milvus_client.query(collection_name, ids=[2])
print(query_results[0])
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(collection_name, filter= "f == 600 or title == 't2'")
for ret in query_results:
print(ret)
vectors_to_search = rng.random((1, dim))
print(fmt.format(f"Start search with retrieve serveral fields."))
result = milvus_client.search(collection_name, vectors_to_search, limit=3, output_fields=["title"])
for hits in result:
for hit in hits:
print(f"hit: {hit}")
field_index_names = milvus_client.list_indexes(collection_name, field_name = "embeddings")
print(f"index names for {collection_name}`s field embeddings:", field_index_names)
try:
milvus_client.drop_index(collection_name, "my_trie")
except Exception as e:
print(f"cacthed {e}")
milvus_client.release_collection(collection_name)
milvus_client.drop_index(collection_name, "my_trie")
milvus_client.drop_collection(collection_name)
================================================
FILE: examples/non_ascii_encode.py
================================================
import numpy as np
from pymilvus import MilvusClient, DataType
dimension = 128
collection_name = "books"
client = MilvusClient(uri="./local_test.db")
client.drop_collection(collection_name)
schema = client.create_schema(auto_id=True)
schema.add_field("id", DataType.INT64, is_primary=True)
schema.add_field("embeddings", DataType.FLOAT_VECTOR, dim=dimension)
schema.add_field("info", DataType.JSON)
index_params = client.prepare_index_params("embeddings", metric_type="L2")
client.create_collection(collection_name, schema=schema, index_params=index_params)
rng = np.random.default_rng(seed=19530)
rows = [
{"embeddings": rng.random((1, dimension))[0],
"info": {"title": "Lord of the Flies", "author": "William Golding"}},
{"embeddings": rng.random((1, dimension))[0],
"info": {"作者": "J.D.塞林格", "title": "麦田里的守望者", }},
{"embeddings": rng.random((1, dimension))[0],
"info": {"Título": "Cien años de soledad", "autor": "Gabriel García Márquez"}},
]
client.insert(collection_name, rows)
result = client.query(collection_name, filter="info['作者'] == 'J.D.塞林格' or info['Título'] == 'Cien años de soledad'",
output_fields=["info"],
consistency_level="Strong")
for hit in result:
print(f"hit: {hit}")
================================================
FILE: examples/simple.py
================================================
import numpy as np
from pymilvus import (
MilvusClient,
)
fmt = "\n=== {:30} ===\n"
dim = 8
collection_name = "hello_milvus"
milvus_client = MilvusClient("./local_test.db")
has_collection = milvus_client.has_collection(collection_name, timeout=5)
if has_collection:
milvus_client.drop_collection(collection_name)
milvus_client.create_collection(collection_name, dim, consistency_level="Strong", metric_type="L2")
print(fmt.format(" all collections "))
print(milvus_client.list_collections())
print(fmt.format(f"schema of collection {collection_name}"))
print(milvus_client.describe_collection(collection_name))
rng = np.random.default_rng(seed=19530)
rows = [
{"id": 1, "vector": rng.random((1, dim))[0], "a": 100},
{"id": 2, "vector": rng.random((1, dim))[0], "b": 200},
{"id": 3, "vector": rng.random((1, dim))[0], "c": 300},
{"id": 4, "vector": rng.random((1, dim))[0], "d": 400},
{"id": 5, "vector": rng.random((1, dim))[0], "e": 500},
{"id": 6, "vector": rng.random((1, dim))[0], "f": 600},
]
print(fmt.format("Start inserting entities"))
insert_result = milvus_client.insert(collection_name, rows, progress_bar=True)
print(fmt.format("Inserting entities done"))
print(insert_result)
print(fmt.format("Start query by specifying primary keys"))
query_results = milvus_client.query(collection_name, ids=[2])
print(query_results[0])
upsert_ret = milvus_client.upsert(collection_name, {"id": 2 , "vector": rng.random((1, dim))[0], "g": 100})
print(upsert_ret)
print(fmt.format("Start query by specifying primary keys"))
query_results = milvus_client.query(collection_name, ids=[2])
print(query_results[0])
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(collection_name, filter= "f == 600")
for ret in query_results:
print(ret)
print(f"start to delete by specifying filter in collection {collection_name}")
delete_result = milvus_client.delete(collection_name, ids=[6])
print(delete_result)
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(collection_name, filter= "f == 600")
assert len(query_results) == 0
rng = np.random.default_rng(seed=19530)
vectors_to_search = rng.random((1, dim))
print(fmt.format(f"Start search with retrieve serveral fields."))
result = milvus_client.search(collection_name, vectors_to_search, limit=3, output_fields=["pk", "a", "b"])
for hits in result:
for hit in hits:
print(f"hit: {hit}")
milvus_client.drop_collection(collection_name)
================================================
FILE: examples/simple_auto_id.py
================================================
import numpy as np
from pymilvus import (
MilvusClient,
)
fmt = "\n=== {:30} ===\n"
dim = 8
collection_name = "hello_milvus"
milvus_client = MilvusClient("./local_test.db")
has_collection = milvus_client.has_collection(collection_name, timeout=5)
if has_collection:
milvus_client.drop_collection(collection_name)
milvus_client.create_collection(collection_name, dim, consistency_level="Strong", metric_type="L2", auto_id=True)
print(fmt.format(" all collections "))
print(milvus_client.list_collections())
print(fmt.format(f"schema of collection {collection_name}"))
print(milvus_client.describe_collection(collection_name))
rng = np.random.default_rng(seed=19530)
rows = [
{"vector": rng.random((1, dim))[0], "a": 100},
{"vector": rng.random((1, dim))[0], "b": 200},
{"vector": rng.random((1, dim))[0], "c": 300},
{"vector": rng.random((1, dim))[0], "d": 400},
{"vector": rng.random((1, dim))[0], "e": 500},
{"vector": rng.random((1, dim))[0], "f": 600},
]
print(fmt.format("Start inserting entities"))
insert_result = milvus_client.insert(collection_name, rows, progress_bar=True)
print("insert done:", insert_result)
print(fmt.format("Start query by specifying filter"))
query_results = milvus_client.query(collection_name, filter= "f == 600")
for ret in query_results:
print(ret)
print(f"start to delete by specifying filter in collection {collection_name}")
delete_result = milvus_client.delete(collection_name, filter = "f == 600")
print(delete_result)
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(collection_name, filter= "f == 600")
assert len(query_results) == 0
rng = np.random.default_rng(seed=19530)
vectors_to_search = rng.random((1, dim))
print(fmt.format(f"Start search with retrieve serveral fields."))
result = milvus_client.search(collection_name, vectors_to_search, limit=3, output_fields=["pk", "a", "b"])
for hits in result:
for hit in hits:
print(f"hit: {hit}")
milvus_client.drop_collection(collection_name)
================================================
FILE: examples/sparse.py
================================================
from pymilvus import (
MilvusClient,
FieldSchema, CollectionSchema, DataType,
)
import random
def generate_sparse_vector(dimension: int, non_zero_count: int) -> dict:
indices = random.sample(range(dimension), non_zero_count)
values = [random.random() for _ in range(non_zero_count)]
sparse_vector = {index: value for index, value in zip(indices, values)}
return sparse_vector
fmt = "\n=== {:30} ===\n"
dim = 100
non_zero_count = 20
collection_name = "hello_sparse"
milvus_client = MilvusClient("./local_test.db")
has_collection = milvus_client.has_collection(collection_name, timeout=5)
if has_collection:
milvus_client.drop_collection(collection_name)
fields = [
FieldSchema(name="pk", dtype=DataType.VARCHAR,
is_primary=True, auto_id=True, max_length=100),
# FieldSchema(name="pk", dtype=DataType.INT64,
# is_primary=True, auto_id=True),
FieldSchema(name="random", dtype=DataType.DOUBLE),
FieldSchema(name="embeddings", dtype=DataType.SPARSE_FLOAT_VECTOR),
]
schema = CollectionSchema(
fields, "demo for using sparse float vector with milvus client")
index_params = milvus_client.prepare_index_params()
index_params.add_index(field_name="embeddings", index_name="sparse_inverted_index",
index_type="SPARSE_INVERTED_INDEX", metric_type="IP", params={"drop_ratio_build": 0.2})
milvus_client.create_collection(collection_name, schema=schema,
index_params=index_params, timeout=5, consistency_level="Strong")
print(fmt.format(" all collections "))
print(milvus_client.list_collections())
print(fmt.format(f"schema of collection {collection_name}"))
print(milvus_client.describe_collection(collection_name))
N = 6
rows = [{"random": i, "embeddings": generate_sparse_vector(
dim, non_zero_count)} for i in range(N)]
print(fmt.format("Start inserting entities"))
insert_result = milvus_client.insert(collection_name, rows, progress_bar=True)
print(fmt.format("Inserting entities done"))
print(insert_result)
print(fmt.format(f"Start vector anns search."))
vectors_to_search = [generate_sparse_vector(dim, non_zero_count)]
search_params = {
"metric_type": "IP",
"params": {
"drop_ratio_search": 0.2,
}
}
# no need to specify anns_field for collections with only 1 vector field
result = milvus_client.search(collection_name, vectors_to_search, limit=3, output_fields=[
"pk", "random", "embeddings"], search_params=search_params)
for hits in result:
for hit in hits:
print(f"hit: {hit}")
print(fmt.format("Start query by specifying filtering expression"))
query_results = milvus_client.query(collection_name, filter="random < 3")
pks = [ret['pk'] for ret in query_results]
for ret in query_results:
print(ret)
# print(fmt.format("Start query by specifying primary keys"))
# query_results = milvus_client.query(
# collection_name, filter=f"pk == '{pks[0]}'")
# print(query_results[0])
print(f"start to delete by specifying filter in collection {collection_name}")
print(pks[:1], 'xxxxxxxxxxxxxxxxxxxxxxxx')
print(milvus_client.query(collection_name, ids=pks[:1]))
delete_result = milvus_client.delete(collection_name, ids=pks[:1])
print(delete_result)
print(milvus_client.query(collection_name, ids=pks[:1]))
# print(fmt.format("Start query by specifying primary keys"))
# query_results = milvus_client.query(
# collection_name, filter=f"pk == '{pks[0]}'")
# print(f'query result should be empty: {query_results}')
milvus_client.drop_collection(collection_name)
================================================
FILE: python/pyproject.toml
================================================
[build-system]
requires = [
"setuptools>=64.0",
"wheel"
]
# Use legacy backend to import local packages in setup.py
build-backend = "setuptools.build_meta:__legacy__"
================================================
FILE: python/requirements.txt
================================================
tqdm
================================================
FILE: python/setup.py
================================================
# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
import shlex
import sys
import os
import pathlib
import unittest
from typing import List
import subprocess
import platform
from setuptools import setup, find_namespace_packages
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
import shutil
MILVUS_BIN = 'milvus'
KNOWHERE_BIN = 'libknowhere.dylib'
MILVUS_PATCH = str(pathlib.Path(__file__).absolute().parent.parent / 'thirdparty' / 'milvus.patch')
MILVUS_ROOT = str(pathlib.Path(__file__).absolute().parent.parent / 'thirdparty' / 'milvus')
class CMakeBuild(_bdist_wheel):
def finalize_options(self):
if sys.platform.lower() == 'linux':
self.plat_name = f"manylinux2014_{platform.machine().lower()}"
elif sys.platform.lower() == 'darwin':
if platform.machine().lower() == 'arm64':
self.plat_name = f"macosx_11_0_{platform.machine().lower()}"
else:
self.plat_name = f"macosx_10_9_{platform.machine().lower()}"
return super().finalize_options()
def copy_lib(self, lib_path, dst_dir, pick_libs):
name = pathlib.Path(lib_path).name
new_file = os.path.join(dst_dir, name)
for lib_prefix in pick_libs:
if name.startswith(lib_prefix):
shutil.copy(lib_path, new_file)
continue
def _pack_macos(self, src_dir: str, dst_dir: str):
mac_pkg = ['libknowhere', 'libmilvus',
'libgflags_nothreads', 'libglog',
'libtbb', 'libomp',
'libdouble-conversion']
milvus_bin = pathlib.Path(src_dir) / MILVUS_BIN
knowhere = pathlib.Path(src_dir) / KNOWHERE_BIN
out_str = subprocess.check_output(['otool', '-L', str(milvus_bin)])
subprocess.check_output(['install_name_tool', '-add_rpath', '@executable_path/.', str(milvus_bin)])
subprocess.check_output(['install_name_tool', '-add_rpath', '@executable_path/.', str(knowhere)])
lines = out_str.decode('utf-8').split('\n')
for line in lines[1:]:
r = line.split(' ')
if not r[0].endswith('dylib'):
continue
if r[0].strip().startswith("@rpath"):
real_path = pathlib.Path(src_dir) / r[0].strip()[len("@rpath/"):]
else:
real_path = r[0].strip()
self.copy_lib(real_path, dst_dir, mac_pkg)
def _pack_linux(self, src_dir: str, dst_dir: str):
linux_pkg = ['libknowhere', 'libmilvus',
'libgflags_nothreads', 'libglog',
'libtbb', 'libm', 'libgcc_s',
'libgomp', 'libopenblas',
'libdouble-conversion', 'libz',
'libgfortran', 'libquadmath']
milvus_bin = pathlib.Path(src_dir) / MILVUS_BIN
out_str = subprocess.check_output(['ldd', str(milvus_bin)])
lines = out_str.decode('utf-8').split('\n')
for line in lines:
r = line.split("=>")
if len(r) != 2:
continue
self.copy_lib(r[1].strip().split(' ')[0].strip(), dst_dir, linux_pkg)
def run(self):
build_lib = self.bdist_dir
build_temp = os.path.abspath(os.path.join(os.path.dirname(build_lib), 'build_milvus'))
if not os.path.exists(build_temp):
os.makedirs(build_temp)
#clean build temp
shutil.rmtree(os.path.join(build_temp, 'lib'), ignore_errors=True)
extdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
env = os.environ
env['LD_LIBRARY_PATH'] = os.path.join(build_temp, 'lib')
if env.get('USE_SYSTEM_DEPS'):
system_deps = '-DUSE_SYSTEM_DEPS=ON'
else:
system_deps = '-DUSE_SYSTEM_DEPS=OFF'
subprocess.call(['conan', 'remote', 'add', 'default-conan-local', 'https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local'],
cwd=build_temp, env=env)
if sys.platform.lower() == 'linux':
subprocess.check_call(['conan', 'install', extdir, '--build=missing', '-s', 'build_type=Release', '-s', 'compiler.libcxx=libstdc++11'],
cwd=build_temp, env=env)
else:
# macos
subprocess.check_call(['conan', 'install', extdir, '--build=missing', '-s', 'build_type=Release'], cwd=build_temp, env=env)
# build
extra_cmake_args = shlex.split(env.get('CMAKE_ARGS', ''))
extra_build_args = shlex.split(env.get('BUILD_ARGS', ''))
subprocess.check_call(['cmake', extdir, '-DENABLE_UNIT_TESTS=OFF', system_deps, *extra_cmake_args], cwd=build_temp, env=env)
subprocess.check_call(['cmake', '--build', '.', '--', '-j4', *extra_build_args],
cwd=build_temp,
env=env,
)
dst_lib_path = os.path.join(build_lib, 'milvus_lite/lib')
shutil.rmtree(dst_lib_path, ignore_errors=True)
os.makedirs(dst_lib_path)
if sys.platform.lower() == 'linux':
self._pack_linux(os.path.join(build_temp, 'lib'), dst_lib_path)
elif sys.platform.lower() == 'darwin':
self._pack_macos(os.path.join(build_temp, 'lib'), dst_lib_path)
else:
raise RuntimeError('Unsupport platform: %s', sys.platform)
shutil.copy(os.path.join(build_temp, 'lib', MILVUS_BIN), os.path.join(dst_lib_path, MILVUS_BIN))
super().run()
def test_suite():
test_loader = unittest.TestLoader()
tests = test_loader.discover('tests', pattern='test_*.py')
return tests
def parse_requirements(file_name: str) -> List[str]:
with open(file_name, encoding='utf-8') as f:
return [
require.strip() for require in f
if require.strip() and not require.startswith('#')
]
setup(name='milvus-lite',
version='2.5.2',
description='A lightweight version of Milvus wrapped with Python.',
author='Milvus Team',
author_email='milvus-team@zilliz.com',
url='https://github.com/milvus-io/milvus-lite.git',
test_suite='setup.test_suite',
install_requires=parse_requirements('requirements.txt'),
package_dir={'': 'src'},
packages=find_namespace_packages('src'),
package_data={},
include_package_data=True,
python_requires='>=3.7',
entry_points={
'console_scripts': ['milvus-lite=milvus_lite.cmdline:main']
},
cmdclass={"bdist_wheel": CMakeBuild},
long_description=open("../README.md", "r", encoding="utf-8").read(),
long_description_content_type='text/markdown'
)
================================================
FILE: python/src/milvus_lite/__init__.py
================================================
# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
from contextlib import suppress
try:
# Python 3.8+
from importlib.metadata import version, PackageNotFoundError
except ImportError: # fallback for older Python (<3.8)
from importlib_metadata import version, PackageNotFoundError # needs backport
__version__ = "0.0.0.dev"
with suppress(DistributionNotFound):
__version__ = version("milvus_lite")
================================================
FILE: python/src/milvus_lite/cmdline.py
================================================
# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
import argparse
import pathlib
import json
from tqdm import tqdm
import numpy as np
from pymilvus import Collection, connections, utility, DataType
from pymilvus.bulk_writer import LocalBulkWriter, BulkFileType
class MilvusEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, np.ndarray):
return o.tolist()
if isinstance(o, np.float32) or isinstance(o, np.float16):
return float(o)
return json.JSONEncoder.default(self, o)
def dump_func(args):
return dump_collection(args.db_file, args.collection, args.path)
def bfloat16_to_float32(byte_data):
bfloat16_array = np.frombuffer(byte_data, dtype=np.uint16)
bfloat16_as_uint16 = bfloat16_array.view(np.uint16)
float32_as_uint32 = (bfloat16_as_uint16.astype(np.uint32)) << 16
float32_array = float32_as_uint32.view(np.float32)
return float32_array
def binary_to_int_list(packed_bytes):
byte_array = np.frombuffer(packed_bytes, dtype=np.uint8)
return np.unpackbits(byte_array)
def dump_collection(db_file, collection_name, path):
if not pathlib.Path(db_file).is_file():
raise RuntimeError('db_file: %s not exists' % db_file)
if not pathlib.Path(path).parent.is_dir():
raise RuntimeError('dump path(%s)\'s parent dir not exists: %s not exists' % path)
connections.connect("default", uri=db_file)
if not utility.has_collection(collection_name):
raise RuntimeError("Collection: %s not exists" % collection_name)
collection = Collection(collection_name)
total_rows = collection.query("", output_fields=["count(*)"])[0]["count(*)"]
is_auto = collection.primary_field.auto_id
pk_name = collection.primary_field.name
bfloat16_fields = [field.name for field in collection.schema.fields if field.dtype == DataType.BFLOAT16_VECTOR]
bin_fields = [field.name for field in collection.schema.fields if field.dtype == DataType.BINARY_VECTOR]
writer = LocalBulkWriter(
schema=collection.schema,
local_path=path,
segment_size=512*1024*1024,
file_type=BulkFileType.JSON
)
it = collection.query_iterator(output_fields=['*'])
progress_bar = tqdm(total=total_rows, desc=f"Dump collection {collection_name}'s data")
while True:
rows = it.next()
if not rows:
it.close()
break
if is_auto:
for row in rows:
del row[pk_name]
if bfloat16_fields:
for row in rows:
for name in bfloat16_fields:
if name in row:
row[name] = bfloat16_to_float32(row[name])
if bin_fields:
for row in rows:
for name in bin_fields:
if name in row:
row[name] = binary_to_int_list(row[name][0])
rows = json.loads(json.dumps(rows, cls=MilvusEncoder))
for row in rows:
writer.append_row(row)
progress_bar.update(len(rows))
writer.commit()
print("Dump collection %s success" % collection_name)
def main():
parser = argparse.ArgumentParser(prog='milvus-lite')
subparsers = parser.add_subparsers(description='milvus-lite command line tool.')
dump_cmd = subparsers.add_parser('dump', help='milvus-lite dump cmd')
dump_cmd.add_argument('-d', '--db-file', type=str, help='milvus lite db file')
dump_cmd.add_argument('-c', '--collection', type=str, help='collection that need to be dumped')
dump_cmd.add_argument('-p', '--path', type=str, help='dump file storage path')
dump_cmd.set_defaults(func=dump_func)
args = parser.parse_args()
if hasattr(args, 'func'):
args.func(args)
else:
parser.print_help()
if __name__ == '__main__':
main()
================================================
FILE: python/src/milvus_lite/server.py
================================================
# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
from typing import Optional
import tempfile
import os
import subprocess
import pathlib
import signal
import logging
import fcntl
import re
BIN_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib')
logger = logging.getLogger()
class Server:
"""
The milvus-lite server
"""
MILVUS_BIN = 'milvus'
def __init__(self, db_file: str, address: Optional[str] = None):
"""
Args:
db_file (str):
The local file to store data.
address (address, optional):
grpc server address, example: localhost:19530,
if not set, the MilvusLite service will use UDS.
"""
if os.environ.get('BIN_PATH') is not None:
self._bin_path = pathlib.Path(os.environ['BIN_PATH']).absolute()
else:
self._bin_path = pathlib.Path(BIN_PATH).absolute()
self._db_file = pathlib.Path(db_file).absolute()
if not re.match(r'^[a-zA-Z0-9.\-_]+$', self._db_file.name):
raise RuntimeError(f"Unsupport db name {self._db_file.name}, the name must match ^[a-zA-Z0-9.\\-_]+$")
if len(self._db_file.name) > 36:
raise RuntimeError(f"Db name {self._db_file.name} is too long, should be less than 36")
self._work_dir = self._db_file.parent
self._address= address
self._p = None
self._uds_path = f"{tempfile.mktemp()}_{self._db_file.name}.sock"
self._lock_path = str(self._db_file.parent / f'.{self._db_file.name}.lock')
self._lock_fd = None
def init(self) -> bool:
if not self._bin_path.exists():
logger.error("Bin path not exists")
return False
if not self._work_dir.exists():
logger.error("Dir %s not exist", self._work_dir)
return True
@property
def milvus_bin(self):
return str(self._bin_path / 'milvus')
@property
def log_level(self):
return os.environ.get("LOG_LEVEL", "ERROR")
@property
def uds_path(self):
return f'unix:{self._uds_path}'
@property
def args(self):
if self._address is not None:
return [self.milvus_bin, self._db_file, self._address, self.log_level]
return [self.milvus_bin, self._db_file, self.uds_path, self.log_level, self._lock_path]
def start(self) -> bool:
assert self._p is None, "Server already started"
self._lock_fd = open(self._lock_path, 'a')
try:
fcntl.lockf(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
start_env = {
"LD_LIBRARY_PATH": str(self._bin_path) + ':' + os.environ.get('LD_LIBRARY_PATH', ''),
"DYLD_LIBRARY_PATH": str(self._bin_path) + ':' + os.environ.get('DYLD_LIBRARY_PATH', '')
}
src_env = os.environ.copy()
if "LD_LIBRARY_PATH" in src_env:
del src_env["LD_LIBRARY_PATH"]
if "DYLD_LIBRARY_PATH" in src_env:
del src_env["DYLD_LIBRARY_PATH"]
start_env.update(src_env)
self._p = subprocess.Popen(
args=self.args,
env=start_env,
cwd=str(self._work_dir),
)
try:
# Wait for 0.5 second to ensure successful startup
self._p.wait(0.5)
logger.error("Start milvus-lite failed")
return False
except subprocess.TimeoutExpired:
return True
except BlockingIOError:
logger.error("Open %s failed, the file has been opened by another program", self._db_file)
return False
def stop(self):
if self._lock_fd:
# When the file lock is released, the milvus-lite service will automatically stop.
fcntl.flock(self._lock_fd, fcntl.LOCK_UN)
self._lock_fd.close()
self._lock_fd = None
if self._p is not None:
try:
self._p.wait(timeout=2)
except subprocess.TimeoutExpired:
self._p.send_signal(signal.SIGKILL)
self._p.wait(timeout=3)
self._p = None
try:
os.unlink(self._uds_path)
except FileNotFoundError:
pass
try:
os.unlink(self._lock_path)
except FileNotFoundError:
pass
def __del__(self):
self.stop()
================================================
FILE: python/src/milvus_lite/server_manager.py
================================================
# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
from typing import Optional
import threading
import logging
import pathlib
from milvus_lite.server import Server
logger = logging.getLogger()
class ServerManager:
def __init__(self):
self._lock = threading.Lock()
self._servers = {}
def start_and_get_uri(self, path: str, args=None) -> Optional[str]:
path = pathlib.Path(path).absolute().resolve()
with self._lock:
if str(path) not in self._servers:
s = Server(str(path), args)
if not s.init():
return None
if not s.start():
return None
self._servers[str(path)] = s
return self._servers[str(path)].uds_path
def release_server(self, path: str):
path = pathlib.Path(path).absolute().resolve()
with self._lock:
if str(path) not in self._servers:
logger.warning("No local milvus in path %s", str(path))
return
self._servers[str(path)].stop()
del self._servers[str(path)]
def release_all(self):
for s in self._servers.values():
s.stop()
self._servers = {}
def __del__(self):
with self._lock:
self.release_all()
server_manager_instance = ServerManager()
================================================
FILE: scripts/Dockerfile.manylinux.aarch64
================================================
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
FROM quay.io/pypa/manylinux_2_28_aarch64:2024.07.07-1
RUN yum install -y openblas-devel libatomic-static git && cp /usr/lib/gcc/aarch64-redhat-linux/8/libatomic.a /usr/lib64/
ENV PATH="/opt/_internal/cpython-3.8.19/bin:${PATH}"
RUN pip3 install conan==1.63.0
RUN curl https://sh.rustup.rs -sSf | bash -s -- --default-toolchain=1.89 -y
ENV PATH="/root/.cargo/bin:${PATH}"
ENV CCFLAGS="-Wno-error=address"
ENV CXXFLAGS="-Wno-error=address"
RUN mkdir -p /workspace
WORKDIR /workspace
RUN conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
COPY build_milvus_lite.sh /workspace/build_milvus_lite.sh
RUN chmod +x /workspace/build_milvus_lite.sh
================================================
FILE: scripts/Dockerfile.manylinux.x86_64
================================================
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
FROM quay.io/pypa/manylinux_2_28_x86_64:2024.07.07-1
RUN yum install -y openblas-devel libatomic-static git && cp /usr/lib/gcc/x86_64-redhat-linux/8/libatomic.a /usr/lib64/
ENV PATH="/opt/_internal/cpython-3.8.19/bin:${PATH}"
RUN pip3 install conan==1.63.0
RUN curl https://sh.rustup.rs -sSf | bash -s -- --default-toolchain=1.89 -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN mkdir -p /workspace
WORKDIR /workspace
RUN conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
COPY build_milvus_lite.sh /workspace/build_milvus_lite.sh
RUN chmod +x /workspace/build_milvus_lite.sh
================================================
FILE: scripts/README.md
================================================
# Build Milvus-Lite In Docker
## build commond
```shell
build.sh ${Dockerfile} ${TAG} {$CONAN_CACHE}
```
================================================
FILE: scripts/build.sh
================================================
#!/bin/bash
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
TAG="main"
IMAGE_TAG="latest"
if [ "$#" -eq 0 ]; then
echo "Please set dockerfile path"
elif [ "$#" -eq 1 ]; then
DOCKERFILE=$1
docker build -t build_milvus_lite:$IMAGE_TAG -f $DOCKERFILE . \
&& docker run --rm -v $PWD:/workspace/dist build_milvus_lite:$IMAGE_TAG /workspace/build_milvus_lite.sh $TAG
elif [ "$#" -eq 2 ]; then
DOCKERFILE=$1
TAG=$2
docker build -t build_milvus_lite:$IMAGE_TAG -f $DOCKERFILE . \
&& docker run --rm -v $PWD:/workspace/dist build_milvus_lite:$IMAGE_TAG /workspace/build_milvus_lite.sh $TAG
elif [ "$#" -eq 3 ]; then
DOCKERFILE=$1
TAG=$2
CACAN_CACHE=$3
docker build -t build_milvus_lite:$IMAGE_TAG -f $DOCKERFILE . \
&& docker run --rm -e CONAN_USER_HOME=/workspace/conan -v $CACAN_CACHE:/workspace/conan -v $PWD:/workspace/dist build_milvus_lite:$IMAGE_TAG /workspace/build_milvus_lite.sh $TAG
fi
================================================
FILE: scripts/build_milvus_lite.sh
================================================
#!/bin/bash
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
tag="main"
if [ "$#" -eq 1 ]; then
tag=$1
fi
echo "Build milvus-lite:$tag"
echo $CONAN_USER_HOME
git clone --recurse-submodules https://github.com/milvus-io/milvus-lite.git \
&& cd milvus-lite \
&& git checkout $tag \
&& cd python \
&& python3 -m build --wheel \
&& cp -r dist /workspace/ \
&& cd /workspace && rm -rf milvus-lite
================================================
FILE: src/CMakeLists.txt
================================================
# Copyright (C) 2019-2024 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License
include_directories(${CMAKE_CURRENT_LIST_DIR})
add_library(milite ${LITE_BUILD_TYPE}
${CMAKE_CURRENT_LIST_DIR}/storage/collection_meta.cpp
${CMAKE_CURRENT_LIST_DIR}/storage/collection_data.cpp
${CMAKE_CURRENT_LIST_DIR}/storage/bm25_stats.cpp
${CMAKE_CURRENT_LIST_DIR}/storage/storage.cpp
${CMAKE_CURRENT_LIST_DIR}/timer.cpp
${CMAKE_CURRENT_LIST_DIR}/index.cpp
${CMAKE_CURRENT_LIST_DIR}/milvus_local.cpp
${CMAKE_CURRENT_LIST_DIR}/segcore_wrapper.cpp
${CMAKE_CURRENT_LIST_DIR}/milvus_proxy.cpp
${CMAKE_CURRENT_LIST_DIR}/create_collection_task.cpp
${CMAKE_CURRENT_LIST_DIR}/create_index_task.cpp
${CMAKE_CURRENT_LIST_DIR}/search_task.cpp
${CMAKE_CURRENT_LIST_DIR}/insert_task.cpp
${CMAKE_CURRENT_LIST_DIR}/query_task.cpp
${CMAKE_CURRENT_LIST_DIR}/delete_task.cpp
${CMAKE_CURRENT_LIST_DIR}/upsert_task.cpp
${CMAKE_CURRENT_LIST_DIR}/schema_util.cpp
${CMAKE_CURRENT_LIST_DIR}/re_scorer.cpp
${CMAKE_CURRENT_LIST_DIR}/hybrid_search_task.cpp
${CMAKE_CURRENT_LIST_DIR}/function/function_executor.cpp
${CMAKE_CURRENT_LIST_DIR}/function/bm25_function.cpp
)
target_link_libraries(
milite
PUBLIC
parser
milvus_core
milvus_log
yaml-cpp
roaring::roaring
SQLiteCpp
${antlr4-cppruntime_LIBRARIES}
${MARISA_LIBRARIES}
TBB::tbb
${PROTOBUF_LIBRARIES}
${ARROW_LIBRARIES}
simdjson
tantivy_binding
)
add_library(
milvus_service
STATIC
"${CMAKE_SOURCE_DIR}/src/milvus_service_impl.cpp"
)
target_link_libraries(
milvus_service
PUBLIC
milvus_grpc_service
)
add_executable(milvus server.cpp)
target_link_libraries(
milvus
milvus_service
milite
)
if(ENABLE_UNIT_TESTS)
add_subdirectory(unittest)
endif()
================================================
FILE: src/common.h
================================================
// Copyright (C) 2019-2024 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
#pragma once
#include <cstdint>
#include <map>
#include <string>
namespace milvus::local {
#define CHECK_STATUS(status, err) \
do { \
Status _s = status; \
if (!_s.IsOk()) { \
return _s; \
} \
} while (0)
#define DELETE_AND_SET_NULL(ptr, deleter) \
do { \
if ((ptr) != nullptr) { \
deleter(ptr); \
(ptr) = nullptr; \
} \
} while (0)
using KVMap = std::map<std::string, std::string>;
using SparseVector = std::map<uint32_t, float>;
// system field id:
// 0: unique row id
// 1: timestamp
// 100: first user field id
// 101: second user field id
// 102: ...
const int64_t kStartOfUserFieldId = 100;
const int64_t kRowIdField = 0;
const int64_t kTimeStampField = 1;
const std::string kRowIdFieldName("RowID");
const std::string kTimeStampFieldName("Timestamp");
const std::string kMetaFieldName("$meta");
const std::string kPlaceholderTag("$0");
const int64_t kTopkLimit = 16384;
const int64_t kSchemaFieldLimit = 64;
const int64_t kMaxLengthLimit = 65535;
// scalar index type
const std::string kDefaultStringIndexType("Trie");
const std::string kInvertedIndexType("INVERTED");
const std::string kDefaultArithmeticIndexType = ("STL_SORT");
const int64_t kMaxIndexRow = 1000000;
// Search, Index parameter keys
const std::string kTopkKey("topk");
// const std::string kSearchParamKey("search_param");
const std::string kSearchParamKey("params");
const std::string kOffsetKey("offset");
const std::string kRoundDecimalKey("round_decimal");
const std::string kGroupByFieldKey("group_by_field");
const std::string kAnnFieldKey("anns_field");
const std::string kSegmentNumKey("segment_num");
const std::string kWithFilterKey("with_filter");
const std::string kWithOptimizeKey("with_optimize");
const std::string kCollectionKey("collection");
const std::string kIndexParamsKey("params");
const std::string kIndexTypeKey("index_type");
const std::string kMetricTypeKey("metric_type");
const std::string kDimKey("dim");
const std::string kMaxLengthKey("max_length");
const std::string kMaxCapacityKey("max_capacity");
const std::string kReduceStopForBestKey("reduce_stop_for_best");
const std::string kLimitKey("limit");
const std::string KMetricsIPName("IP");
const std::string kMetricsCosineName("COSINE");
const std::string kMetricsBM25Name("BM25");
const std::string kMetricsL2Name("L2");
const std::string kBM25AvgName("bm25_avgdl");
const std::string kCountStr("count(*)");
inline int64_t
GetCollectionId(const std::string& collection_name) {
std::hash<std::string> hasher;
size_t hash_value = hasher(collection_name);
return static_cast<int64_t>(hash_value);
}
inline int64_t
GetIndexId(const std::string& index_name) {
std::hash<std::string> hasher;
size_t hash_value = hasher(index_name);
return static_cast<int64_t>(hash_value);
}
struct NonCopyableNonMovable {
constexpr NonCopyableNonMovable() noexcept = default;
virtual ~NonCopyableNonMovable() noexcept = default;
NonCopyableNonMovable(NonCopyableNonMovable&&) = delete;
NonCopyableNonMovable&
operator=(NonCopyableNonMovable&&) = delete;
NonCopyableNonMovable(const NonCopyableNonMovable&) = delete;
NonCopyableNonMovable&
operator=(const NonCopyableNonMovable&) = delete;
};
} // namespace milvus::local
================================================
FILE: src/create_collection_task.cpp
================================================
// Copyright (C) 2019-2024 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
#include "create_collection_task.h"
#include <cstddef>
#include <cstdint>
#include <exception>
#include <limits>
#include <string>
#include "common.h"
#include "log/Log.h"
#include "pb/schema.pb.h"
#include "status.h"
#include "string_util.hpp"
namespace milvus::local {
using DType = ::milvus::proto::schema::DataType;
using DCase = ::milvus::proto::schema::ValueField::DataCase;
bool
CreateCollectionTask::HasSystemFields(
const ::milvus::proto::schema::CollectionSchema& schema) {
for (const auto& f : schema.fields()) {
if (f.name() == kRowIdFieldName || f.name() == kTimeStampFieldName ||
f.name() == kMetaFieldName) {
return true;
}
}
return false;
}
Status
CheckFieldParams(const ::milvus::proto::schema::FieldSchema& field) {
if (field.nullable()) {
return Status::ParameterInvalid("MilvusLite doesn't support nullable");
}
// Treat the partition key as a normal field
if (field.is_partition_key()) {
auto f = const_cast<::milvus::proto::schema::FieldSchema*>(&field);
f->set_is_partition_key(false);
}
return Status::Ok();
}
Status
CreateCollectionTask::GetVarcharFieldMaxLength(
const ::milvus::proto::schema::FieldSchema& field, uint64_t* max_len) {
if (field.data_type() != DType::VarChar &&
field.element_type() != DType::VarChar) {
return Status::ParameterInvalid("{} is not varchar field",
field.name());
}
for (const auto& kv_pair : field.type_params()) {
if (kv_pair.key() == kMaxLengthKey) {
try {
auto length = std::stoll(kv_pair.value());
if (length <= 0 || length > kMaxLengthLimit) {
return Status::ParameterInvalid(
"the maximum length specified for a VarChar should "
"be "
"in (0, 65535])");
} else {
*max_len = static_cast<uint64_t>(length);
return Status::Ok();
}
} catch (std::exception& e) {
return Status::ParameterInvalid("Invalid max length {}",
kv_pair.value());
}
}
}
for (const auto& kv_pair : field.index_params()) {
if (kv_pair.key() == kMaxLengthKey) {
try {
auto length = std::stoll(kv_pair.value());
if (length <= 0 || length > kMaxLengthLimit) {
return Status::ParameterInvalid(
"the maximum length specified for a VarChar should "
"be "
"in (0, 65535])");
} else {
*max_len = static_cast<uint64_t>(length);
return Status::Ok();
}
} catch (std::exception& e) {
return Status::ParameterInvalid("Invalid max length {}",
kv_pair.value());
}
}
}
return Status::ParameterInvalid(
"type param(max_length) should be specified for varChar field of "
"collection");
}
bool
CreateCollectionTask::CheckDefaultValue(
const ::milvus::proto::schema::CollectionSchema& schema) {
for (const auto& f : schema.fields()) {
if (!f.has_default_value())
continue;
switch (f.default_value().data_case()) {
case DCase::kBoolData:
if (f.data_type() != DType::Bool) {
LOG_ERROR(
"{} field's default value is Bool type, mismatches "
"field type",
f.name());
return false;
}
break;
case DCase::kIntData: {
if (f.data_type() != DType::Int16 &&
f.data_type() != DType::Int32 &&
f.data_type() != DType::Int8) {
LOG_ERROR(
"{} field's default value is Int type, mismatches "
"field type",
f.name());
return false;
}
auto default_value = f.default_value().int_data();
if (f.data_type() == DType::Int16) {
if (default_value < std::numeric_limits<int16_t>::min() ||
default_value > std::numeric_limits<int16_t>::max()) {
LOG_ERROR("{} field's default value out of range.",
f.name());
return false;
}
}
if (f.data_type() == DType::Int8) {
if (default_value < std::numeric_limits<int8_t>::min() ||
default_value > std::numeric_limits<int8_t>::max()) {
LOG_ERROR("{} field's default value out of range.",
f.name());
return false;
}
}
} break;
case DCase::kLongData:
if (f.data_type() != DType::Int64) {
LOG_ERROR(
"{} field's default value is Long type, mismatches "
"field type",
f.name());
return false;
}
break;
case DCase::kFloatData:
if (f.data_type() != DType::Float) {
LOG_ERROR(
"{} field's default value is Float type, "
"mismatches "
"field type",
f.name());
return false;
}
break;
case DCase::kDoubleData:
if (f.data_type() != DType::Double) {
LOG_ERROR(
"{} field's default value is Double type, "
"mismatches "
"field type",
f.name());
return false;
}
break;
case DCase::kStringData: {
if (f.data_type() != DType::VarChar) {
LOG_ERROR(
"{} field's default value is VarChar type, "
"mismatches field type",
f.name());
return false;
}
auto string_len = f.default_value().string_data().size();
uint64_t max_length = 0;
auto s = GetVarcharFieldMaxLength(f, &max_length);
if (s.IsErr()) {
LOG_ERROR(s.Detail());
return false;
}
if (string_len > max_length) {
return false;
}
} break;
// case DCase::kBytesData: // not used
// break;
default:
return false;
break;
}
}
return true;
}
void
CreateCollectionTask::AssignFieldId(
::milvus::proto::schema::CollectionSchema* schema) {
for (int i = 0; i < schema->fields_size(); i++) {
schema->mutable_fields(i)->set_fieldid(kStartOfUserFieldId + i);
}
}
void
CreateCollectionTask::AppendDynamicField(
::milvus::proto::schema::CollectionSchema* schema) {
if (schema->enable_dynamic_field()) {
auto dynamice_field = schema->add_fields();
dynamice_field->set_name(kMetaFieldName);
dynamice_field->set_description("dynamic schema");
dynamice_field->set_data_type(DType::JSON);
dynamice_field->set_is_dynamic(true);
}
}
void
CreateCollectionTask::AppendSysFields(
::milvus::proto::schema::CollectionSchema* schema) {
auto row_id_field = schema->add_fields();
row_id_field->set_fieldid(kRowIdField);
row_id_field->set_name(kRowIdFieldName);
row_id_field->set_is_primary_key(false);
row_id_field->set_description("row id");
row_id_field->set_data_type(DType::Int64);
auto ts_field = schema->add_fields();
ts_field->set_fieldid(kTimeStampField);
ts_field->set_name(kTimeStampFieldName);
ts_field->set_is_primary_key(false);
ts_field->set_description("time stamp");
ts_field->set_data_type(DType::Int64);
}
Status
VaildBM25Functions(const ::milvus::proto::schema::CollectionSchema& schema,
const milvus::proto::schema::FunctionSchema& fs) {
if (fs.type() != milvus::proto::schema::FunctionType::BM25) {
return Status::ParameterInvalid(
"MilvusLite only supports BM25 function");
}
if (fs.input_field_names_size() != 1 || fs.output_field_names_size() != 1) {
return Status::ParameterInvalid(
"The input and output of the BM25 function must be 1");
}
bool found_input = false;
bool found_output = false;
for (const auto& field : schema.fields()) {
if (field.name() == fs.input_field_names(0)) {
found_input = true;
if (field.is_dynamic()) {
return Status::ParameterInvalid(
"Function's input cannot be dynamic field");
}
if (field.data_type() != milvus::proto::schema::DataType::VarChar) {
return Status::ParameterInvalid(
"BM25 funciton's input must be of VARCHAR type");
}
if (field.nullable()) {
return Status::ParameterInvalid(
"Function's input cannot be nullable field");
}
}
if (field.name() == fs.output_field_names(0)) {
found_output = true;
if (field.is_dynamic()) {
return Status::ParameterInvalid(
"Function's output cannot be dynamic field");
}
if (field.data_type() !=
milvus::proto::schema::DataType::SparseFloatVector) {
return Status::ParameterInvalid(
"BM25 funciton's output must be of SparseFloatVector "
"type");
}
if (field.nullable()) {
return Status::ParameterInvalid(
"Function's output cannot be nullable field");
}
}
}
if (!found_input) {
return Status::ParameterInvalid("function's input {} not found",
fs.input_field_names(0));
}
if (!found_output) {
return Status::ParameterInvalid("function's output {} not found",
fs.output_field_names(0));
}
return Status::Ok();
}
Status
CreateCollectionTask::ValidateSchema(
const ::milvus::proto::schema::CollectionSchema& schema) {
if (schema.fields_size() > kSchemaFieldLimit)
return Status::ParameterInvalid(
"maximum field's number should be limited to {}",
kSchemaFieldLimit);
std::set<std::string> field_names;
std::string pk_name;
for (const auto& field_schema : schema.fields()) {
if (field_names.find(field_schema.name()) != field_names.end()) {
return Status::ParameterInvalid("Duplicated field name: {}",
field_schema.name());
}
if (field_schema.is_primary_key()) {
if (!pk_name.empty()) {
return Status::ParameterInvalid(
"there are more than one primary key, field_name = {}, "
"{}",
pk_name,
field_schema.name());
} else {
pk_name = field_schema.name();
}
}
if (field_schema.is_dynamic()) {
return Status::ParameterInvalid(
"cannot explicitly set a field as a dynamic field");
}
CHECK_STATUS(CheckFieldParams(field_schema), "");
CHECK_STATUS(CheckFieldName(field_schema.name()), "");
if (field_schema.data_type() == DType::VarChar) {
uint64_t max_length = 0;
CHECK_STATUS(GetVarcharFieldMaxLength(field_schema, &max_length),
"");
}
}
// valid functions
for (const auto& f : schema.functions()) {
CHECK_STATUS(VaildBM25Functions(schema, f), "");
}
return Status::Ok();
}
Status
CreateCollectionTask::Process(
::milvus::proto::schema::CollectionSchema* schema) {
if (!schema->ParseFromString(create_collection_request_->schema())) {
LOG_ERROR("Failed parse schema");
return Status::ParameterInvalid("Failed parse schema");
}
if (create_collection_request_->collection_name() != schema->name()) {
auto err = string_util::SFormat(
"collection name [{}] not matches schema name [{}]",
create_collection_request_->collection_name(),
schema->name());
LOG_ERROR(err);
return Status::ParameterInvalid(err);
}
CHECK_STATUS(ValidateSchema(*schema), "");
if (HasSystemFields(*schema)) {
auto err_msg =
string_util::SFormat("Schema contains system field {}, {}, {}",
kRowIdFieldName,
kTimeStampFieldName,
kMetaFieldName);
LOG_ERROR(err_msg);
return Status::ParameterInvalid(err_msg);
}
if (!CheckDefaultValue(*schema)) {
return Status::ParameterInvalid();
}
AppendDynamicField(schema);
AssignFieldId(schema);
AppendSysFields(schema);
return Status::Ok();
}
Status
CreateCollectionTask::CheckFieldName(const std::string& field_name) {
std::string name = string_util::Trim(field_name);
if (name.empty()) {
return Status::ParameterInvalid("field {} should not be empty", name);
}
std::string invalid_msg =
string_util::SFormat("Invalid field name {}. ", name);
if (name.size() > 255) {
return Status::ParameterInvalid(
"{}, the length of a field name must "
"be less than 255 characters",
invalid_msg);
}
char first = name[0];
if (first != '_' && !string_util::IsAlpha(first)) {
return Status::ParameterInvalid(
"{} the first character of a field {} must be an underscore "
"or letter",
invalid_msg,
name);
}
std::regex pattern("^[a-zA-Z_][a-zA-Z0-9_]*$");
if (!std::regex_match(name, pattern)) {
auto err = string_util::SFormat(
"{}, field name can only contain "
"numbers, letters and underscores",
invalid_msg);
LOG_ERROR(err);
return Status::ParameterInvalid(err);
}
return Status::Ok();
}
} // namespace milvus::local
================================================
FILE: src/create_collection_task.h
================================================
// Copyright (C) 2019-2024 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
#pragma once
#include <cstdint>
#include "pb/milvus.pb.h"
#include "pb/schema.pb.h"
#include "pb/segcore.pb.h"
#include "common.h"
#include "status.h"
namespace milvus::local {
class CreateCollectionTask final : NonCopyableNonMovable {
public:
explicit CreateCollectionTask(
const ::milvus::proto::milvus::CreateCollectionRequest*
create_collection_request)
: create_collection_request_(create_collection_request) {
}
virtual ~CreateCollectionTask() = default;
public:
Status
Process(::milvus::proto::schema::CollectionSchema* schema);
private:
bool
CheckDefaultValue(const ::milvus::proto::schema::CollectionSchema& schema);
bool
HasSystemFields(const ::milvus::proto::schema::CollectionSchema& schema);
void
AssignFieldId(::milvus::proto::schema::CollectionSchema* schema);
void
AppendDynamicField(::milvus::proto::schema::CollectionSchema* schema);
void
AppendSysFields(::milvus::proto::schema::CollectionSchema* schema);
Status
GetVarcharFieldMaxLength(const ::milvus::proto::schema::FieldSchema& field,
uint64_t* max_len);
Status
ValidateSchema(const ::milvus::proto::schema::CollectionSchema& schema);
Status
CheckFieldName(const std::string& field_name);
private:
const ::milvus::proto::milvus::CreateCollectionRequest*
create_collection_request_;
};
} // namespace milvus::local
================================================
FILE: src/create_index_task.cpp
================================================
// Copyright (C) 2019-2024 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
#include "create_index_task.h"
#include <algorithm>
#include <cstdint>
#include <functional>
#include <map>
#include <string>
#include <vector>
#include "common.h"
#include "nlohmann/json.hpp"
#include "pb/common.pb.h"
#include "pb/schema.pb.h"
#include "pb/segcore.pb.h"
#include "schema_util.h"
#include "log/Log.h"
#include "status.h"
#include "string_util.hpp"
namespace milvus::local {
/* dtype index_type metric_type
* FloatVector: FLAT, HNWS L2, IP, COSINE, AUTOINDEX
* BinaryVector: BIN_FLAT HAMMING, JACCARD, SUBSTRUCTURE, SUPERSTRUCTURE
* Float16Vector: FLAT L2, IP, COSINE
* BFloat16Vector: FLAT L2, IP, COSINE
* SparseFloatVector: SPARSE_INVERTED_INDEX, SPARSE_WAND IP, BM25
*/
// metrics type
const char* kL2 = "L2";
const char* kIP = "IP";
const char* kCosine = "COSINE";
const char* kHamming = "HAMMING";
const char* kJaccard = "JACCARD";
const char* kSubStructure = "SUBSTRUCTURE";
const char* kSuperStructure = "SUPERSTRUCTURE";
// index_type
const char* kAutoIndex = "AUTOINDEX";
const char* kFlat = "FLAT";
const char* kBin_Flat = "BIN_FLAT";
// const char* kHNSW = "HNSW";
const char* kIvf_Flat = "IVF_FLAT";
const char* kSparseInvertedIndex = "SPARSE_INVERTED_INDEX";
const char* kSparseWand = "SPARSE_WAND";
// default metric
const char* kFloatVectorDefaultMetricType = kIP;
const char* kSparseFloatVectorDefaultMetricType = kIP;
const char* kBinaryVectorDefaultMetricType = kJaccard;
//bm25 params
const char* kBm25K1 = "bm25_k1";
const char* kBm25B = "bm25_b";
class AutoIndexConfig final : NonCopyableNonMovable {
public:
AutoIndexConfig()
: index_param({{"M", "18"},
{"efConstruction", "240"},
{"index_type", kAutoIndex},
{"metric_type", kIP}}) {
}
~AutoIndexConfig() = default;
public:
const KVMap index_param;
};
using DType = ::milvus::proto::schema::DataType;
static const AutoIndexConfig kAutoIndexConfig;
class IndexChecker : NonCopyableNonMovable {
public:
IndexChecker(const std::string& index_type,
const std::string& metric,
int64_t dim)
: index_type_(index_type),
metric_(metric),
dim_(dim),
need_check_dim_(true) {
}
virtual ~IndexChecker() = default;
Status
Check();
protected:
std::string index_type_;
std::string metric_;
int64_t dim_;
bool need_check_dim_;
int64_t min_dim_;
int64_t max_dim_;
std::vector<std::string> supported_index_;
std::vector<std::string> supported_metric_;
};
Status
IndexChecker::Check() {
if (need_check_dim_ && (dim_ < min_dim_ || dim_ > max_dim_)) {
auto err = string_util::SFormat(
"invalid dimension: {}. should be in range {} ~ {}",
dim_,
min_dim_,
max_dim_);
return Status::Undefined(err);
}
if (std::find(supported_index_.begin(),
supported_index_.end(),
index_type_) == supported_index_.end()) {
auto err = string_util::SFormat(
"invalid index type: {}, local mode only support {}",
index_type_,
string_util::Join(" ", supported_index_));
return Status::Undefined(err);
}
if (std::find(supported_metric_.begin(),
supported_metric_.end(),
metric_) == supported_metric_.end()) {
auto err = string_util::SFormat(
"metric type {} not found or not supported, supported: {}",
metric_,
string_util::Join(" ", supported_metric_));
return Status::Undefined(err);
}
return Status::Ok();
}
class FloatVectorIndexChecker : public virtual IndexChecker {
public:
FloatVectorIndexChecker(const std::string& index_type,
const std::string& metric,
int64_t dim)
: IndexChecker(index_type, metric, dim) {
min_dim_ = 2;
max_dim_ = 32768;
supported_index_ = {kFlat, kIvf_Flat, kAutoIndex};
supported_metric_ = {kL2, kIP, kCosine};
}
virtual ~FloatVectorIndexChecker() = default;
};
class BinaryVectorChecker : public virtual IndexChecker {
public:
BinaryVectorChecker(const std::string& index_type,
const std::string& metric,
int64_t dim)
: IndexChecker(index_type, metric, dim) {
min_dim_ = 2;
max_dim_ = 32768;
supported_index_ = {kBin_Flat};
supported_metric_ = {
kHamming, kJaccard, kSubStructure, kSuperStructure};
}
virtual ~BinaryVectorChecker() = default;
};
class Float16VectorChecker : public virtual IndexChecker {
public:
Float16VectorChecker(const std::string& index_type,
const std::string& metric,
int64_t dim)
: IndexChecker(index_type, metric, dim) {
min_dim_ = 2;
max_dim_ = 32768;
supported_index_ = {kFlat};
supported_metric_ = {kL2, kIP, kCosine};
}
virtual ~Float16VectorChecker() = default;
};
class BFloat16VectorChecker : public virtual IndexChecker {
public:
BFloat16VectorChecker(const std::string& index_type,
const std::string& metric,
int64_t dim)
: IndexChecker(index_type, metric, dim) {
min_dim_ = 2;
max_dim_ = 32768;
supported_index_ = {kFlat};
supported_metric_ = {kL2, kIP, kCosine};
}
virtual ~BFloat16VectorChecker() = default;
};
class SparseFloatVectorChecker : public virtual IndexChecker {
public:
SparseFloatVectorChecker(const std::string& index_type,
const std::string& metric,
int64_t dim)
: IndexChecker(index_type, metric, dim) {
min_dim_ = -1;
max_dim_ = -1;
need_check_dim_ = false;
supported_index_ = {kSparseInvertedIndex, kSparseWand};
supported_metric_ = {kIP, kMetricsBM25Name};
}
virtual ~SparseFloatVectorChecker() = default;
};
Status
Check(DType field_type,
const std::string& index_type,
const std::string& metric,
int64_t dim) {
if (field_type == DType::FloatVector) {
return FloatVectorIndexChecker(index_type, metric, dim).Check();
} else if (field_type == DType::Float16Vector) {
return Float16VectorChecker(index_type, metric, dim).Check();
} else if (field_type == DType::BFloat16Vector) {
return BFloat16VectorChecker(index_type, metric, dim).Check();
} else if (field_type == DType::BinaryVector) {
return BinaryVectorChecker(index_type, metric, dim).Check();
} else if (field_type == DType::SparseFloatVector) {
return SparseFloatVectorChecker(index_type, metric, dim).Check();
} else {
return Status::ParameterInvalid("Unknow data type");
}
}
void
CreateIndexTask::WrapUserIndexParams(const std::string& metrics_type) {
::milvus::proto::common::KeyValuePair p1;
p1.set_key(kIndexTypeKey);
p1.set_value(kAutoIndex);
new_extra_params_.push_back(p1);
::milvus::proto::common::KeyValuePair p2;
p2.set_key(kMetricTypeKey);
p2.set_value(metrics_type);
new_extra_params_.push_back(p2);
}
bool
CreateIndexTask::AddAutoIndexParams(KVMap* index_params) {
is_auto_index_ = true;
if (index_params->find(kMetricTypeKey) != index_params->end()) {
const auto metrics_type = index_params->at(kMetricTypeKey);
WrapUserIndexParams(metrics_type);
index_params->insert(kAutoIndexConfig.index_param.begin(),
kAutoIndexConfig.index_param.end());
(*index_params)[kMetricTypeKey] = metrics_type;
return true;
} else {
auto it = kAutoIndexConfig.index_param.find(kMetricTypeKey);
WrapUserIndexParams(it->second);
index_params->insert(kAutoIndexConfig.index_param.begin(),
kAutoIndexConfig.index_param.end());
(*index_params)[kMetricTypeKey] = it->second;
return true;
}
return true;
}
Status
CreateIndexTask::CheckTrain(const ::milvus::proto::schema::FieldSchema& field,
KVMap& index_params) {
auto index_type = index_params.at(kIndexTypeKey);
if (!IsVectorIndex(field.data_type())) {
return Status::Ok();
}
if (schema_util::IsSparseVectorType(field.data_type())) {
if (index_params[kMetricTypeKey] == kMetricsBM25Name) {
{
auto [succ, num] =
string_util::ToNumber(index_params[kBM25AvgName]);
if (!succ) {
return Status::ParameterInvalid("{}: {} is not a number",
kBM25AvgName,
index_params[kBM25AvgName]);
}
}
{
auto [succ, num] = string_util::ToNumber(index_params[kBm25K1]);
if (!succ) {
return Status::ParameterInvalid("{}: {} is not a number",
kBm25K1,
index_params[kBm25K1]);
}
if (num < 0 || num > 3.0) {
return Status::ParameterInvalid(
"{}: {} should be in range [0.0, 3.0]",
kBm25K1,
index_params[kBm25K1]);
}
}
{
auto [succ, num] = string_util::ToNumber(index_params[kBm25B]);
if (!succ) {
return Status::ParameterInvalid(
"{}: {} is not a number", kBm25B, index_params[kBm25B]);
}
if (num < 0.0 || num > 1.0) {
return Status::ParameterInvalid(
"{}: {} should be in range [0.0, 1.0]",
kBm25B,
index_params[kBm25B]);
}
}
}
}
if (!schema_util::IsSparseVectorType(field.data_type())) {
if (!FillDimension(field, &index_params)) {
return Status::ParameterInvalid();
}
}
int64_t dim = -1;
if (!schema_util::IsSparseVectorType(field.data_type())) {
dim = std::stoll(index_params.at(kDimKey));
}
std::string metric = index_params.at(kMetricTypeKey);
return Check(field.data_type(), index_type, metric, dim);
}
Status
CreateIndexTask::ParseIndexParams() {
const milvus::proto::schema::FieldSchema* field_ptr = nullptr;
for (const auto& field : schema_->fields()) {
if (field.name() == create_index_request_->field_name()) {
field_ptr = &field;
}
}
if (field_ptr == nullptr) {
auto err = string_util::SFormat("Can not found field {}",
create_index_request_->field_name());
LOG_ERROR(err);
return Status::ParameterInvalid(err);
}
if (!create_index_request_->index_name().empty()) {
index_name_ = create_index_request_->index_name();
} else {
index_name_ = field_ptr->name();
}
field_id_ = field_ptr->fieldid();
collectionid_ = GetCollectionId(schema_->name());
KVMap index_params;
for (const auto& param : create_index_request_->extra_params()) {
if (param.key() == kIndexParamsKey) {
try {
nlohmann::json data = nlohmann::json::parse(param.value());
for (auto& [key, value] : data.items()) {
if (!value.is_string()) {
index_params[key] = value.dump();
}
}
} catch (nlohmann::json::parse_error& e) {
auto err =
string_util::SFormat("Index params err: {}", e.what());
LOG_ERROR(err);
return Status::ParameterInvalid(err);
}
} else {
index_params[param.key()] = param.value();
}
}
if (IsVectorIndex(field_ptr->data_type())) {
auto it = index_params.find(kIndexTypeKey);
if (it == index_params.end() || it->second == kAutoIndex) {
// default index and auto index only support floatvector type
if (!AddAutoIndexParams(&index_params))
return Status::SegcoreErr();
}
auto metric_it = index_params.find(kMetricTypeKey);
if (metric_it == index_params.end()) {
if (field_ptr->data_type() == DType::FloatVector ||
field_ptr->data_type() == DType::BFloat16Vector ||
field_ptr->data_type() == DType::Float16Vector) {
index_params[kMetricTypeKey] = kFloatVectorDefaultMetricType;
} else if (field_ptr->data_type() == DType::BinaryVector) {
index_params[kMetricTypeKey] = kBinaryVectorDefaultMetricType;
} else if (field_ptr->data_type() == DType::SparseFloatVector) {
index_params[kMetricTypeKey] =
kSparseFloatVectorDefaultMetricType;
} else {
LOG_ERROR("Unkwon index data type: {}", field_ptr->data_type());
return Status::ParameterInvalid();
}
}
if (index_params[kMetricTypeKey] == kMetricsBM25Name) {
if (index_params.find(kBm25K1) == index_params.end()) {
index_params[kBm25K1] = "1.2";
}
if (index_params.find(kBm25B) == index_params.end()) {
index_params[kBm25B] = "0.75";
}
if (index_params.find(kBM25AvgName) == index_params.end()) {
index_params[kBM25AvgName] = "1000";
}
}
} else {
// scalar index
auto it = index_params.find(kIndexTypeKey);
if (field_ptr->data_type() == DType::VarChar) {
if (it == index_params.end()) {
index_params[kIndexTypeKey] = kDefaultStringIndexType;
} else if (!ValidateStringIndexType(it->second)) {
auto err =
string_util::SFormat("Unkown index type {}", it->second);
LOG_ERROR(err);
return Status::ParameterInvalid(err);
}
} else if (field_ptr->data_type() == DType::Float ||
field_ptr->data_type() == DType::Double ||
field_ptr->data_type() == DType::Int16 ||
field_ptr->data_type() == DType::Int8 ||
field_ptr->data_type() == DType::Int32 ||
field_ptr->data_type() == DType::Int64) {
if (it == index_params.end()) {
index_params[kIndexTypeKey] = kDefaultArithmeticIndexType;
} else if (!ValidateArithmeticIndexType(it->second)) {
auto err =
string_util::SFormat("Unkown index type {}", it->second);
LOG_ERROR(err);
return Status::ParameterInvalid(err);
}
} else if (field_ptr->data_type() == DType::Bool) {
if (it == index_params.end()) {
LOG_ERROR("no index type specified");
return Status::ParameterInvalid("no index type specified");
}
if (it->second != kInvertedIndexType) {
auto err = string_util::SFormat(
"index type {} not supported for boolean, supported: {}",
it->second,
kInvertedIndexType);
LOG_ERROR(err);
return Status::ParameterInvalid(err);
}
} else {
LOG_ERROR(
"Only int, varchar, float, double and bool fields support "
"scalar index.");
return Status::ParameterInvalid();
}
}
auto it = index_params.find(kIndexTypeKey);
if (it == index_params.end()) {
LOG_ERROR("IndexType not specified");
return Status::ParameterInvalid();
}
CHECK_STATUS(CheckTrain(*field_ptr, index_params), "");
index_params.erase(kDimKey);
index_params.erase(kMaxLengthKey);
for (const auto& param : index_params) {
::milvus::proto::common::KeyValuePair p;
p.set_key(param.first);
p.set_value(param.second);
new_index_params_.push_back(p);
}
auto type_params = field_ptr->type_params();
KVMap type_params_map;
for (const auto& param : type_params) {
::milvus::proto::common::KeyValuePair p;
p.set_key(param.key());
p.set_value(param.value());
new_type_params_.push_back(p);
}
return Status::Ok();
}
bool
CreateIndexTask::IsVectorIndex(::milvus::proto::schema::DataType dtype) {
return schema_util::IsVectorField(dtype);
}
bool
CreateIndexTask::FillDimension(
const ::milvus::proto::schema::FieldSchema& field, KVMap* index_params) {
if (!IsVectorIndex(field.data_type())) {
return true;
}
std::string dim;
if (!schema_util::FindDimFromFieldParams(field, &dim)) {
LOG_ERROR("Dimension not found in schema");
return false;
}
auto it = index_params->find(kDimKey);
if (it != index_params->end() && it->second != dim) {
LOG_ERROR("dimension mismatch, dimension in schema: {}, dimension: {}",
dim,
it->second);
return false;
} else {
(*index_params)[kDimKey] = dim;
}
return true;
}
Status
CreateIndexTask::Process(milvus::proto::segcore::FieldIndexMeta* field_meta) {
CHECK_STATUS(ParseIndexParams(), "");
field_meta->set_index_name(index_name_);
field_meta->set_fieldid(field_id_);
field_meta->set_collectionid(collectionid_);
field_meta->set_is_auto_index(is_auto_index_);
std::set<std::string> kset;
for (const auto& param : new_index_params_) {
if (kset.find(param.key()) == kset.end()) {
auto pair = field_meta->add_index_params();
pair->set_key(param.key());
pair->set_value(param.value());
kset.insert(param.key());
}
}
for (const auto& param
gitextract_0fobvlb0/
├── .clang-format
├── .gitignore
├── .gitmodules
├── .pylintrc
├── CMakeLists.txt
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── ci/
│ ├── e2e.groovy
│ ├── entrypoint.sh
│ ├── get_author_email.sh
│ ├── nightly.groovy
│ ├── rte-build.yaml
│ ├── set_docker_mirror.sh
│ └── test.sh
├── cmake/
│ ├── Findantlr4-runtime.cmake
│ └── milvus-storage.cmake
├── conanfile.py
├── examples/
│ ├── bfloat16_example.py
│ ├── binary_example.py
│ ├── bm25.py
│ ├── customize_schema.py
│ ├── customize_schema_auto_id.py
│ ├── dynamic_field.py
│ ├── example_group_by.py
│ ├── float16_example.py
│ ├── fuzzy_match.py
│ ├── hello_hybrid_bm25.py
│ ├── hello_milvus.py
│ ├── hello_milvus_array.py
│ ├── hello_milvus_delete.py
│ ├── index.py
│ ├── non_ascii_encode.py
│ ├── simple.py
│ ├── simple_auto_id.py
│ └── sparse.py
├── python/
│ ├── pyproject.toml
│ ├── requirements.txt
│ ├── setup.py
│ └── src/
│ └── milvus_lite/
│ ├── __init__.py
│ ├── cmdline.py
│ ├── server.py
│ └── server_manager.py
├── scripts/
│ ├── Dockerfile.manylinux.aarch64
│ ├── Dockerfile.manylinux.x86_64
│ ├── README.md
│ ├── build.sh
│ └── build_milvus_lite.sh
├── src/
│ ├── CMakeLists.txt
│ ├── common.h
│ ├── create_collection_task.cpp
│ ├── create_collection_task.h
│ ├── create_index_task.cpp
│ ├── create_index_task.h
│ ├── delete_task.cpp
│ ├── delete_task.h
│ ├── function/
│ │ ├── bm25_function.cpp
│ │ ├── bm25_function.h
│ │ ├── function.h
│ │ ├── function_executor.cpp
│ │ ├── function_executor.h
│ │ └── function_util.h
│ ├── hybrid_search_task.cpp
│ ├── hybrid_search_task.h
│ ├── index.cpp
│ ├── index.h
│ ├── insert_task.cpp
│ ├── insert_task.h
│ ├── milvus_id.hpp
│ ├── milvus_local.cpp
│ ├── milvus_local.h
│ ├── milvus_proxy.cpp
│ ├── milvus_proxy.h
│ ├── milvus_service_impl.cpp
│ ├── milvus_service_impl.h
│ ├── parser/
│ │ ├── Plan.g4
│ │ ├── antlr/
│ │ │ ├── PlanBaseVisitor.cpp
│ │ │ ├── PlanBaseVisitor.h
│ │ │ ├── PlanLexer.cpp
│ │ │ ├── PlanLexer.h
│ │ │ ├── PlanParser.cpp
│ │ │ ├── PlanParser.h
│ │ │ ├── PlanVisitor.cpp
│ │ │ └── PlanVisitor.h
│ │ ├── parser.cc
│ │ ├── parser.h
│ │ └── utils.h
│ ├── query_task.cpp
│ ├── query_task.h
│ ├── re_scorer.cpp
│ ├── re_scorer.h
│ ├── retrieve_result.h
│ ├── schema_util.cpp
│ ├── schema_util.h
│ ├── search_result.h
│ ├── search_task.cpp
│ ├── search_task.h
│ ├── segcore_wrapper.cpp
│ ├── segcore_wrapper.h
│ ├── server.cpp
│ ├── status.h
│ ├── storage/
│ │ ├── bm25_stats.cpp
│ │ ├── bm25_stats.h
│ │ ├── collection_data.cpp
│ │ ├── collection_data.h
│ │ ├── collection_meta.cpp
│ │ ├── collection_meta.h
│ │ ├── storage.cpp
│ │ └── storage.h
│ ├── string_util.hpp
│ ├── timer.cpp
│ ├── timer.h
│ ├── type.h
│ ├── unittest/
│ │ ├── CMakeLists.txt
│ │ ├── bm25_function_test.cpp
│ │ ├── grpc_server_test.cpp
│ │ ├── milvus_local_test.cpp
│ │ ├── milvus_proxy_test.cpp
│ │ ├── run_examples.py
│ │ ├── storage_test.cpp
│ │ ├── test_util.cpp
│ │ └── test_util.h
│ ├── upsert_task.cpp
│ └── upsert_task.h
├── tests/
│ ├── base/
│ │ ├── client_base.py
│ │ ├── collection_wrapper.py
│ │ ├── connections_wrapper.py
│ │ ├── database_wrapper.py
│ │ ├── high_level_api_wrapper.py
│ │ ├── index_wrapper.py
│ │ ├── partition_wrapper.py
│ │ ├── schema_wrapper.py
│ │ └── utility_wrapper.py
│ ├── check/
│ │ ├── func_check.py
│ │ └── param_check.py
│ ├── common/
│ │ ├── bulk_insert_data.py
│ │ ├── code_mapping.py
│ │ ├── common_func.py
│ │ ├── common_type.py
│ │ ├── constants.py
│ │ ├── cus_resource_opts.py
│ │ ├── milvus_sys.py
│ │ └── minio_comm.py
│ ├── config/
│ │ └── log_config.py
│ ├── conftest.py
│ ├── customize/
│ │ ├── README.md
│ │ ├── milvus_operator.py
│ │ ├── template/
│ │ │ ├── default.yaml
│ │ │ └── minimum.yaml
│ │ ├── test_customize_segment_size.py
│ │ └── test_simd_compat.py
│ ├── data/
│ │ └── train_embeddings.csv
│ ├── milvus_lite/
│ │ ├── test_milvus_lite_close.py
│ │ ├── test_milvus_lite_collection.py
│ │ ├── test_milvus_lite_delete.py
│ │ ├── test_milvus_lite_index.py
│ │ ├── test_milvus_lite_init.py
│ │ ├── test_milvus_lite_insert.py
│ │ ├── test_milvus_lite_migrate.py
│ │ ├── test_milvus_lite_query.py
│ │ ├── test_milvus_lite_search.py
│ │ ├── test_milvus_lite_sparse.py
│ │ └── test_milvus_lite_stability.py
│ ├── pytest.ini
│ ├── requirements.txt
│ ├── run.sh
│ ├── test_bm25.py
│ ├── test_delete.py
│ ├── test_dump_tool.py
│ ├── test_hybrid_search.py
│ ├── test_query.py
│ ├── test_schema.py
│ ├── test_search.py
│ ├── test_sparse.py
│ ├── testcases/
│ │ └── test_collection.py
│ └── utils/
│ ├── api_request.py
│ ├── thread_util.py
│ ├── util_common.py
│ ├── util_k8s.py
│ ├── util_log.py
│ ├── util_pymilvus.py
│ └── wrapper.py
└── thirdparty/
├── knowhere-android.patch
├── knowhere.patch
└── milvus.patch
SYMBOL INDEX (1557 symbols across 132 files)
FILE: conanfile.py
class MilvusLiteConan (line 5) | class MilvusLiteConan(ConanFile):
method configure (line 70) | def configure(self):
method config_options (line 78) | def config_options(self):
method requirements (line 82) | def requirements(self):
method imports (line 88) | def imports(self):
method build (line 92) | def build(self):
FILE: examples/bfloat16_example.py
function gen_bf16_vectors (line 13) | def gen_bf16_vectors(num, dim):
function bf16_vector_search (line 23) | def bf16_vector_search():
FILE: examples/binary_example.py
function gen_binary_vectors (line 14) | def gen_binary_vectors(num, dim):
function binary_vector_search (line 25) | def binary_vector_search():
FILE: examples/example_group_by.py
function generate_random_hex_string (line 7) | def generate_random_hex_string(length):
FILE: examples/float16_example.py
function gen_fp16_vectors (line 16) | def gen_fp16_vectors(num, dim):
function fp16_vector_search (line 26) | def fp16_vector_search():
FILE: examples/hello_hybrid_bm25.py
function random_embedding (line 67) | def random_embedding(texts):
FILE: examples/hello_milvus_delete.py
function fetch_data_by_pk (line 34) | def fetch_data_by_pk(pk):
FILE: examples/sparse.py
function generate_sparse_vector (line 7) | def generate_sparse_vector(dimension: int, non_zero_count: int) -> dict:
FILE: python/setup.py
class CMakeBuild (line 34) | class CMakeBuild(_bdist_wheel):
method finalize_options (line 35) | def finalize_options(self):
method copy_lib (line 45) | def copy_lib(self, lib_path, dst_dir, pick_libs):
method _pack_macos (line 53) | def _pack_macos(self, src_dir: str, dst_dir: str):
method _pack_linux (line 74) | def _pack_linux(self, src_dir: str, dst_dir: str):
method run (line 90) | def run(self):
function test_suite (line 137) | def test_suite():
function parse_requirements (line 143) | def parse_requirements(file_name: str) -> List[str]:
FILE: python/src/milvus_lite/cmdline.py
class MilvusEncoder (line 24) | class MilvusEncoder(json.JSONEncoder):
method default (line 25) | def default(self, o):
function dump_func (line 33) | def dump_func(args):
function bfloat16_to_float32 (line 37) | def bfloat16_to_float32(byte_data):
function binary_to_int_list (line 45) | def binary_to_int_list(packed_bytes):
function dump_collection (line 50) | def dump_collection(db_file, collection_name, path):
function main (line 105) | def main():
FILE: python/src/milvus_lite/server.py
class Server (line 30) | class Server:
method __init__ (line 37) | def __init__(self, db_file: str, address: Optional[str] = None):
method init (line 62) | def init(self) -> bool:
method milvus_bin (line 71) | def milvus_bin(self):
method log_level (line 75) | def log_level(self):
method uds_path (line 79) | def uds_path(self):
method args (line 83) | def args(self):
method start (line 88) | def start(self) -> bool:
method stop (line 119) | def stop(self):
method __del__ (line 143) | def __del__(self):
FILE: python/src/milvus_lite/server_manager.py
class ServerManager (line 24) | class ServerManager:
method __init__ (line 25) | def __init__(self):
method start_and_get_uri (line 29) | def start_and_get_uri(self, path: str, args=None) -> Optional[str]:
method release_server (line 41) | def release_server(self, path: str):
method release_all (line 50) | def release_all(self):
method __del__ (line 55) | def __del__(self):
FILE: src/common.h
function namespace (line 21) | namespace milvus::local {
FILE: src/create_collection_task.cpp
type milvus::local (line 28) | namespace milvus::local {
function Status (line 45) | Status
function Status (line 58) | Status
function Status (line 253) | Status
function Status (line 313) | Status
function Status (line 359) | Status
function Status (line 398) | Status
FILE: src/create_collection_task.h
function namespace (line 24) | namespace milvus::local {
FILE: src/create_index_task.cpp
type milvus::local (line 32) | namespace milvus::local {
class AutoIndexConfig (line 70) | class AutoIndexConfig final : NonCopyableNonMovable {
method AutoIndexConfig (line 72) | AutoIndexConfig()
class IndexChecker (line 88) | class IndexChecker : NonCopyableNonMovable {
method IndexChecker (line 90) | IndexChecker(const std::string& index_type,
function Status (line 115) | Status
class FloatVectorIndexChecker (line 146) | class FloatVectorIndexChecker : public virtual IndexChecker {
method FloatVectorIndexChecker (line 148) | FloatVectorIndexChecker(const std::string& index_type,
class BinaryVectorChecker (line 161) | class BinaryVectorChecker : public virtual IndexChecker {
method BinaryVectorChecker (line 163) | BinaryVectorChecker(const std::string& index_type,
class Float16VectorChecker (line 177) | class Float16VectorChecker : public virtual IndexChecker {
method Float16VectorChecker (line 179) | Float16VectorChecker(const std::string& index_type,
class BFloat16VectorChecker (line 192) | class BFloat16VectorChecker : public virtual IndexChecker {
method BFloat16VectorChecker (line 194) | BFloat16VectorChecker(const std::string& index_type,
class SparseFloatVectorChecker (line 207) | class SparseFloatVectorChecker : public virtual IndexChecker {
method SparseFloatVectorChecker (line 209) | SparseFloatVectorChecker(const std::string& index_type,
function Status (line 223) | Status
function Status (line 277) | Status
function Status (line 338) | Status
function Status (line 526) | Status
FILE: src/create_index_task.h
function namespace (line 28) | namespace milvus::local {
FILE: src/delete_task.cpp
type milvus::local (line 24) | namespace milvus::local {
function Status (line 26) | Status
FILE: src/delete_task.h
function namespace (line 23) | namespace milvus::local {
FILE: src/function/bm25_function.cpp
type milvus::local::function (line 25) | namespace milvus::local::function {
function Status (line 31) | Status
function Status (line 76) | Status
function Status (line 106) | Status
FILE: src/function/function.h
function namespace (line 30) | namespace milvus::local::function {
FILE: src/function/function_executor.cpp
type milvus::local::function (line 27) | namespace milvus::local::function {
function Status (line 86) | Status
function Status (line 94) | Status
function Status (line 115) | Status
FILE: src/function/function_executor.h
function namespace (line 27) | namespace milvus::local::function {
FILE: src/function/function_util.h
function namespace (line 21) | namespace milvus::local::function {
FILE: src/hybrid_search_task.cpp
type milvus::local (line 31) | namespace milvus::local {
function Status (line 33) | Status
function Status (line 90) | Status
function Status (line 134) | Status
function Status (line 197) | Status
function Status (line 289) | Status
FILE: src/hybrid_search_task.h
function namespace (line 29) | namespace milvus::local {
FILE: src/index.cpp
type milvus::local (line 21) | namespace milvus::local {
function Status (line 23) | Status
function Status (line 48) | Status
function Status (line 60) | Status
function Status (line 71) | Status
function Status (line 82) | Status
function Status (line 95) | Status
FILE: src/index.h
function namespace (line 25) | namespace milvus::local {
FILE: src/insert_task.cpp
type milvus::local (line 33) | namespace milvus::local {
function Status (line 145) | Status
function Status (line 203) | Status
FILE: src/insert_task.h
function namespace (line 26) | namespace milvus::local {
FILE: src/milvus_id.hpp
type milvus::local (line 21) | namespace milvus::local {
class MilvusID (line 23) | class MilvusID {
type IDType (line 24) | enum IDType { INT, STR }
method MilvusID (line 27) | explicit MilvusID(int64_t id) {
method MilvusID (line 32) | explicit MilvusID(const std::string& id) {
method MilvusID (line 37) | explicit MilvusID(const ::milvus::proto::schema::IDs& ids, int64_t i...
method AppendToIDs (line 68) | void
FILE: src/milvus_local.cpp
type milvus::local (line 29) | namespace milvus::local {
function Status (line 56) | Status
function Status (line 105) | Status
function Status (line 116) | Status
function Status (line 126) | Status
function Status (line 136) | Status
function Status (line 166) | Status
function Status (line 194) | Status
function Status (line 226) | Status
function Status (line 246) | Status
function Status (line 268) | Status
function Status (line 277) | Status
function Status (line 287) | Status
function Status (line 302) | Status
function Status (line 314) | Status
function Status (line 336) | Status
function Status (line 348) | Status
FILE: src/milvus_local.h
function namespace (line 31) | namespace milvus::local {
FILE: src/milvus_proxy.cpp
type milvus::local (line 48) | namespace milvus::local {
function Status (line 61) | Status
function Status (line 66) | Status
function Status (line 91) | Status
function Status (line 102) | Status
function Status (line 116) | Status
function Status (line 132) | Status
function Status (line 149) | Status
function Status (line 180) | Status
function Status (line 215) | Status
function Status (line 235) | Status
function Status (line 301) | Status
function Status (line 348) | Status
function Status (line 372) | Status
function Status (line 419) | Status
function Status (line 433) | Status
function Status (line 453) | Status
function Status (line 486) | Status
function Status (line 493) | Status
function Status (line 536) | Status
FILE: src/milvus_proxy.h
function namespace (line 24) | namespace milvus::local {
FILE: src/milvus_service_impl.cpp
type milvus::local (line 23) | namespace milvus::local {
function Status2Response (line 25) | void
FILE: src/milvus_service_impl.h
function namespace (line 21) | namespace milvus::local {
FILE: src/parser/antlr/PlanBaseVisitor.h
function class (line 15) | class PlanBaseVisitor : public PlanVisitor {
function virtual (line 22) | virtual std::any visitParens(PlanParser::ParensContext *ctx) override {
function virtual (line 26) | virtual std::any visitString(PlanParser::StringContext *ctx) override {
function virtual (line 30) | virtual std::any visitFloating(PlanParser::FloatingContext *ctx) override {
function virtual (line 34) | virtual std::any visitJSONContainsAll(PlanParser::JSONContainsAllContext...
function virtual (line 38) | virtual std::any visitLogicalOr(PlanParser::LogicalOrContext *ctx) overr...
function virtual (line 42) | virtual std::any visitMulDivMod(PlanParser::MulDivModContext *ctx) overr...
function virtual (line 46) | virtual std::any visitIdentifier(PlanParser::IdentifierContext *ctx) ove...
function virtual (line 50) | virtual std::any visitLike(PlanParser::LikeContext *ctx) override {
function virtual (line 54) | virtual std::any visitLogicalAnd(PlanParser::LogicalAndContext *ctx) ove...
function virtual (line 58) | virtual std::any visitEquality(PlanParser::EqualityContext *ctx) override {
function virtual (line 62) | virtual std::any visitBoolean(PlanParser::BooleanContext *ctx) override {
function virtual (line 66) | virtual std::any visitShift(PlanParser::ShiftContext *ctx) override {
function virtual (line 70) | virtual std::any visitReverseRange(PlanParser::ReverseRangeContext *ctx)...
function virtual (line 74) | virtual std::any visitBitOr(PlanParser::BitOrContext *ctx) override {
function virtual (line 78) | virtual std::any visitAddSub(PlanParser::AddSubContext *ctx) override {
function virtual (line 82) | virtual std::any visitRelational(PlanParser::RelationalContext *ctx) ove...
function virtual (line 86) | virtual std::any visitArrayLength(PlanParser::ArrayLengthContext *ctx) o...
function virtual (line 90) | virtual std::any visitTerm(PlanParser::TermContext *ctx) override {
function virtual (line 94) | virtual std::any visitJSONContains(PlanParser::JSONContainsContext *ctx)...
function virtual (line 98) | virtual std::any visitRange(PlanParser::RangeContext *ctx) override {
function virtual (line 102) | virtual std::any visitUnary(PlanParser::UnaryContext *ctx) override {
function virtual (line 106) | virtual std::any visitInteger(PlanParser::IntegerContext *ctx) override {
function virtual (line 110) | virtual std::any visitArray(PlanParser::ArrayContext *ctx) override {
function virtual (line 114) | virtual std::any visitJSONContainsAny(PlanParser::JSONContainsAnyContext...
function virtual (line 118) | virtual std::any visitBitXor(PlanParser::BitXorContext *ctx) override {
function virtual (line 122) | virtual std::any visitExists(PlanParser::ExistsContext *ctx) override {
function virtual (line 126) | virtual std::any visitBitAnd(PlanParser::BitAndContext *ctx) override {
function virtual (line 130) | virtual std::any visitEmptyTerm(PlanParser::EmptyTermContext *ctx) overr...
function virtual (line 134) | virtual std::any visitPower(PlanParser::PowerContext *ctx) override {
FILE: src/parser/antlr/PlanLexer.cpp
type PlanLexerStaticData (line 16) | struct PlanLexerStaticData final {
method PlanLexerStaticData (line 17) | PlanLexerStaticData(std::vector<std::string> ruleNames,
method PlanLexerStaticData (line 27) | PlanLexerStaticData(const PlanLexerStaticData&) = delete;
method PlanLexerStaticData (line 28) | PlanLexerStaticData(PlanLexerStaticData&&) = delete;
method PlanLexerStaticData (line 29) | PlanLexerStaticData& operator=(const PlanLexerStaticData&) = delete;
method PlanLexerStaticData (line 30) | PlanLexerStaticData& operator=(PlanLexerStaticData&&) = delete;
function planlexerLexerInitialize (line 50) | void planlexerLexerInitialize() {
FILE: src/parser/antlr/PlanLexer.h
function class (line 12) | class PlanLexer : public antlr4::Lexer {
FILE: src/parser/antlr/PlanParser.cpp
type PlanParserStaticData (line 16) | struct PlanParserStaticData final {
method PlanParserStaticData (line 17) | PlanParserStaticData(std::vector<std::string> ruleNames,
method PlanParserStaticData (line 24) | PlanParserStaticData(const PlanParserStaticData&) = delete;
method PlanParserStaticData (line 25) | PlanParserStaticData(PlanParserStaticData&&) = delete;
method PlanParserStaticData (line 26) | PlanParserStaticData& operator=(const PlanParserStaticData&) = delete;
method PlanParserStaticData (line 27) | PlanParserStaticData& operator=(PlanParserStaticData&&) = delete;
function planParserInitialize (line 45) | void planParserInitialize() {
FILE: src/parser/antlr/PlanParser.h
function class (line 12) | class PlanParser : public antlr4::Parser {
FILE: src/parser/antlr/PlanVisitor.h
function class (line 16) | class PlanVisitor : public antlr4::tree::AbstractParseTreeVisitor {
FILE: src/parser/parser.cc
type milvus::local (line 17) | namespace milvus::local {
function ParserToMessage (line 19) | std::string
function ParseIdentifier (line 46) | std::shared_ptr<milvus::proto::plan::Expr>
FILE: src/parser/parser.h
function namespace (line 27) | namespace milvus::local {
function virtual (line 931) | virtual std::any
function virtual (line 1034) | virtual std::any
function virtual (line 1059) | virtual std::any
function virtual (line 1142) | virtual std::any
FILE: src/parser/utils.h
function namespace (line 26) | namespace milvus::local {
function arithmeticDtype (line 164) | inline bool
function proto (line 184) | inline proto::schema::DataType
function canArithmeticDtype (line 197) | inline bool
function proto (line 214) | inline proto::schema::DataType
function GetPartitionKeyField (line 262) | struct SchemaHelper {
function proto (line 284) | const proto::schema::FieldSchema&
function proto (line 291) | const proto::schema::FieldSchema&
function proto (line 300) | const proto::schema::FieldSchema&
function proto (line 312) | const proto::schema::FieldSchema&
function GetVectorDimFromID (line 319) | int
function SchemaHelper (line 337) | inline SchemaHelper
function std (line 361) | inline std::string
function proto (line 774) | inline proto::plan::OpType
function checkDirectComparisonBinaryField (line 818) | inline bool
FILE: src/query_task.cpp
type milvus::local (line 31) | namespace milvus::local {
function Status (line 80) | Status
function Status (line 138) | Status
FILE: src/query_task.h
function namespace (line 25) | namespace milvus::local {
FILE: src/re_scorer.cpp
type milvus::local (line 25) | namespace milvus::local {
function Status (line 68) | Status
FILE: src/re_scorer.h
function namespace (line 25) | namespace milvus::local {
FILE: src/retrieve_result.h
function namespace (line 20) | namespace milvus::local {
FILE: src/schema_util.cpp
type milvus::local (line 27) | namespace milvus::local {
type schema_util (line 28) | namespace schema_util {
function GetField (line 32) | std::any
function IsVectorField (line 100) | bool
function IsSparseVectorType (line 109) | bool
function FindDimFromFieldParams (line 114) | bool
function GetDim (line 133) | int64_t
function FillEmptyField (line 158) | bool
function FindDimFromSchema (line 227) | bool
function DataTypeToVectorType (line 238) | std::optional<::milvus::proto::plan::VectorType>
function Status (line 255) | Status
function MergeIndexs (line 294) | std::string
function GetIndexMetricType (line 304) | std::string
function GetPkId (line 326) | std::optional<int64_t>
function GetPkName (line 336) | std::optional<std::string>
function PickFieldDataByIndex (line 346) | bool
function SliceFieldData (line 460) | bool
function TranslateOutputFields (line 484) | bool
function ReduceFieldByIDs (line 571) | bool
function Status (line 606) | Status
function SchemaEquals (line 621) | bool
function CheckParamsEqual (line 664) | bool
function CheckValueFieldEqual (line 685) | bool
function FillInFieldInfo (line 722) | void
function GetOutputFieldsIds (line 763) | bool
FILE: src/schema_util.h
function namespace (line 32) | namespace milvus::local {
FILE: src/search_result.h
function namespace (line 24) | namespace milvus::local {
FILE: src/search_task.cpp
type milvus::local (line 32) | namespace milvus::local {
function Status (line 158) | Status
FILE: src/search_task.h
function namespace (line 26) | namespace milvus::local {
FILE: src/segcore_wrapper.cpp
type milvus::local (line 35) | namespace milvus::local {
class RetrievePlanWrapper (line 39) | class RetrievePlanWrapper final : private NonCopyableNonMovable {
method RetrievePlanWrapper (line 41) | RetrievePlanWrapper() : plan_(nullptr) {
class SearchPlanWrapper (line 51) | class SearchPlanWrapper final : private NonCopyableNonMovable {
method SearchPlanWrapper (line 53) | SearchPlanWrapper() : plan_(nullptr) {
class PlaceholderGroupWrapper (line 63) | class PlaceholderGroupWrapper final : private NonCopyableNonMovable {
method PlaceholderGroupWrapper (line 65) | PlaceholderGroupWrapper() : group_(nullptr) {
class SearchResultWrapper (line 75) | class SearchResultWrapper final : private NonCopyableNonMovable {
method SearchResultWrapper (line 77) | SearchResultWrapper() : ret_(nullptr) {
function Status (line 104) | Status
function Status (line 143) | Status
function Status (line 154) | Status
function Status (line 195) | Status
function Status (line 240) | Status
function Status (line 297) | Status
FILE: src/segcore_wrapper.h
function namespace (line 33) | namespace milvus::local {
FILE: src/server.cpp
function BlockLock (line 25) | int
function main (line 52) | int
FILE: src/status.h
function namespace (line 21) | namespace milvus::local {
FILE: src/storage/bm25_stats.cpp
type milvus::local (line 19) | namespace milvus::local {
FILE: src/storage/bm25_stats.h
function namespace (line 38) | namespace milvus::local {
FILE: src/storage/collection_data.cpp
type milvus::local (line 26) | namespace milvus::local {
FILE: src/storage/collection_data.h
function namespace (line 54) | namespace milvus::local {
FILE: src/storage/collection_meta.cpp
type milvus::local (line 24) | namespace milvus::local {
FILE: src/storage/collection_meta.h
function std (line 83) | const std::string&
function AddIndex (line 87) | bool
function GetIndex (line 97) | bool
FILE: src/storage/storage.cpp
type milvus::local (line 32) | namespace milvus::local {
FILE: src/storage/storage.h
function class (line 39) | class Storage final {
function GetTokenNQ (line 139) | int32_t
FILE: src/string_util.hpp
type milvus::local (line 27) | namespace milvus::local {
type string_util (line 29) | namespace string_util {
function ToLower (line 31) | inline std::string
function ToUpper (line 41) | inline std::string
function Trim (line 51) | inline std::string
function SFormat (line 61) | inline std::string
function Join (line 67) | inline std::string
function IsAlpha (line 72) | inline bool
function GenRandomString (line 80) | inline std::string
function ToNumber (line 101) | inline std::pair<bool, double>
FILE: src/timer.cpp
type milvus::local (line 18) | namespace milvus::local {
function InitializeTimer (line 21) | void
function RecordEvent (line 26) | void
function PrintTimerRecords (line 31) | void
function StopTimer (line 36) | void
FILE: src/timer.h
function Start (line 35) | void
function DoRecord (line 45) | void
FILE: src/type.h
function namespace (line 21) | namespace milvus::local {
FILE: src/unittest/bm25_function_test.cpp
type milvus::local::function (line 19) | namespace milvus::local::function {
type test (line 20) | namespace test {
function TEST (line 26) | TEST(BM25FunctionTest, normal) {
FILE: src/unittest/grpc_server_test.cpp
type milvus::local (line 24) | namespace milvus::local {
type test (line 25) | namespace test {
function TEST (line 29) | TEST(MilvusServiceImplTest, create_collection) {
function TEST (line 68) | TEST(MilvusServiceImplTest, CreateIndex) {
function TEST (line 98) | TEST(MilvusServiceImplTest, Insert) {
function TEST (line 119) | TEST(MilvusServiceImplTest, Upsert) {
function TEST (line 160) | TEST(MilvusServiceImplTest, Search) {
function TEST (line 213) | TEST(MilvusServiceImplTest, Query) {
function TEST (line 243) | TEST(MilvusServiceImplTest, Delete) {
function TEST (line 297) | TEST(MilvusServiceImplTest, describe_collection) {
FILE: src/unittest/milvus_local_test.cpp
type milvus::local (line 30) | namespace milvus::local {
type test (line 32) | namespace test {
function TEST (line 34) | TEST(MilvusLocal, h) {
function TEST (line 147) | TEST(MilvusLocal, parser) {
FILE: src/unittest/milvus_proxy_test.cpp
type milvus::local (line 25) | namespace milvus::local {
type test (line 26) | namespace test {
function TEST (line 30) | TEST(MilvusProxyTest, CreateCollection) {
function TEST (line 57) | TEST(MilvusProxyTest, CreateIndex) {
function TEST (line 77) | TEST(MilvusProxyTest, Insert) {
function TEST (line 93) | TEST(MilvusProxyTest, search) {
function TEST (line 161) | TEST(MilvusProxyTest, query) {
function TEST (line 197) | TEST(MilvusProxyTest, delete) {
FILE: src/unittest/run_examples.py
function run_all (line 18) | def run_all(py_path):
FILE: src/unittest/storage_test.cpp
type milvus::local (line 20) | namespace milvus::local {
function TEST (line 22) | TEST(Storage, h) {
FILE: src/unittest/test_util.cpp
type milvus::local (line 30) | namespace milvus::local {
type test (line 31) | namespace test {
function CreateCollection (line 33) | std::string
function CreateBM25Function (line 81) | std::string
function CreateVectorIndex (line 93) | std::string
function CreateData (line 110) | milvus::local::Rows
function GetCreateCollectionRequestProto (line 147) | ::milvus::proto::milvus::CreateCollectionRequest
function GetLoadCollectionRequestProto (line 157) | ::milvus::proto::milvus::LoadCollectionRequest
function GetCreateIndexRequestProto (line 164) | ::milvus::proto::milvus::CreateIndexRequest
function GetInsertRequestProto (line 187) | ::milvus::proto::milvus::InsertRequest
function GetUpsertRequestProto (line 228) | ::milvus::proto::milvus::UpsertRequest
function GetSearchRequestProto (line 269) | ::milvus::proto::milvus::SearchRequest
function GetQueryRequestProto (line 322) | ::milvus::proto::milvus::QueryRequest
function GetDeleteRequestProto (line 354) | ::milvus::proto::milvus::DeleteRequest
function GetDescribeCollectionRequest (line 363) | ::milvus::proto::milvus::DescribeCollectionRequest
FILE: src/unittest/test_util.h
function namespace (line 22) | namespace milvus::local {
FILE: src/upsert_task.cpp
type milvus::local (line 7) | namespace milvus::local {
function Status (line 9) | Status
FILE: src/upsert_task.h
function namespace (line 8) | namespace milvus::local {
FILE: tests/base/client_base.py
class Base (line 24) | class Base:
method setup_class (line 38) | def setup_class(self):
method teardown_class (line 41) | def teardown_class(self):
method setup_method (line 44) | def setup_method(self, method):
method teardown_method (line 57) | def teardown_method(self, method):
class TestcaseBase (line 124) | class TestcaseBase(Base):
method _connect (line 130) | def _connect(self, enable_milvus_client_api=False, enable_milvus_local...
method init_collection_wrap (line 163) | def init_collection_wrap(self, name=None, schema=None, check_task=None...
method init_multi_fields_collection_wrap (line 177) | def init_multi_fields_collection_wrap(self, name=cf.gen_unique_str()):
method init_partition_wrap (line 186) | def init_partition_wrap(self, collection_wrap=None, name=None, descrip...
method insert_data_general (line 197) | def insert_data_general(self, prefix="test", insert_data=False, nb=ct....
method init_collection_general (line 239) | def init_collection_general(self, prefix="test", insert_data=False, nb...
method insert_entities_into_two_partitions_in_half (line 321) | def insert_entities_into_two_partitions_in_half(self, half, prefix='qu...
method collection_insert_multi_segments_one_shard (line 342) | def collection_insert_multi_segments_one_shard(self, collection_prefix...
method init_resource_group (line 361) | def init_resource_group(self, name, using="default", timeout=None, che...
method init_user_with_privilege (line 372) | def init_user_with_privilege(self, privilege_object, object_name, priv...
FILE: tests/base/collection_wrapper.py
class ApiCollectionWrapper (line 24) | class ApiCollectionWrapper:
method __init__ (line 27) | def __init__(self, active_trace=False):
method init_collection (line 30) | def init_collection(self, name, schema=None, using="default", check_ta...
method schema (line 45) | def schema(self):
method description (line 49) | def description(self):
method name (line 53) | def name(self):
method is_empty (line 57) | def is_empty(self):
method num_entities (line 62) | def num_entities(self):
method num_shards (line 67) | def num_shards(self):
method num_entities_without_flush (line 71) | def num_entities_without_flush(self):
method primary_field (line 75) | def primary_field(self):
method aliases (line 79) | def aliases(self):
method construct_from_dataframe (line 83) | def construct_from_dataframe(self, name, dataframe, check_task=None, c...
method drop (line 92) | def drop(self, check_task=None, check_items=None, **kwargs):
method load (line 102) | def load(self, partition_names=None, replica_number=NaN, timeout=None,...
method release (line 113) | def release(self, check_task=None, check_items=None, **kwargs):
method insert (line 124) | def insert(self, data, partition_name=None, check_task=None, check_ite...
method flush (line 153) | def flush(self, check_task=None, check_items=None, **kwargs):
method search (line 164) | def search(self, data, anns_field, param, limit, expr=None,
method hybrid_search (line 180) | def hybrid_search(self, reqs, rerank, limit,
method search_iterator (line 195) | def search_iterator(self, data, anns_field, param, batch_size, limit=-...
method query (line 211) | def query(self, expr, output_fields=None, partition_names=None, timeou...
method query_iterator (line 225) | def query_iterator(self, batch_size=1000, limit=-1, expr=None, output_...
method partitions (line 240) | def partitions(self):
method partition (line 244) | def partition(self, partition_name, check_task=None, check_items=None):
method has_partition (line 252) | def has_partition(self, partition_name, check_task=None, check_items=N...
method drop_partition (line 260) | def drop_partition(self, partition_name, check_task=None, check_items=...
method create_partition (line 271) | def create_partition(self, partition_name, check_task=None, check_item...
method indexes (line 279) | def indexes(self):
method index (line 283) | def index(self, check_task=None, check_items=None):
method create_index (line 290) | def create_index(self, field_name, index_params=None, index_name=None,...
method has_index (line 301) | def has_index(self, index_name=None, check_task=None, check_items=None...
method drop_index (line 312) | def drop_index(self, index_name=None, check_task=None, check_items=Non...
method delete (line 324) | def delete(self, expr, partition_name=None, timeout=None, check_task=N...
method upsert (line 332) | def upsert(self, data, partition_name=None, timeout=None, check_task=N...
method compact (line 340) | def compact(self, timeout=None, check_task=None, check_items=None, **k...
method get_compaction_state (line 348) | def get_compaction_state(self, timeout=None, check_task=None, check_it...
method get_compaction_plans (line 356) | def get_compaction_plans(self, timeout=None, check_task=None, check_it...
method wait_for_compaction_completed (line 363) | def wait_for_compaction_completed(self, timeout=None, **kwargs):
method get_replicas (line 370) | def get_replicas(self, timeout=None, check_task=None, check_items=None...
method describe (line 378) | def describe(self, timeout=None, check_task=None, check_items=None):
method alter_index (line 386) | def alter_index(self, index_name, extra_params={}, timeout=None, chec...
method set_properties (line 394) | def set_properties(self, extra_params={}, timeout=None, check_task=No...
FILE: tests/base/connections_wrapper.py
class ApiConnectionsWrapper (line 11) | class ApiConnectionsWrapper:
method __init__ (line 12) | def __init__(self):
method add_connection (line 15) | def add_connection(self, check_task=None, check_items=None, **kwargs):
method disconnect (line 21) | def disconnect(self, alias, check_task=None, check_items=None):
method remove_connection (line 27) | def remove_connection(self, alias, check_task=None, check_items=None):
method connect (line 33) | def connect(self, alias=DefaultConfig.DEFAULT_USING, user="", password...
method has_connection (line 41) | def has_connection(self, alias=DefaultConfig.DEFAULT_USING, check_task...
method list_connections (line 53) | def list_connections(self, check_task=None, check_items=None):
method get_connection_addr (line 59) | def get_connection_addr(self, alias, check_task=None, check_items=None):
method MilvusClient (line 66) | def MilvusClient(self, check_task=None, check_items=None, **kwargs):
FILE: tests/base/database_wrapper.py
class ApiDatabaseWrapper (line 9) | class ApiDatabaseWrapper:
method create_database (line 13) | def create_database(self, db_name, using="default", timeout=None, chec...
method using_database (line 19) | def using_database(self, db_name, using="default", check_task=None, ch...
method drop_database (line 25) | def drop_database(self, db_name, using="default", timeout=None, check_...
method list_database (line 31) | def list_database(self, using="default", timeout=None, check_task=None...
FILE: tests/base/high_level_api_wrapper.py
class HighLevelApiWrapper (line 25) | class HighLevelApiWrapper:
method __init__ (line 29) | def __init__(self, active_trace=False):
method init_milvus_client (line 32) | def init_milvus_client(self, uri, user="", password="", db_name="", to...
method init_collection (line 44) | def init_collection(self, name, schema=None, using="default", check_ta...
method create_schema (line 60) | def create_schema(self, client, timeout=None, check_task=None,
method create_collection (line 72) | def create_collection(self, client, collection_name, dimension, timeou...
method has_collection (line 84) | def has_collection(self, client, collection_name, timeout=None, check_...
method insert (line 97) | def insert(self, client, collection_name, data, timeout=None, check_ta...
method upsert (line 109) | def upsert(self, client, collection_name, data, timeout=None, check_ta...
method search (line 121) | def search(self, client, collection_name, data, limit=10, filter=None,...
method query (line 135) | def query(self, client, collection_name, timeout=None, check_task=None...
method get (line 147) | def get(self, client, collection_name, ids, output_fields=None,
method num_entities (line 161) | def num_entities(self, client, collection_name, timeout=None, check_ta...
method delete (line 173) | def delete(self, client, collection_name, timeout=None, check_task=Non...
method flush (line 185) | def flush(self, client, collection_name, timeout=None, check_task=None...
method describe_collection (line 197) | def describe_collection(self, client, collection_name, timeout=None, c...
method list_collections (line 209) | def list_collections(self, client, timeout=None, check_task=None, chec...
method drop_collection (line 220) | def drop_collection(self, client, collection_name, check_task=None, ch...
method list_partitions (line 229) | def list_partitions(self, client, collection_name, check_task=None, ch...
method list_indexes (line 238) | def list_indexes(self, client, collection_name, check_task=None, check...
method get_load_state (line 247) | def get_load_state(self, client, collection_name, check_task=None, che...
method prepare_index_params (line 256) | def prepare_index_params(self, client, timeout=None, check_task=None, ...
method load_collection (line 267) | def load_collection(self, client, collection_name, timeout=None, check...
method release_collection (line 279) | def release_collection(self, client, collection_name, timeout=None, ch...
method load_partitions (line 291) | def load_partitions(self, client, collection_name, partition_names, ti...
method release_partitions (line 305) | def release_partitions(self, client, collection_name, partition_names,...
method rename_collection (line 319) | def rename_collection(self, client, old_name, new_name, timeout=None, ...
method use_database (line 333) | def use_database(self, client, db_name, timeout=None, check_task=None,...
method create_partition (line 346) | def create_partition(self, client, collection_name, partition_name, ti...
method list_partitions (line 360) | def list_partitions(self, client, collection_name, timeout=None, check...
method drop_partition (line 373) | def drop_partition(self, client, collection_name, partition_name, time...
method has_partition (line 387) | def has_partition(self, client, collection_name, partition_name, timeo...
method get_partition_stats (line 401) | def get_partition_stats(self, client, collection_name, partition_name,...
method prepare_index_params (line 415) | def prepare_index_params(self, client, check_task=None, check_items=No...
method create_index (line 425) | def create_index(self, client, collection_name, index_params, timeout=...
method drop_index (line 439) | def drop_index(self, client, collection_name, index_name, timeout=None...
method describe_index (line 453) | def describe_index(self, client, collection_name, index_name, timeout=...
method list_indexes (line 467) | def list_indexes(self, client, collection_name, timeout=None, check_ta...
method create_alias (line 480) | def create_alias(self, client, collection_name, alias, timeout=None, c...
method drop_alias (line 494) | def drop_alias(self, client, alias, timeout=None, check_task=None, che...
method alter_alias (line 507) | def alter_alias(self, client, collection_name, alias, timeout=None, ch...
method describe_alias (line 521) | def describe_alias(self, client, alias, timeout=None, check_task=None,...
method list_aliases (line 534) | def list_aliases(self, client, collection_name, timeout=None, check_ta...
method using_database (line 546) | def using_database(self, client, db_name, timeout=None, check_task=Non...
method create_user (line 558) | def create_user(self, user_name, password, timeout=None, check_task=No...
method drop_user (line 569) | def drop_user(self, user_name, timeout=None, check_task=None, check_it...
method update_password (line 579) | def update_password(self, user_name, old_password, new_password, reset...
method list_users (line 593) | def list_users(self, timeout=None, check_task=None, check_items=None, ...
method describe_user (line 603) | def describe_user(self, user_name, timeout=None, check_task=None, chec...
method create_role (line 613) | def create_role(self, role_name, timeout=None, check_task=None, check_...
method drop_role (line 623) | def drop_role(self, role_name, timeout=None, check_task=None, check_it...
method describe_role (line 633) | def describe_role(self, role_name, timeout=None, check_task=None, chec...
method list_roles (line 643) | def list_roles(self, timeout=None, check_task=None, check_items=None, ...
method grant_role (line 653) | def grant_role(self, user_name, role_name, timeout=None, check_task=No...
method revoke_role (line 663) | def revoke_role(self, user_name, role_name, timeout=None, check_task=N...
method grant_privilege (line 673) | def grant_privilege(self, role_name, object_type, privilege, object_na...
method revoke_privilege (line 686) | def revoke_privilege(self, role_name, object_type, privilege, object_n...
method hybrid_search (line 699) | def hybrid_search(self, client, reqs, rerank, limit,
FILE: tests/base/index_wrapper.py
class ApiIndexWrapper (line 12) | class ApiIndexWrapper:
method init_index (line 15) | def init_index(self, collection, field_name, index_params, index_name=...
method drop (line 32) | def drop(self, index_name=None, check_task=None, check_items=None, **k...
method params (line 44) | def params(self):
method collection_name (line 48) | def collection_name(self):
method field_name (line 52) | def field_name(self):
FILE: tests/base/partition_wrapper.py
class ApiPartitionWrapper (line 15) | class ApiPartitionWrapper:
method init_partition (line 18) | def init_partition(self, collection, name, description="",
method description (line 29) | def description(self):
method name (line 33) | def name(self):
method is_empty (line 37) | def is_empty(self):
method num_entities (line 42) | def num_entities(self):
method drop (line 46) | def drop(self, check_task=None, check_items=None, **kwargs):
method load (line 56) | def load(self, replica_number=NaN, timeout=None, check_task=None, chec...
method release (line 67) | def release(self, check_task=None, check_items=None, **kwargs):
method flush (line 78) | def flush(self, check_task=None, check_items=None, **kwargs):
method insert (line 89) | def insert(self, data, check_task=None, check_items=None, **kwargs):
method search (line 100) | def search(self, data, anns_field, params, limit, expr=None, output_fi...
method query (line 114) | def query(self, expr, output_fields=None, timeout=None, check_task=Non...
method delete (line 124) | def delete(self, expr, check_task=None, check_items=None, **kwargs):
method upsert (line 135) | def upsert(self, data, check_task=None, check_items=None, **kwargs):
method get_replicas (line 146) | def get_replicas(self, timeout=None, check_task=None, check_items=None...
FILE: tests/base/schema_wrapper.py
class ApiCollectionSchemaWrapper (line 9) | class ApiCollectionSchemaWrapper:
method init_collection_schema (line 12) | def init_collection_schema(self, fields, description="", check_task=No...
method primary_field (line 22) | def primary_field(self):
method partition_key_field (line 26) | def partition_key_field(self):
method fields (line 30) | def fields(self):
method description (line 34) | def description(self):
method auto_id (line 38) | def auto_id(self):
method enable_dynamic_field (line 42) | def enable_dynamic_field(self):
method to_dict (line 46) | def to_dict(self):
method verify (line 50) | def verify(self):
method add_field (line 53) | def add_field(self, field_name, datatype, check_task=None, check_items...
class ApiFieldSchemaWrapper (line 61) | class ApiFieldSchemaWrapper:
method init_field_schema (line 64) | def init_field_schema(self, name, dtype, description="", check_task=No...
method description (line 74) | def description(self):
method params (line 78) | def params(self):
method dtype (line 82) | def dtype(self):
FILE: tests/base/utility_wrapper.py
class ApiUtilityWrapper (line 16) | class ApiUtilityWrapper:
method do_bulk_insert (line 22) | def do_bulk_insert(self, collection_name, files="", partition_name=Non...
method get_bulk_insert_state (line 37) | def get_bulk_insert_state(self, task_id, timeout=None, using="default"...
method list_bulk_insert_tasks (line 45) | def list_bulk_insert_tasks(self, limit=0, collection_name=None, timeou...
method get_bulk_insert_pending_list (line 53) | def get_bulk_insert_pending_list(self):
method get_bulk_insert_working_list (line 60) | def get_bulk_insert_working_list(self):
method list_all_bulk_insert_tasks (line 67) | def list_all_bulk_insert_tasks(self, limit=0):
method wait_for_bulk_insert_tasks_completed (line 97) | def wait_for_bulk_insert_tasks_completed(self, task_ids, target_state=...
method wait_all_pending_tasks_finished (line 158) | def wait_all_pending_tasks_finished(self):
method wait_index_build_completed (line 191) | def wait_index_build_completed(self, collection_name, timeout=None):
method get_query_segment_info (line 209) | def get_query_segment_info(self, collection_name, timeout=None, using=...
method loading_progress (line 217) | def loading_progress(self, collection_name, partition_names=None,
method load_state (line 226) | def load_state(self, collection_name, partition_names=None, using="def...
method wait_for_loading_complete (line 234) | def wait_for_loading_complete(self, collection_name, partition_names=N...
method index_building_progress (line 246) | def index_building_progress(self, collection_name, index_name="", usin...
method wait_for_index_building_complete (line 255) | def wait_for_index_building_complete(self, collection_name, index_name...
method has_collection (line 267) | def has_collection(self, collection_name, using="default", check_task=...
method has_partition (line 274) | def has_partition(self, collection_name, partition_name, using="default",
method drop_collection (line 283) | def drop_collection(self, collection_name, timeout=None, using="defaul...
method list_collections (line 291) | def list_collections(self, timeout=None, using="default", check_task=N...
method calc_distance (line 300) | def calc_distance(self, vectors_left, vectors_right, params=None, time...
method load_balance (line 311) | def load_balance(self, collection_name, src_node_id, dst_node_ids, sea...
method create_alias (line 322) | def create_alias(self, collection_name, alias, timeout=None, using="de...
method drop_alias (line 330) | def drop_alias(self, alias, timeout=None, using="default", check_task=...
method alter_alias (line 338) | def alter_alias(self, collection_name, alias, timeout=None, using="def...
method list_aliases (line 346) | def list_aliases(self, collection_name, timeout=None, using="default",...
method mkts_from_datetime (line 354) | def mkts_from_datetime(self, d_time=None, milliseconds=0., delta=None):
method mkts_from_hybridts (line 359) | def mkts_from_hybridts(self, hybridts, milliseconds=0., delta=None):
method create_user (line 363) | def create_user(self, user, password, using="default", check_task=None...
method list_usernames (line 369) | def list_usernames(self, using="default", check_task=None, check_items...
method reset_password (line 376) | def reset_password(self, user, old_password, new_password, check_task=...
method update_password (line 382) | def update_password(self, user, old_password, new_password, check_task...
method delete_user (line 388) | def delete_user(self, user, using="default", check_task=None, check_it...
method list_roles (line 395) | def list_roles(self, include_user_info: bool, using="default", check_t...
method list_user (line 401) | def list_user(self, username: str, include_role_info: bool, using="def...
method list_users (line 407) | def list_users(self, include_role_info: bool, using="default", check_t...
method init_role (line 413) | def init_role(self, name, using="default", check_task=None, check_item...
method create_role (line 421) | def create_role(self, check_task=None, check_items=None, **kwargs):
method role_drop (line 428) | def role_drop(self, check_task=None, check_items=None, **kwargs):
method role_is_exist (line 434) | def role_is_exist(self, check_task=None, check_items=None, **kwargs):
method role_add_user (line 440) | def role_add_user(self, username: str, check_task=None, check_items=No...
method role_remove_user (line 446) | def role_remove_user(self, username: str, check_task=None, check_items...
method role_get_users (line 452) | def role_get_users(self, check_task=None, check_items=None, **kwargs):
method role_name (line 459) | def role_name(self):
method role_grant (line 462) | def role_grant(self, object: str, object_name: str, privilege: str, db...
method role_revoke (line 468) | def role_revoke(self, object: str, object_name: str, privilege: str, d...
method role_list_grant (line 474) | def role_list_grant(self, object: str, object_name: str, db_name: str ...
method role_list_grants (line 480) | def role_list_grants(self, db_name: str = "", check_task=None, check_i...
method create_resource_group (line 486) | def create_resource_group(self, name, using="default", timeout=None, c...
method drop_resource_group (line 492) | def drop_resource_group(self, name, using="default", timeout=None, che...
method list_resource_groups (line 498) | def list_resource_groups(self, using="default", timeout=None, check_ta...
method describe_resource_group (line 504) | def describe_resource_group(self, name, using="default", timeout=None,...
method transfer_node (line 510) | def transfer_node(self, source, target, num_node, using="default", tim...
method transfer_replica (line 517) | def transfer_replica(self, source, target, collection_name, num_replic...
method rename_collection (line 525) | def rename_collection(self, old_collection_name, new_collection_name, ...
method flush_all (line 535) | def flush_all(self, using="default", timeout=None, check_task=None, ch...
method get_server_type (line 542) | def get_server_type(self, using="default", check_task=None, check_item...
method list_indexes (line 549) | def list_indexes(self, collection_name, using="default", timeout=None,...
FILE: tests/check/func_check.py
class ResponseChecker (line 14) | class ResponseChecker:
method __init__ (line 15) | def __init__(self, response, func_name, check_task, check_items, is_su...
method run (line 27) | def run(self):
method assert_succ (line 111) | def assert_succ(self, actual, expect):
method assert_exception (line 115) | def assert_exception(self, res, actual=True, error_dict=None):
method check_value_equal (line 133) | def check_value_equal(res, func_name, params):
method check_collection_property (line 165) | def check_collection_property(res, func_name, check_items):
method check_describe_collection_property (line 203) | def check_describe_collection_property(res, func_name, check_items):
method check_partition_property (line 239) | def check_partition_property(partition, func_name, check_items):
method check_rg_property (line 258) | def check_rg_property(rg, func_name, check_items):
method check_search_results (line 281) | def check_search_results(search_res, func_name, check_items):
method check_search_iterator (line 354) | def check_search_iterator(search_res, func_name, check_items):
method check_query_results (line 394) | def check_query_results(query_res, func_name, check_items):
method check_query_iterator (line 429) | def check_query_iterator(query_res, func_name, check_items):
method check_query_empty (line 463) | def check_query_empty(query_res, func_name):
method check_query_not_empty (line 480) | def check_query_not_empty(query_res, func_name):
method check_distance (line 497) | def check_distance(distance_res, func_name, check_items):
method check_delete_compact_plan (line 515) | def check_delete_compact_plan(compaction_plans, func_name, check_items):
method check_merge_compact_plan (line 542) | def check_merge_compact_plan(compaction_plans, func_name, check_items):
method check_role_property (line 568) | def check_role_property(role, func_name, check_items):
method check_permission_deny (line 581) | def check_permission_deny(res, actual=True):
method check_auth_failure (line 591) | def check_auth_failure(res, actual=True):
FILE: tests/check/param_check.py
function ip_check (line 9) | def ip_check(ip):
function number_check (line 20) | def number_check(num):
function exist_check (line 29) | def exist_check(param, _list):
function dict_equal_check (line 38) | def dict_equal_check(dict1, dict2):
function list_de_duplication (line 45) | def list_de_duplication(_list):
function list_equal_check (line 61) | def list_equal_check(param1, param2):
function list_contain_check (line 84) | def list_contain_check(sublist, superlist):
function get_connect_object_name (line 104) | def get_connect_object_name(_list):
function equal_entity (line 132) | def equal_entity(exp, actual):
function entity_in (line 151) | def entity_in(entity, entities, primary_field):
function remove_entity (line 173) | def remove_entity(entity, entities, primary_field):
function equal_entities_list (line 194) | def equal_entities_list(exp, actual, primary_field, with_vec=False):
function output_field_value_check (line 229) | def output_field_value_check(search_res, original):
FILE: tests/common/bulk_insert_data.py
class DataField (line 22) | class DataField:
class DataErrorType (line 43) | class DataErrorType:
function gen_file_prefix (line 55) | def gen_file_prefix(is_row_based=True, auto_id=True, prefix=""):
function entity_suffix (line 68) | def entity_suffix(rows):
function gen_float_vectors (line 78) | def gen_float_vectors(nb, dim):
function gen_str_invalid_vectors (line 84) | def gen_str_invalid_vectors(nb, dim):
function gen_binary_vectors (line 89) | def gen_binary_vectors(nb, dim):
function gen_fp16_vectors (line 96) | def gen_fp16_vectors(num, dim):
function gen_bf16_vectors (line 114) | def gen_bf16_vectors(num, dim):
function gen_row_based_json_file (line 132) | def gen_row_based_json_file(row_file, str_pk, data_fields, float_vect,
function gen_column_base_json_file (line 256) | def gen_column_base_json_file(col_file, str_pk, data_fields, float_vect,
function gen_vectors_in_numpy_file (line 353) | def gen_vectors_in_numpy_file(dir, data_field, float_vector, rows, dim, ...
function gen_string_in_numpy_file (line 381) | def gen_string_in_numpy_file(dir, data_field, rows, start=0, force=False):
function gen_dynamic_field_in_numpy_file (line 396) | def gen_dynamic_field_in_numpy_file(dir, rows, start=0, force=False):
function gen_bool_in_numpy_file (line 410) | def gen_bool_in_numpy_file(dir, data_field, rows, start=0, force=False):
function gen_json_in_numpy_file (line 425) | def gen_json_in_numpy_file(dir, data_field, rows, start=0, force=False):
function gen_int_or_float_in_numpy_file (line 438) | def gen_int_or_float_in_numpy_file(dir, data_field, rows, start=0, force...
function gen_vectors (line 460) | def gen_vectors(float_vector, rows, dim):
function gen_data_by_data_field (line 470) | def gen_data_by_data_field(data_field, rows, start=0, float_vector=True,...
function gen_file_name (line 526) | def gen_file_name(is_row_based, rows, dim, auto_id, str_pk,
function gen_subfolder (line 550) | def gen_subfolder(root, dim, rows, file_num):
function gen_json_files (line 559) | def gen_json_files(is_row_based, rows, dim, auto_id, str_pk,
function gen_dict_data_by_data_field (line 593) | def gen_dict_data_by_data_field(data_fields, rows, start=0, float_vector...
function gen_new_json_files (line 644) | def gen_new_json_files(float_vector, rows, dim, data_fields, file_nums=1...
function gen_npy_files (line 677) | def gen_npy_files(float_vector, rows, dim, data_fields, file_size=None, ...
function gen_dynamic_field_data_in_parquet_file (line 752) | def gen_dynamic_field_data_in_parquet_file(rows, start=0):
function gen_parquet_files (line 759) | def gen_parquet_files(float_vector, rows, dim, data_fields, file_size=No...
function prepare_bulk_insert_json_files (line 820) | def prepare_bulk_insert_json_files(minio_endpoint="", bucket_name="milvu...
function prepare_bulk_insert_new_json_files (line 893) | def prepare_bulk_insert_new_json_files(minio_endpoint="", bucket_name="m...
function prepare_bulk_insert_numpy_files (line 905) | def prepare_bulk_insert_numpy_files(minio_endpoint="", bucket_name="milv...
function prepare_bulk_insert_parquet_files (line 944) | def prepare_bulk_insert_parquet_files(minio_endpoint="", bucket_name="mi...
function gen_csv_file (line 982) | def gen_csv_file(file, float_vector, data_fields, rows, dim, start_uid):
function gen_csv_files (line 1013) | def gen_csv_files(rows, dim, auto_id, float_vector, data_fields, file_nu...
function prepare_bulk_insert_csv_files (line 1028) | def prepare_bulk_insert_csv_files(minio_endpoint="", bucket_name="milvus...
FILE: tests/common/code_mapping.py
class ErrorCode (line 5) | class ErrorCode(Enum):
class ErrorMap (line 14) | class ErrorMap:
method __init__ (line 15) | def __init__(self, err_code, err_msg):
class ConnectionErrorMessage (line 20) | class ConnectionErrorMessage(ExceptionsMessage):
class CollectionErrorMessage (line 25) | class CollectionErrorMessage(ExceptionsMessage):
class PartitionErrorMessage (line 29) | class PartitionErrorMessage(ExceptionsMessage):
class IndexErrorMessage (line 33) | class IndexErrorMessage(ExceptionsMessage):
FILE: tests/common/common_func.py
function to_serializable (line 29) | def to_serializable(val):
function ts_float32 (line 35) | def ts_float32(val):
class ParamInfo (line 40) | class ParamInfo:
method __init__ (line 41) | def __init__(self):
method prepare_param_info (line 52) | def prepare_param_info(self, host, port, handler, replica_num, user, p...
function gen_unique_str (line 67) | def gen_unique_str(str_value=None):
function gen_str_by_length (line 72) | def gen_str_by_length(length=8, letters_only=False):
function gen_digits_by_length (line 78) | def gen_digits_by_length(length=8):
function gen_bool_field (line 82) | def gen_bool_field(name=ct.default_bool_field_name, description=ct.defau...
function gen_string_field (line 88) | def gen_string_field(name=ct.default_string_field_name, description=ct.d...
function gen_json_field (line 96) | def gen_json_field(name=ct.default_json_field_name, description=ct.defau...
function gen_array_field (line 102) | def gen_array_field(name=ct.default_array_field_name, element_type=DataT...
function gen_int8_field (line 111) | def gen_int8_field(name=ct.default_int8_field_name, description=ct.defau...
function gen_int16_field (line 117) | def gen_int16_field(name=ct.default_int16_field_name, description=ct.def...
function gen_int32_field (line 123) | def gen_int32_field(name=ct.default_int32_field_name, description=ct.def...
function gen_int64_field (line 129) | def gen_int64_field(name=ct.default_int64_field_name, description=ct.def...
function gen_float_field (line 135) | def gen_float_field(name=ct.default_float_field_name, is_primary=False, ...
function gen_double_field (line 141) | def gen_double_field(name=ct.default_double_field_name, is_primary=False...
function gen_float_vec_field (line 147) | def gen_float_vec_field(name=ct.default_float_vec_field_name, is_primary...
function gen_binary_vec_field (line 167) | def gen_binary_vec_field(name=ct.default_binary_vec_field_name, is_prima...
function gen_float16_vec_field (line 175) | def gen_float16_vec_field(name=ct.default_float_vec_field_name, is_prima...
function gen_bfloat16_vec_field (line 183) | def gen_bfloat16_vec_field(name=ct.default_float_vec_field_name, is_prim...
function gen_sparse_vec_field (line 191) | def gen_sparse_vec_field(name=ct.default_sparse_vec_field_name, is_prima...
function gen_default_collection_schema (line 198) | def gen_default_collection_schema(description=ct.default_desc, primary_f...
function gen_all_datatype_collection_schema (line 241) | def gen_all_datatype_collection_schema(description=ct.default_desc, prim...
function gen_array_collection_schema (line 263) | def gen_array_collection_schema(description=ct.default_desc, primary_fie...
function gen_bulk_insert_collection_schema (line 291) | def gen_bulk_insert_collection_schema(description=ct.default_desc, prima...
function gen_general_collection_schema (line 314) | def gen_general_collection_schema(description=ct.default_desc, primary_f...
function gen_string_pk_default_collection_schema (line 325) | def gen_string_pk_default_collection_schema(description=ct.default_desc,...
function gen_json_default_collection_schema (line 333) | def gen_json_default_collection_schema(description=ct.default_desc, prim...
function gen_multiple_json_default_collection_schema (line 341) | def gen_multiple_json_default_collection_schema(description=ct.default_d...
function gen_collection_schema_all_datatype (line 350) | def gen_collection_schema_all_datatype(description=ct.default_desc,
function gen_collection_schema (line 378) | def gen_collection_schema(fields, primary_field=None, description=ct.def...
function gen_default_binary_collection_schema (line 384) | def gen_default_binary_collection_schema(description=ct.default_desc, pr...
function gen_default_sparse_schema (line 393) | def gen_default_sparse_schema(description=ct.default_desc, primary_field...
function gen_schema_multi_vector_fields (line 411) | def gen_schema_multi_vector_fields(vec_fields):
function gen_schema_multi_string_fields (line 420) | def gen_schema_multi_string_fields(string_fields):
function gen_vectors (line 429) | def gen_vectors(nb, dim, vector_data_type="FLOAT_VECTOR"):
function gen_string (line 447) | def gen_string(nb):
function gen_binary_vectors (line 452) | def gen_binary_vectors(num, dim):
function gen_default_dataframe_data (line 463) | def gen_default_dataframe_data(nb=ct.default_nb, dim=ct.default_dim, sta...
function gen_general_default_list_data (line 501) | def gen_general_default_list_data(nb=ct.default_nb, dim=ct.default_dim, ...
function gen_default_rows_data (line 538) | def gen_default_rows_data(nb=ct.default_nb, dim=ct.default_dim, start=0,...
function gen_json_data_for_diff_json_types (line 566) | def gen_json_data_for_diff_json_types(nb=ct.default_nb, start=0, json_ty...
function gen_default_data_for_upsert (line 591) | def gen_default_data_for_upsert(nb=ct.default_nb, dim=ct.default_dim, st...
function gen_array_dataframe_data (line 607) | def gen_array_dataframe_data(nb=ct.default_nb, dim=ct.default_dim, start...
function gen_dataframe_multi_vec_fields (line 636) | def gen_dataframe_multi_vec_fields(vec_fields, nb=ct.default_nb):
function gen_dataframe_multi_string_fields (line 662) | def gen_dataframe_multi_string_fields(string_fields, nb=ct.default_nb):
function gen_dataframe_all_data_type (line 685) | def gen_dataframe_all_data_type(nb=ct.default_nb, dim=ct.default_dim, st...
function gen_general_list_all_data_type (line 732) | def gen_general_list_all_data_type(nb=ct.default_nb, dim=ct.default_dim,...
function gen_default_rows_data_all_data_type (line 772) | def gen_default_rows_data_all_data_type(nb=ct.default_nb, dim=ct.default...
function gen_default_binary_dataframe_data (line 810) | def gen_default_binary_dataframe_data(nb=ct.default_nb, dim=ct.default_d...
function gen_default_list_data (line 831) | def gen_default_list_data(nb=ct.default_nb, dim=ct.default_dim, start=0,...
function gen_default_list_sparse_data (line 845) | def gen_default_list_sparse_data(nb=ct.default_nb, dim=ct.default_dim, s...
function gen_default_list_data_for_bulk_insert (line 859) | def gen_default_list_data_for_bulk_insert(nb=ct.default_nb, varchar_len=...
function prepare_bulk_insert_data (line 873) | def prepare_bulk_insert_data(schema=None,
function get_column_data_by_schema (line 904) | def get_column_data_by_schema(nb=ct.default_nb, schema=None, skip_vector...
function get_row_data_by_schema (line 922) | def get_row_data_by_schema(nb=ct.default_nb, schema=None):
function get_fields_map (line 939) | def get_fields_map(schema=None):
function get_int64_field_name (line 949) | def get_int64_field_name(schema=None):
function get_float_field_name (line 959) | def get_float_field_name(schema=None):
function get_float_vec_field_name (line 969) | def get_float_vec_field_name(schema=None):
function get_float_vec_field_name_list (line 979) | def get_float_vec_field_name_list(schema=None):
function get_scalar_field_name_list (line 990) | def get_scalar_field_name_list(schema=None):
function get_binary_vec_field_name (line 1002) | def get_binary_vec_field_name(schema=None):
function get_binary_vec_field_name_list (line 1012) | def get_binary_vec_field_name_list(schema=None):
function get_dim_by_schema (line 1023) | def get_dim_by_schema(schema=None):
function gen_data_by_type (line 1034) | def gen_data_by_type(field, nb=None, start=None):
function gen_json_files_for_bulk_insert (line 1142) | def gen_json_files_for_bulk_insert(data, schema, data_dir):
function gen_npy_files_for_bulk_insert (line 1181) | def gen_npy_files_for_bulk_insert(data, schema, data_dir):
function gen_default_tuple_data (line 1215) | def gen_default_tuple_data(nb=ct.default_nb, dim=ct.default_dim):
function gen_numpy_data (line 1224) | def gen_numpy_data(nb=ct.default_nb, dim=ct.default_dim):
function gen_default_binary_list_data (line 1235) | def gen_default_binary_list_data(nb=ct.default_nb, dim=ct.default_dim):
function gen_simple_index (line 1245) | def gen_simple_index():
function gen_autoindex_params (line 1257) | def gen_autoindex_params():
function gen_invalid_field_types (line 1274) | def gen_invalid_field_types():
function gen_invalid_search_params_type (line 1287) | def gen_invalid_search_params_type():
function gen_search_param (line 1321) | def gen_search_param(index_type, metric_type="L2"):
function gen_autoindex_search_params (line 1363) | def gen_autoindex_search_params():
function gen_invalid_search_param (line 1374) | def gen_invalid_search_param(index_type, metric_type="L2"):
function gen_all_type_fields (line 1403) | def gen_all_type_fields():
function gen_normal_expressions (line 1412) | def gen_normal_expressions():
function gen_json_field_expressions (line 1429) | def gen_json_field_expressions():
function gen_array_field_expressions (line 1444) | def gen_array_field_expressions():
function gen_field_compare_expressions (line 1459) | def gen_field_compare_expressions(fields1=None, fields2=None):
function gen_normal_string_expressions (line 1477) | def gen_normal_string_expressions(fields=None):
function gen_invalid_string_expressions (line 1496) | def gen_invalid_string_expressions():
function gen_invalid_bool_expressions (line 1504) | def gen_invalid_bool_expressions():
function gen_normal_expressions_field (line 1516) | def gen_normal_expressions_field(field):
function gen_integer_overflow_expressions (line 1537) | def gen_integer_overflow_expressions():
function l2 (line 1553) | def l2(x, y):
function ip (line 1557) | def ip(x, y):
function cosine (line 1561) | def cosine(x, y):
function jaccard (line 1565) | def jaccard(x, y):
function hamming (line 1571) | def hamming(x, y):
function tanimoto (line 1577) | def tanimoto(x, y):
function tanimoto_calc (line 1588) | def tanimoto_calc(x, y):
function substructure (line 1594) | def substructure(x, y):
function superstructure (line 1600) | def superstructure(x, y):
function compare_distance_2d_vector (line 1606) | def compare_distance_2d_vector(x, y, distance, metric, sqrt):
function compare_distance_vector_and_vector_list (line 1628) | def compare_distance_vector_and_vector_list(x, y, metric, distance):
function modify_file (line 1653) | def modify_file(file_path_list, is_modify=False, input_content=""):
function index_to_dict (line 1681) | def index_to_dict(index):
function get_index_params_params (line 1690) | def get_index_params_params(index_type):
function get_search_params_params (line 1695) | def get_search_params_params(index_type):
function assert_json_contains (line 1700) | def assert_json_contains(expr, list_data):
function assert_equal_index (line 1729) | def assert_equal_index(index_1, index_2):
function gen_partitions (line 1733) | def gen_partitions(collection_w, partition_num=1):
function insert_data (line 1749) | def insert_data(collection_w, nb=ct.default_nb, is_binary=False, is_all_...
function _check_primary_keys (line 1835) | def _check_primary_keys(primary_keys, nb):
function get_segment_distribution (line 1845) | def get_segment_distribution(res):
function percent_to_int (line 1859) | def percent_to_int(string):
function gen_grant_list (line 1876) | def gen_grant_list(collection_name):
function install_milvus_operator_specific_config (line 1904) | def install_milvus_operator_specific_config(namespace, milvus_mode, rele...
function get_wildcard_output_field_names (line 1962) | def get_wildcard_output_field_names(collection_w, output_fields):
function extract_vector_field_name_list (line 1971) | def extract_vector_field_name_list(collection_w):
function get_activate_func_from_metric_type (line 1991) | def get_activate_func_from_metric_type(metric_type):
function get_hybrid_search_base_results_rrf (line 2002) | def get_hybrid_search_base_results_rrf(search_res_dict_array, round_deci...
function get_hybrid_search_base_results (line 2033) | def get_hybrid_search_base_results(search_res_dict_array, weights, metri...
function gen_bf16_vectors (line 2067) | def gen_bf16_vectors(num, dim):
function gen_fp16_vectors (line 2085) | def gen_fp16_vectors(num, dim):
function gen_sparse_vectors (line 2103) | def gen_sparse_vectors(nb, dim):
function gen_vectors_based_on_vector_type (line 2118) | def gen_vectors_based_on_vector_type(num, dim, vector_data_type):
FILE: tests/common/common_type.py
class CheckTasks (line 277) | class CheckTasks:
class BulkLoadStates (line 301) | class BulkLoadStates:
class CaseLabel (line 308) | class CaseLabel:
FILE: tests/common/cus_resource_opts.py
class CustomResourceOperations (line 15) | class CustomResourceOperations(object):
method __init__ (line 16) | def __init__(self, kind, group=_GROUP, version=_VERSION, namespace=_NA...
method create (line 33) | def create(self, body):
method delete (line 46) | def delete(self, metadata_name, raise_ex=True):
method patch (line 60) | def patch(self, metadata_name, body):
method list_all (line 74) | def list_all(self):
method get (line 87) | def get(self, metadata_name):
method delete_all (line 100) | def delete_all(self):
FILE: tests/common/milvus_sys.py
class MilvusSys (line 11) | class MilvusSys:
method __init__ (line 12) | def __init__(self, alias='default'):
method build_version (line 27) | def build_version(self):
method build_time (line 32) | def build_time(self):
method deploy_mode (line 37) | def deploy_mode(self):
method simd_type (line 42) | def simd_type(self):
method query_nodes (line 52) | def query_nodes(self):
method data_nodes (line 61) | def data_nodes(self):
method index_nodes (line 70) | def index_nodes(self):
method proxy_nodes (line 79) | def proxy_nodes(self):
method nodes (line 88) | def nodes(self):
method get_nodes_by_type (line 94) | def get_nodes_by_type(self, node_type=None):
FILE: tests/common/minio_comm.py
function copy_files_to_bucket (line 7) | def copy_files_to_bucket(client, r_source, target_files, bucket_name, fo...
function copy_files_to_minio (line 31) | def copy_files_to_minio(host, r_source, files, bucket_name, access_key="...
FILE: tests/config/log_config.py
class LogConfig (line 4) | class LogConfig:
method __init__ (line 5) | def __init__(self):
method get_env_variable (line 13) | def get_env_variable(var="CI_LOG_PATH"):
method create_path (line 25) | def create_path(log_path):
method get_default_config (line 31) | def get_default_config(self):
FILE: tests/conftest.py
function pytest_addoption (line 21) | def pytest_addoption(parser):
function host (line 57) | def host(request):
function service (line 62) | def service(request):
function port (line 67) | def port(request):
function user (line 72) | def user(request):
function password (line 77) | def password(request):
function db_name (line 82) | def db_name(request):
function secure (line 87) | def secure(request):
function milvus_ns (line 92) | def milvus_ns(request):
function http_port (line 97) | def http_port(request):
function handler (line 102) | def handler(request):
function tag (line 107) | def tag(request):
function dry_run (line 112) | def dry_run(request):
function connect_name (line 117) | def connect_name(request):
function database_name (line 122) | def database_name(request):
function partition_name (line 127) | def partition_name(request):
function descriptions (line 132) | def descriptions(request):
function collection_name (line 137) | def collection_name(request):
function search_vectors (line 142) | def search_vectors(request):
function index_param (line 147) | def index_param(request):
function data (line 152) | def data(request):
function clean_log (line 157) | def clean_log(request):
function schema (line 162) | def schema(request):
function err_msg (line 167) | def err_msg(request):
function term_expr (line 172) | def term_expr(request):
function check_content (line 177) | def check_content(request):
function field_name (line 184) | def field_name(request):
function minio_host (line 189) | def minio_host(request):
function uri (line 194) | def uri(request):
function token (line 199) | def token(request):
function request_duration (line 204) | def request_duration(request):
function enable_milvus_local_api (line 209) | def enable_milvus_local_api(request):
function initialize_env (line 217) | def initialize_env(request):
function get_invalid_string (line 245) | def get_invalid_string(request):
function get_index_param (line 250) | def get_index_param(request):
function get_invalid_collection_name (line 255) | def get_invalid_collection_name(request):
function get_invalid_field_name (line 260) | def get_invalid_field_name(request):
function get_invalid_index_type (line 265) | def get_invalid_index_type(request):
function get_invalid_index_params (line 273) | def get_invalid_index_params(request):
function get_invalid_partition_name (line 278) | def get_invalid_partition_name(request):
function get_invalid_vector_dict (line 283) | def get_invalid_vector_dict(request):
function pytest_configure (line 287) | def pytest_configure(config):
function pytest_runtest_setup (line 294) | def pytest_runtest_setup(item):
function pytest_runtestloop (line 305) | def pytest_runtestloop(session):
function check_server_connection (line 326) | def check_server_connection(request):
function connect (line 369) | def connect(request):
function dis_connect (line 396) | def dis_connect(request):
function args (line 410) | def args(request):
function milvus (line 423) | def milvus(request):
function collection (line 434) | def collection(request, connect):
function id_collection (line 455) | def id_collection(request, connect):
function binary_collection (line 475) | def binary_collection(request, connect):
function binary_id_collection (line 496) | def binary_id_collection(request, connect):
FILE: tests/customize/milvus_operator.py
class MilvusOperator (line 18) | class MilvusOperator(object):
method __init__ (line 19) | def __init__(self):
method _update_configs (line 25) | def _update_configs(configs, template=None):
method install (line 52) | def install(self, configs, template=None):
method uninstall (line 69) | def uninstall(self, release_name, namespace='default', delete_depends=...
method upgrade (line 95) | def upgrade(self, release_name, configs, namespace='default'):
method wait_for_healthy (line 117) | def wait_for_healthy(self, release_name, namespace='default', timeout=...
method endpoint (line 142) | def endpoint(self, release_name, namespace='default'):
FILE: tests/customize/test_customize_segment_size.py
function _install_milvus (line 18) | def _install_milvus(seg_size):
class TestCustomizeSegmentSize (line 41) | class TestCustomizeSegmentSize:
method teardown_method (line 43) | def teardown_method(self):
method test_customize_segment_size (line 52) | def test_customize_segment_size(self, seg_size, seg_count):
FILE: tests/customize/test_simd_compat.py
function _install_milvus (line 25) | def _install_milvus(simd):
class TestSimdCompatibility (line 47) | class TestSimdCompatibility:
method teardown_method (line 49) | def teardown_method(self):
method test_simd_compat_e2e (line 55) | def test_simd_compat_e2e(self, simd):
FILE: tests/milvus_lite/test_milvus_lite_close.py
class TestMilvusLocalCloseAPI (line 22) | class TestMilvusLocalCloseAPI(TestcaseBase):
method auto_id (line 27) | def auto_id(self, request):
method metric_type (line 31) | def metric_type(self, request):
method scalar_index (line 35) | def scalar_index(self, request):
method test_milvus_local_reconnect (line 39) | def test_milvus_local_reconnect(self, enable_milvus_local_api):
method test_milvus_client_index_with_params (line 55) | def test_milvus_client_index_with_params(self, enable_milvus_local_api...
class TestMilvusLocalCloseInvalid (line 97) | class TestMilvusLocalCloseInvalid(TestcaseBase):
method test_milvus_local_close_multiple (line 100) | def test_milvus_local_close_multiple(self, enable_milvus_local_api):
FILE: tests/milvus_lite/test_milvus_lite_collection.py
class TestMilvusClientCollectionInvalid (line 50) | class TestMilvusClientCollectionInvalid(TestcaseBase):
method auto_id (line 54) | def auto_id(self, request):
method metric_type (line 58) | def metric_type(self, request):
method test_milvus_client_collection_invalid_collection_name (line 69) | def test_milvus_client_collection_invalid_collection_name(self, enable...
method test_milvus_client_collection_name_over_max_length (line 83) | def test_milvus_client_collection_name_over_max_length(self, enable_mi...
method test_milvus_client_collection_name_empty (line 99) | def test_milvus_client_collection_name_empty(self, enable_milvus_local...
method test_milvus_client_collection_invalid_dim (line 114) | def test_milvus_client_collection_invalid_dim(self, enable_milvus_loca...
method test_milvus_client_collection_invalid_primary_field (line 129) | def test_milvus_client_collection_invalid_primary_field(self, enable_m...
method test_milvus_client_collection_string_auto_id (line 143) | def test_milvus_client_collection_string_auto_id(self, enable_milvus_l...
method test_milvus_client_create_same_collection_different_params (line 158) | def test_milvus_client_create_same_collection_different_params(self, e...
method test_milvus_client_collection_invalid_metric_type (line 179) | def test_milvus_client_collection_invalid_metric_type(self, enable_mil...
method test_milvus_client_collection_invalid_schema_field_name (line 194) | def test_milvus_client_collection_invalid_schema_field_name(self, enab...
class TestMilvusClientCollectionValid (line 213) | class TestMilvusClientCollectionValid(TestcaseBase):
method auto_id (line 217) | def auto_id(self, request):
method metric_type (line 221) | def metric_type(self, request):
method id_type (line 225) | def id_type(self, request):
method test_milvus_client_collection_fast_creation_default (line 237) | def test_milvus_client_collection_fast_creation_default(self, enable_m...
method test_milvus_client_collection_fast_creation_all_params (line 265) | def test_milvus_client_collection_fast_creation_all_params(self, enabl...
method test_milvus_client_collection_self_creation_default (line 291) | def test_milvus_client_collection_self_creation_default(self, enable_m...
method test_milvus_client_array_insert_search (line 324) | def test_milvus_client_array_insert_search(self, enable_milvus_local_a...
method test_milvus_client_search_query_string (line 357) | def test_milvus_client_search_query_string(self, enable_milvus_local_a...
method test_milvus_client_search_different_metric_types_not_specifying_in_search_params (line 395) | def test_milvus_client_search_different_metric_types_not_specifying_in...
method test_milvus_client_search_different_metric_types_specifying_in_search_params (line 426) | def test_milvus_client_search_different_metric_types_specifying_in_sea...
method test_milvus_client_delete_with_ids (line 458) | def test_milvus_client_delete_with_ids(self, enable_milvus_local_api):
method test_milvus_client_delete_with_filters (line 499) | def test_milvus_client_delete_with_filters(self, enable_milvus_local_a...
method test_milvus_client_collection_rename_collection (line 540) | def test_milvus_client_collection_rename_collection(self, enable_milvu...
method test_milvus_client_collection_rename_collection_target_db (line 576) | def test_milvus_client_collection_rename_collection_target_db(self, en...
class TestMilvusClientDropCollectionInvalid (line 613) | class TestMilvusClientDropCollectionInvalid(TestcaseBase):
method test_milvus_client_drop_collection_invalid_collection_name (line 624) | def test_milvus_client_drop_collection_invalid_collection_name(self, e...
method test_milvus_client_drop_collection_not_existed (line 637) | def test_milvus_client_drop_collection_not_existed(self, enable_milvus...
class TestMilvusClientReleaseCollectionInvalid (line 648) | class TestMilvusClientReleaseCollectionInvalid(TestcaseBase):
method test_milvus_client_release_collection_invalid_collection_name (line 659) | def test_milvus_client_release_collection_invalid_collection_name(self...
method test_milvus_client_release_collection_not_existed (line 672) | def test_milvus_client_release_collection_not_existed(self, enable_mil...
method test_milvus_client_release_collection_name_over_max_length (line 686) | def test_milvus_client_release_collection_name_over_max_length(self, e...
class TestMilvusClientReleaseCollectionValid (line 702) | class TestMilvusClientReleaseCollectionValid(TestcaseBase):
method auto_id (line 706) | def auto_id(self, request):
method metric_type (line 710) | def metric_type(self, request):
method id_type (line 714) | def id_type(self, request):
method test_milvus_client_release_unloaded_collection (line 724) | def test_milvus_client_release_unloaded_collection(self, enable_milvus...
method test_milvus_client_load_partially_loaded_collection (line 740) | def test_milvus_client_load_partially_loaded_collection(self, enable_m...
class TestMilvusClientLoadCollectionInvalid (line 761) | class TestMilvusClientLoadCollectionInvalid(TestcaseBase):
method test_milvus_client_load_collection_invalid_collection_name (line 772) | def test_milvus_client_load_collection_invalid_collection_name(self, e...
method test_milvus_client_load_collection_not_existed (line 785) | def test_milvus_client_load_collection_not_existed(self, enable_milvus...
method test_milvus_client_load_collection_over_max_length (line 799) | def test_milvus_client_load_collection_over_max_length(self, enable_mi...
method test_milvus_client_load_collection_without_index (line 814) | def test_milvus_client_load_collection_without_index(self, enable_milv...
class TestMilvusClientLoadCollectionValid (line 833) | class TestMilvusClientLoadCollectionValid(TestcaseBase):
method auto_id (line 837) | def auto_id(self, request):
method metric_type (line 841) | def metric_type(self, request):
method id_type (line 845) | def id_type(self, request):
method test_milvus_client_load_loaded_collection (line 855) | def test_milvus_client_load_loaded_collection(self, enable_milvus_loca...
method test_milvus_client_load_partially_loaded_collection (line 870) | def test_milvus_client_load_partially_loaded_collection(self, enable_m...
class TestMilvusClientDescribeCollectionInvalid (line 892) | class TestMilvusClientDescribeCollectionInvalid(TestcaseBase):
method test_milvus_client_describe_collection_invalid_collection_name (line 903) | def test_milvus_client_describe_collection_invalid_collection_name(sel...
method test_milvus_client_describe_collection_not_existed (line 916) | def test_milvus_client_describe_collection_not_existed(self, enable_mi...
method test_milvus_client_describe_collection_deleted_collection (line 930) | def test_milvus_client_describe_collection_deleted_collection(self, en...
class TestMilvusClientHasCollectionInvalid (line 947) | class TestMilvusClientHasCollectionInvalid(TestcaseBase):
method test_milvus_client_has_collection_invalid_collection_name (line 958) | def test_milvus_client_has_collection_invalid_collection_name(self, en...
method test_milvus_client_has_collection_not_existed (line 971) | def test_milvus_client_has_collection_not_existed(self, enable_milvus_...
method test_milvus_client_has_collection_deleted_collection (line 983) | def test_milvus_client_has_collection_deleted_collection(self, enable_...
class TestMilvusClientRenameCollectionInValid (line 998) | class TestMilvusClientRenameCollectionInValid(TestcaseBase):
method test_milvus_client_rename_collection_invalid_collection_name (line 1009) | def test_milvus_client_rename_collection_invalid_collection_name(self,...
method test_milvus_client_rename_collection_not_existed_collection (line 1021) | def test_milvus_client_rename_collection_not_existed_collection(self, ...
method test_milvus_client_rename_collection_duplicated_collection (line 1034) | def test_milvus_client_rename_collection_duplicated_collection(self, e...
method test_milvus_client_rename_deleted_collection (line 1050) | def test_milvus_client_rename_deleted_collection(self, enable_milvus_l...
class TestMilvusClientRenameCollectionValid (line 1067) | class TestMilvusClientRenameCollectionValid(TestcaseBase):
method test_milvus_client_rename_collection_multiple_times (line 1077) | def test_milvus_client_rename_collection_multiple_times(self, enable_m...
method test_milvus_client_rename_collection_deleted_collection (line 1094) | def test_milvus_client_rename_collection_deleted_collection(self, enab...
class TestMilvusClientUsingDatabaseInvalid (line 1112) | class TestMilvusClientUsingDatabaseInvalid(TestcaseBase):
method test_milvus_client_using_database_invalid_db_name (line 1124) | def test_milvus_client_using_database_invalid_db_name(self, enable_mil...
method test_milvus_client_using_database_not_exist_db_name (line 1137) | def test_milvus_client_using_database_not_exist_db_name(self, enable_m...
method test_milvus_client_using_database_db_name_over_max_length (line 1151) | def test_milvus_client_using_database_db_name_over_max_length(self, en...
FILE: tests/milvus_lite/test_milvus_lite_delete.py
class TestMilvusClientDeleteInvalid (line 49) | class TestMilvusClientDeleteInvalid(TestcaseBase):
method auto_id (line 53) | def auto_id(self, request):
method metric_type (line 57) | def metric_type(self, request):
method test_milvus_client_delete_with_filters_and_ids (line 66) | def test_milvus_client_delete_with_filters_and_ids(self, enable_milvus...
method test_milvus_client_delete_with_invalid_id_type (line 92) | def test_milvus_client_delete_with_invalid_id_type(self, enable_milvus...
method test_milvus_client_delete_with_not_all_required_params (line 109) | def test_milvus_client_delete_with_not_all_required_params(self, enabl...
class TestMilvusClientDeleteValid (line 126) | class TestMilvusClientDeleteValid(TestcaseBase):
method auto_id (line 130) | def auto_id(self, request):
method metric_type (line 134) | def metric_type(self, request):
method test_milvus_client_delete_with_ids (line 144) | def test_milvus_client_delete_with_ids(self, enable_milvus_local_api):
method test_milvus_client_delete_with_filters (line 185) | def test_milvus_client_delete_with_filters(self, enable_milvus_local_a...
method test_milvus_client_delete_with_filters_partition (line 226) | def test_milvus_client_delete_with_filters_partition(self, enable_milv...
FILE: tests/milvus_lite/test_milvus_lite_index.py
class TestMilvusClientIndexInvalid (line 50) | class TestMilvusClientIndexInvalid(TestcaseBase):
method auto_id (line 54) | def auto_id(self, request):
method metric_type (line 58) | def metric_type(self, request):
method test_milvus_client_index_invalid_collection_name (line 69) | def test_milvus_client_index_invalid_collection_name(self, name, enabl...
method test_milvus_client_index_collection_name_over_max_length (line 93) | def test_milvus_client_index_collection_name_over_max_length(self, ena...
method test_milvus_client_index_not_exist_collection_name (line 116) | def test_milvus_client_index_not_exist_collection_name(self, enable_mi...
method test_milvus_client_index_invalid_index_type (line 141) | def test_milvus_client_index_invalid_index_type(self, enable_milvus_lo...
method test_milvus_client_index_invalid_metric_type (line 165) | def test_milvus_client_index_invalid_metric_type(self, enable_milvus_l...
method test_milvus_client_index_drop_index_before_release (line 188) | def test_milvus_client_index_drop_index_before_release(self, enable_mi...
method test_milvus_client_index_multiple_indexes_one_field (line 206) | def test_milvus_client_index_multiple_indexes_one_field(self, enable_m...
method test_milvus_client_create_diff_index_without_release (line 232) | def test_milvus_client_create_diff_index_without_release(self, enable_...
class TestMilvusClientIndexValid (line 250) | class TestMilvusClientIndexValid(TestcaseBase):
method auto_id (line 254) | def auto_id(self, request):
method metric_type (line 258) | def metric_type(self, request):
method scalar_index (line 262) | def scalar_index(self, request):
method test_milvus_client_index_default (line 276) | def test_milvus_client_index_default(self, enable_milvus_local_api, in...
method test_milvus_client_index_with_params (line 325) | def test_milvus_client_index_with_params(self, enable_milvus_local_api...
method test_milvus_client_index_after_insert (line 373) | def test_milvus_client_index_after_insert(self, enable_milvus_local_ap...
method test_milvus_client_index_auto_index (line 416) | def test_milvus_client_index_auto_index(self, enable_milvus_local_api,...
method test_milvus_client_index_multiple_vectors (line 468) | def test_milvus_client_index_multiple_vectors(self, enable_milvus_loca...
method test_milvus_client_index_drop_create_same_index (line 519) | def test_milvus_client_index_drop_create_same_index(self, enable_milvu...
method test_milvus_client_index_drop_create_different_index (line 571) | def test_milvus_client_index_drop_create_different_index(self, enable_...
FILE: tests/milvus_lite/test_milvus_lite_init.py
class TestMilvusLocalFilePath (line 20) | class TestMilvusLocalFilePath(TestcaseBase):
method test_milvus_local_file_path (line 24) | def test_milvus_local_file_path(self, file_path):
method test_milvus_local_invalid_file_path (line 48) | def test_milvus_local_invalid_file_path(self, file_path):
FILE: tests/milvus_lite/test_milvus_lite_insert.py
class TestMilvusClientInsertInvalid (line 50) | class TestMilvusClientInsertInvalid(TestcaseBase):
method auto_id (line 54) | def auto_id(self, request):
method metric_type (line 58) | def metric_type(self, request):
method test_milvus_client_insert_column_data (line 68) | def test_milvus_client_insert_column_data(self, enable_milvus_local_api):
method test_milvus_client_insert_empty_collection_name (line 87) | def test_milvus_client_insert_empty_collection_name(self, enable_milvu...
method test_milvus_client_insert_invalid_collection_name (line 104) | def test_milvus_client_insert_invalid_collection_name(self, enable_mil...
method test_milvus_client_insert_collection_name_over_max_length (line 120) | def test_milvus_client_insert_collection_name_over_max_length(self, en...
method test_milvus_client_insert_not_exist_collection_name (line 138) | def test_milvus_client_insert_not_exist_collection_name(self, enable_m...
method test_milvus_client_insert_data_invalid_type (line 156) | def test_milvus_client_insert_data_invalid_type(self, enable_milvus_lo...
method test_milvus_client_insert_data_empty (line 172) | def test_milvus_client_insert_data_empty(self, enable_milvus_local_api):
method test_milvus_client_insert_data_vector_field_missing (line 187) | def test_milvus_client_insert_data_vector_field_missing(self, enable_m...
method test_milvus_client_insert_data_id_field_missing (line 206) | def test_milvus_client_insert_data_id_field_missing(self, enable_milvu...
method test_milvus_client_insert_data_extra_field (line 225) | def test_milvus_client_insert_data_extra_field(self, enable_milvus_loc...
method test_milvus_client_insert_data_dim_not_match (line 245) | def test_milvus_client_insert_data_dim_not_match(self, enable_milvus_l...
method test_milvus_client_insert_not_matched_data (line 265) | def test_milvus_client_insert_not_matched_data(self, enable_milvus_loc...
method test_milvus_client_insert_invalid_partition_name (line 287) | def test_milvus_client_insert_invalid_partition_name(self, enable_milv...
method test_milvus_client_insert_not_exist_partition_name (line 307) | def test_milvus_client_insert_not_exist_partition_name(self, enable_mi...
method test_milvus_client_insert_collection_partition_not_match (line 327) | def test_milvus_client_insert_collection_partition_not_match(self, ena...
class TestMilvusClientInsertValid (line 350) | class TestMilvusClientInsertValid(TestcaseBase):
method auto_id (line 354) | def auto_id(self, request):
method metric_type (line 358) | def metric_type(self, request):
method test_milvus_client_insert_default (line 368) | def test_milvus_client_insert_default(self, enable_milvus_local_api):
method test_milvus_client_insert_different_fields (line 410) | def test_milvus_client_insert_different_fields(self, enable_milvus_loc...
method test_milvus_client_insert_empty_data (line 450) | def test_milvus_client_insert_empty_data(self, enable_milvus_local_api):
method test_milvus_client_insert_partition (line 476) | def test_milvus_client_insert_partition(self, enable_milvus_local_api):
class TestMilvusClientUpsertInvalid (line 520) | class TestMilvusClientUpsertInvalid(TestcaseBase):
method auto_id (line 524) | def auto_id(self, request):
method metric_type (line 528) | def metric_type(self, request):
method test_milvus_client_upsert_column_data (line 538) | def test_milvus_client_upsert_column_data(self, enable_milvus_local_api):
method test_milvus_client_upsert_empty_collection_name (line 557) | def test_milvus_client_upsert_empty_collection_name(self, enable_milvu...
method test_milvus_client_upsert_invalid_collection_name (line 574) | def test_milvus_client_upsert_invalid_collection_name(self, collection...
method test_milvus_client_upsert_collection_name_over_max_length (line 590) | def test_milvus_client_upsert_collection_name_over_max_length(self, en...
method test_milvus_client_upsert_not_exist_collection_name (line 608) | def test_milvus_client_upsert_not_exist_collection_name(self, enable_m...
method test_milvus_client_upsert_data_invalid_type (line 626) | def test_milvus_client_upsert_data_invalid_type(self, data, enable_mil...
method test_milvus_client_upsert_data_empty (line 642) | def test_milvus_client_upsert_data_empty(self, enable_milvus_local_api):
method test_milvus_client_upsert_data_vector_field_missing (line 657) | def test_milvus_client_upsert_data_vector_field_missing(self, enable_m...
method test_milvus_client_upsert_data_id_field_missing (line 676) | def test_milvus_client_upsert_data_id_field_missing(self, enable_milvu...
method test_milvus_client_upsert_data_extra_field (line 695) | def test_milvus_client_upsert_data_extra_field(self, enable_milvus_loc...
method test_milvus_client_upsert_data_dim_not_match (line 715) | def test_milvus_client_upsert_data_dim_not_match(self, enable_milvus_l...
method test_milvus_client_upsert_not_matched_data (line 735) | def test_milvus_client_upsert_not_matched_data(self, enable_milvus_loc...
method test_milvus_client_upsert_invalid_partition_name (line 757) | def test_milvus_client_upsert_invalid_partition_name(self, partition_n...
method test_milvus_client_upsert_not_exist_partition_name (line 777) | def test_milvus_client_upsert_not_exist_partition_name(self, enable_mi...
method test_milvus_client_upsert_collection_partition_not_match (line 797) | def test_milvus_client_upsert_collection_partition_not_match(self, ena...
class TestMilvusClientUpsertValid (line 820) | class TestMilvusClientUpsertValid(TestcaseBase):
method auto_id (line 824) | def auto_id(self, request):
method metric_type (line 828) | def metric_type(self, request):
method test_milvus_client_upsert_default (line 838) | def test_milvus_client_upsert_default(self, enable_milvus_local_api):
method test_milvus_client_upsert_empty_data (line 880) | def test_milvus_client_upsert_empty_data(self, enable_milvus_local_api):
method test_milvus_client_upsert_partition (line 906) | def test_milvus_client_upsert_partition(self, enable_milvus_local_api):
method test_milvus_client_insert_upsert (line 953) | def test_milvus_client_insert_upsert(self, enable_milvus_local_api):
FILE: tests/milvus_lite/test_milvus_lite_migrate.py
class TestMilvusLiteMigrate (line 37) | class TestMilvusLiteMigrate(TestcaseBase):
method test_milvus_lite_migrate_json_file (line 44) | def test_milvus_lite_migrate_json_file(self):
method test_check_json_file_key (line 93) | def test_check_json_file_key(self):
FILE: tests/milvus_lite/test_milvus_lite_query.py
class TestMilvusClientQueryInvalid (line 49) | class TestMilvusClientQueryInvalid(TestcaseBase):
method auto_id (line 53) | def auto_id(self, request):
method metric_type (line 57) | def metric_type(self, request):
method test_milvus_client_query_not_all_required_params (line 67) | def test_milvus_client_query_not_all_required_params(self, enable_milv...
class TestMilvusClientQueryValid (line 95) | class TestMilvusClientQueryValid(TestcaseBase):
method auto_id (line 99) | def auto_id(self, request):
method metric_type (line 103) | def metric_type(self, request):
method test_milvus_client_query_default (line 113) | def test_milvus_client_query_default(self, enable_milvus_local_api):
method test_milvus_client_query_output_fields (line 143) | def test_milvus_client_query_output_fields(self, enable_milvus_local_a...
method test_milvus_client_query_output_fields_all (line 177) | def test_milvus_client_query_output_fields_all(self, enable_milvus_loc...
method test_milvus_client_query_limit (line 210) | def test_milvus_client_query_limit(self, enable_milvus_local_api):
class TestMilvusClientGetInvalid (line 243) | class TestMilvusClientGetInvalid(TestcaseBase):
method test_milvus_client_get_invalid_collection_name (line 255) | def test_milvus_client_get_invalid_collection_name(self, name, enable_...
method test_milvus_client_get_not_exist_collection_name (line 279) | def test_milvus_client_get_not_exist_collection_name(self, enable_milv...
method test_milvus_client_get_invalid_ids (line 305) | def test_milvus_client_get_invalid_ids(self, invalid_ids, enable_milvu...
class TestMilvusClientGetValid (line 328) | class TestMilvusClientGetValid(TestcaseBase):
method auto_id (line 332) | def auto_id(self, request):
method metric_type (line 336) | def metric_type(self, request):
method test_milvus_client_get_normal (line 345) | def test_milvus_client_get_normal(self, enable_milvus_local_api):
method test_milvus_client_get_output_fields (line 370) | def test_milvus_client_get_output_fields(self, enable_milvus_local_api):
method test_milvus_client_get_normal_string (line 400) | def test_milvus_client_get_normal_string(self, enable_milvus_local_api):
method test_milvus_client_get_normal_string_output_fields (line 427) | def test_milvus_client_get_normal_string_output_fields(self, enable_mi...
FILE: tests/milvus_lite/test_milvus_lite_search.py
class TestMilvusClientSearchInvalid (line 62) | class TestMilvusClientSearchInvalid(TestcaseBase):
method auto_id (line 66) | def auto_id(self, request):
method metric_type (line 70) | def metric_type(self, request):
method test_milvus_client_collection_invalid_primary_field (line 80) | def test_milvus_client_collection_invalid_primary_field(self, enable_m...
method test_milvus_client_collection_string_auto_id (line 94) | def test_milvus_client_collection_string_auto_id(self, enable_milvus_l...
method test_milvus_client_create_same_collection_different_params (line 109) | def test_milvus_client_create_same_collection_different_params(self, e...
method test_milvus_client_collection_invalid_metric_type (line 130) | def test_milvus_client_collection_invalid_metric_type(self, enable_mil...
method test_milvus_client_search_not_consistent_metric_type (line 146) | def test_milvus_client_search_not_consistent_metric_type(self, enable_...
class TestMilvusClientSearchValid (line 168) | class TestMilvusClientSearchValid(TestcaseBase):
method auto_id (line 172) | def auto_id(self, request):
method metric_type (line 176) | def metric_type(self, request):
method test_milvus_client_search_query_default (line 186) | def test_milvus_client_search_query_default(self, enable_milvus_local_...
method test_milvus_client_rename_search_query_default (line 230) | def test_milvus_client_rename_search_query_default(self, enable_milvus...
method test_milvus_client_array_insert_search (line 276) | def test_milvus_client_array_insert_search(self, enable_milvus_local_a...
method test_milvus_client_search_query_string (line 309) | def test_milvus_client_search_query_string(self, enable_milvus_local_a...
method test_milvus_client_search_different_metric_types_not_specifying_in_search_params (line 348) | def test_milvus_client_search_different_metric_types_not_specifying_in...
method test_milvus_client_search_different_metric_types_specifying_in_search_params (line 380) | def test_milvus_client_search_different_metric_types_specifying_in_sea...
method test_milvus_client_delete_with_ids (line 413) | def test_milvus_client_delete_with_ids(self, enable_milvus_local_api):
method test_milvus_client_delete_with_filters (line 454) | def test_milvus_client_delete_with_filters(self, enable_milvus_local_a...
class TestCollectionHybridSearchValid (line 495) | class TestCollectionHybridSearchValid(TestcaseBase):
method nq (line 499) | def nq(self, request):
method nb (line 503) | def nb(self, request):
method dim (line 507) | def dim(self, request):
method auto_id (line 511) | def auto_id(self, request):
method _async (line 515) | def _async(self, request):
method metrics (line 519) | def metrics(self, request):
method is_flush (line 523) | def is_flush(self, request):
method enable_dynamic_field (line 527) | def enable_dynamic_field(self, request):
method metric_type (line 531) | def metric_type(self, request):
method random_primary_key (line 535) | def random_primary_key(self, request):
method vector_data_type (line 539) | def vector_data_type(self, request):
method test_hybrid_search_normal_max_nq (line 550) | def test_hybrid_search_normal_max_nq(self, nq, enable_milvus_local_api):
method test_hybrid_search_normal_over_max_nq (line 585) | def test_hybrid_search_normal_over_max_nq(self, nq):
method test_hybrid_search_no_limit (line 618) | def test_hybrid_search_no_limit(self):
method test_hybrid_search_WeightedRanker_empty_reqs (line 645) | def test_hybrid_search_WeightedRanker_empty_reqs(self, primary_field):
method test_hybrid_search_as_search (line 666) | def test_hybrid_search_as_search(self, nq, primary_field, is_flush):
method test_hybrid_search_different_metric_type (line 715) | def test_hybrid_search_different_metric_type(self, nq, primary_field, ...
method test_hybrid_search_different_metric_type_each_field (line 754) | def test_hybrid_search_different_metric_type_each_field(self, nq, prim...
method test_hybrid_search_different_dim (line 811) | def test_hybrid_search_different_dim(self, nq, primary_field, metric_t...
method test_hybrid_search_overall_limit_larger_sum_each_limit (line 859) | def test_hybrid_search_overall_limit_larger_sum_each_limit(self, nq, p...
method test_hybrid_search_overall_different_limit (line 923) | def test_hybrid_search_overall_different_limit(self, primary_field, me...
method test_hybrid_search_min_limit (line 962) | def test_hybrid_search_min_limit(self, primary_field, metric_type):
method test_hybrid_search_max_limit (line 1010) | def test_hybrid_search_max_limit(self, primary_field, metric_type):
method test_hybrid_search_max_min_limit (line 1049) | def test_hybrid_search_max_min_limit(self, primary_field, metric_type):
method test_hybrid_search_same_anns_field (line 1091) | def test_hybrid_search_same_anns_field(self, primary_field, metric_type):
method test_hybrid_search_different_offset_single_field (line 1130) | def test_hybrid_search_different_offset_single_field(self, primary_fie...
method test_hybrid_search_max_reqs_num (line 1169) | def test_hybrid_search_max_reqs_num(self, primary_field):
method test_hybrid_search_WeightedRanker_different_parameters (line 1211) | def test_hybrid_search_WeightedRanker_different_parameters(self, prima...
method test_hybrid_search_invalid_WeightedRanker_params (line 1249) | def test_hybrid_search_invalid_WeightedRanker_params(self):
method test_hybrid_search_with_range_search (line 1325) | def test_hybrid_search_with_range_search(self, primary_field):
method test_hybrid_search_RRFRanker_default_parameter (line 1367) | def test_hybrid_search_RRFRanker_default_parameter(self, primary_field):
method test_hybrid_search_offset_inside_outside_params (line 1494) | def test_hybrid_search_offset_inside_outside_params(self, primary_fiel...
method test_hybrid_search_RRFRanker_empty_reqs (line 1553) | def test_hybrid_search_RRFRanker_empty_reqs(self):
method test_hybrid_search_different_limit_round_decimal (line 1631) | def test_hybrid_search_different_limit_round_decimal(self, primary_fie...
method test_hybrid_search_limit_out_of_range_max (line 1690) | def test_hybrid_search_limit_out_of_range_max(self):
method test_hybrid_search_limit_out_of_range_min (line 1723) | def test_hybrid_search_limit_out_of_range_min(self):
method test_hybrid_search_with_output_fields (line 1756) | def test_hybrid_search_with_output_fields(self, nq, dim, auto_id, is_f...
method test_hybrid_search_with_output_fields_all_fields (line 1837) | def test_hybrid_search_with_output_fields_all_fields(self, nq, dim, au...
method test_hybrid_search_with_output_fields_all_fields (line 1920) | def test_hybrid_search_with_output_fields_all_fields(self, nq, dim, au...
method test_hybrid_search_with_output_fields_sync_async (line 2002) | def test_hybrid_search_with_output_fields_sync_async(self, nq, primary...
method test_hybrid_search_offset_both_inside_outside_params (line 2086) | def test_hybrid_search_offset_both_inside_outside_params(self, rerank):
method test_hybrid_search_result_order (line 2200) | def test_hybrid_search_result_order(self, nq):
method test_hybrid_search_sparse_normal (line 2232) | def test_hybrid_search_sparse_normal(self):
method test_milvus_lite_delete_search (line 2287) | def test_milvus_lite_delete_search(self, enable_milvus_local_api):
FILE: tests/milvus_lite/test_milvus_lite_sparse.py
class TestMilvusLocalSparse (line 25) | class TestMilvusLocalSparse(TestcaseBase):
method test_milvus_local_sparse_search (line 29) | def test_milvus_local_sparse_search(self, index, index_params, enable...
method test_milvus_lcoal_sparse_index_dim (line 47) | def test_milvus_lcoal_sparse_index_dim(self, index, index_params, dim,...
method test_sparse_index_invalid_dim (line 72) | def test_sparse_index_invalid_dim(self, index, index_params, dim, enab...
FILE: tests/milvus_lite/test_milvus_lite_stability.py
class TestMilvusLocalStability (line 29) | class TestMilvusLocalStability(TestcaseBase):
method test_milvus_local_stability (line 32) | def test_milvus_local_stability(self):
method test_milvus_local_hnsw_serial_performance (line 88) | def test_milvus_local_hnsw_serial_performance(self):
FILE: tests/test_bm25.py
class TestBM25 (line 17) | class TestBM25(unittest.TestCase):
method test_single_function (line 18) | def test_single_function(self):
method test_multi_functions (line 91) | def test_multi_functions(self):
method test_bm25_hybrid_search (line 182) | def test_bm25_hybrid_search(self):
FILE: tests/test_delete.py
class TestDefaultSearch (line 17) | class TestDefaultSearch(unittest.TestCase):
method test_delete_by_ids (line 18) | def test_delete_by_ids(self):
method test_delete_by_filter (line 56) | def test_delete_by_filter(self):
FILE: tests/test_dump_tool.py
function gen_binary_vector (line 34) | def gen_binary_vector():
function gen_float_vector (line 39) | def gen_float_vector():
function all_types_data (line 42) | def all_types_data(bin_vec: bool)->list:
class TestDumpTool (line 67) | class TestDumpTool(unittest.TestCase):
method setUp (line 68) | def setUp(self):
method tearDown (line 75) | def tearDown(self):
method test_simple_collection (line 80) | def test_simple_collection(self):
method test_all_type_collection (line 111) | def test_all_type_collection(self):
method test_all_type_bin_vec_collection (line 141) | def test_all_type_bin_vec_collection(self):
FILE: tests/test_hybrid_search.py
class TestHybridSearch (line 28) | class TestHybridSearch(unittest.TestCase):
method test_hybrid_search (line 29) | def test_hybrid_search(self):
method test_dynamic (line 151) | def test_dynamic(self):
method test_dense (line 247) | def test_dense(self):
FILE: tests/test_query.py
class TestQuery (line 17) | class TestQuery(unittest.TestCase):
method setUp (line 18) | def setUp(self):
method tearDown (line 27) | def tearDown(self):
method test_query_empty (line 30) | def test_query_empty(self):
method test_query (line 40) | def test_query(self):
class TestArray (line 69) | class TestArray(unittest.TestCase):
method test_query (line 70) | def test_query(self):
FILE: tests/test_schema.py
class TestDefaultSearch (line 17) | class TestDefaultSearch(unittest.TestCase):
method test_schema_field_limits (line 18) | def test_schema_field_limits(self):
method test_schema_field_out_limits (line 33) | def test_schema_field_out_limits(self):
method test_varchar_field_maxlen (line 49) | def test_varchar_field_maxlen(self):
method test_varchar_field_out_maxlen (line 63) | def test_varchar_field_out_maxlen(self):
method test_collection_name (line 78) | def test_collection_name(self):
FILE: tests/test_search.py
class TestDefaultSearch (line 17) | class TestDefaultSearch(unittest.TestCase):
method setUp (line 18) | def setUp(self):
method tearDown (line 27) | def tearDown(self):
method test_index_info (line 30) | def test_index_info(self):
method test_error_metrics (line 35) | def test_error_metrics(self):
method test_search (line 52) | def test_search(self):
method test_multi_search (line 112) | def test_multi_search(self):
class TestIndexMetric (line 139) | class TestIndexMetric(unittest.TestCase):
method setUp (line 140) | def setUp(self):
method tearDown (line 148) | def tearDown(self):
method test_L2 (line 151) | def test_L2(self):
method test_IP (line 181) | def test_IP(self):
FILE: tests/test_sparse.py
function csr_to_dict (line 19) | def csr_to_dict(csr):
class TestSparse (line 27) | class TestSparse(unittest.TestCase):
method setUp (line 28) | def setUp(self):
method compare (line 49) | def compare(self, crs, sparse_return):
method tearDown (line 54) | def tearDown(self):
method test_query (line 57) | def test_query(self):
method test_search (line 89) | def test_search(self):
FILE: tests/testcases/test_collection.py
class TestCollectionParams (line 60) | class TestCollectionParams(TestcaseBase):
method get_none_removed_invalid_strings (line 64) | def get_none_removed_invalid_strings(self, request):
method get_invalid_type_fields (line 70) | def get_invalid_type_fields(self, request):
method get_unsupported_primary_field (line 76) | def get_unsupported_primary_field(self, request):
method get_invalid_dim (line 82) | def get_invalid_dim(self, request):
method test_collection (line 88) | def test_collection(self, enable_milvus_local_api):
method test_collection_empty_name (line 103) | def test_collection_empty_name(self, enable_milvus_local_api):
method test_collection_illegal_name (line 118) | def test_collection_illegal_name(self, name, enable_milvus_local_api):
method test_collection_naming_rules (line 135) | def test_collection_naming_rules(self, name, enable_milvus_local_api):
method test_collection_invalid_name (line 152) | def test_collection_invalid_name(self, name, enable_milvus_local_api):
method test_collection_dup_name (line 164) | def test_collection_dup_name(self, enable_milvus_local_api):
method test_collection_dup_name_with_desc (line 181) | def test_collection_dup_name_with_desc(self, enable_milvus_local_api):
method test_collection_dup_name_new_schema (line 199) | def test_collection_dup_name_new_schema(self, enable_milvus_local_api):
method test_collection_dup_name_new_primary (line 217) | def test_collection_dup_name_new_primary(self,enable_milvus_local_api):
method test_collection_dup_name_new_dim (line 242) | def test_collection_dup_name_new_dim(self, enable_milvus_local_api):
method test_collection_dup_name_invalid_schema_type (line 263) | def test_collection_dup_name_invalid_schema_type(self, get_none_remove...
method test_collection_dup_name_same_schema (line 280) | def test_collection_dup_name_same_schema(self, enable_milvus_local_api):
method test_collection_none_schema (line 297) | def test_collection_none_schema(self, enable_milvus_local_api):
method test_collection_invalid_type_schema (line 309) | def test_collection_invalid_type_schema(self, get_none_removed_invalid...
method test_collection_invalid_type_fields (line 322) | def test_collection_invalid_type_fields(self, get_invalid_type_fields,...
method test_collection_with_unknown_type (line 335) | def test_collection_with_unknown_type(self, enable_milvus_local_api):
method test_collection_invalid_type_field (line 349) | def test_collection_invalid_type_field(self, name, enable_milvus_local...
method test_collection_invalid_field_name (line 365) | def test_collection_invalid_field_name(self, name, enable_milvus_local...
method test_collection_none_field_name (line 381) | def test_collection_none_field_name(self, enable_milvus_local_api):
method test_collection_invalid_field_type (line 396) | def test_collection_invalid_field_type(self, dtype, enable_milvus_loca...
method test_collection_field_dtype_float_value (line 409) | def test_collection_field_dtype_float_value(self, enable_milvus_local_...
method test_collection_empty_fields (line 424) | def test_collection_empty_fields(self, enable_milvus_local_api):
method test_collection_dup_field (line 437) | def test_collection_dup_field(self, enable_milvus_local_api):
method test_collection_only_vector_field (line 454) | def test_collection_only_vector_field(self, field, enable_milvus_local...
method test_collection_multi_float_vectors (line 465) | def test_collection_multi_float_vectors(self, enable_milvus_local_api):
method test_collection_mix_vectors (line 483) | def test_collection_mix_vectors(self, enable_milvus_local_api):
method test_collection_without_vectors (line 498) | def test_collection_without_vectors(self, enable_milvus_local_api):
method test_collection_without_primary_field (line 511) | def test_collection_without_primary_field(self, enable_milvus_local_api):
method test_collection_is_primary_false (line 526) | def test_collection_is_primary_false(self, enable_milvus_local_api):
method test_collection_invalid_is_primary (line 540) | def test_collection_invalid_is_primary(self, is_primary, enable_milvus...
method test_collection_invalid_primary_field (line 554) | def test_collection_invalid_primary_field(self, primary_field, enable_...
method test_collection_non_string_primary_field (line 568) | def test_collection_non_string_primary_field(self, primary_field, enab...
method test_collection_not_existed_primary_field (line 581) | def test_collection_not_existed_primary_field(self, enable_milvus_loca...
method test_collection_primary_in_schema (line 596) | def test_collection_primary_in_schema(self, enable_milvus_local_api):
method test_collection_primary_in_field (line 609) | def test_collection_primary_in_field(self, enable_milvus_local_api):
method test_collection_unsupported_primary_field (line 622) | def test_collection_unsupported_primary_field(self, get_unsupported_pr...
method test_collection_multi_primary_fields (line 636) | def test_collection_multi_primary_fields(self, enable_milvus_local_api):
method test_collection_primary_inconsistent (line 651) | def test_collection_primary_inconsistent(self, enable_milvus_local_api):
method test_collection_primary_consistent (line 666) | def test_collection_primary_consistent(self, enable_milvus_local_api):
method test_collection_auto_id_in_field_schema (line 682) | def test_collection_auto_id_in_field_schema(self, auto_id, enable_milv...
method test_collection_auto_id_in_collection_schema (line 699) | def test_collection_auto_id_in_collection_schema(self, auto_id, enable...
method test_collection_auto_id_non_primary_field (line 715) | def test_collection_auto_id_non_primary_field(self, enable_milvus_loca...
method test_collection_auto_id_false_non_primary (line 727) | def test_collection_auto_id_false_non_primary(self, enable_milvus_loca...
method test_collection_auto_id_inconsistent (line 741) | def test_collection_auto_id_inconsistent(self, enable_milvus_local_api):
method test_collection_auto_id_consistent (line 756) | def test_collection_auto_id_consistent(self, auto_id, enable_milvus_lo...
method test_collection_auto_id_none_in_field (line 769) | def test_collection_auto_id_none_in_field(self, enable_milvus_local_api):
method test_collection_invalid_auto_id (line 783) | def test_collection_invalid_auto_id(self, auto_id, enable_milvus_local...
method test_collection_multi_fields_auto_id (line 797) | def test_collection_multi_fields_auto_id(self, enable_milvus_local_api):
method test_collection_vector_without_dim (line 812) | def test_collection_vector_without_dim(self, dtype, enable_milvus_loca...
method test_collection_vector_invalid_dim (line 827) | def test_collection_vector_invalid_dim(self, get_invalid_dim, enable_m...
method test_collection_vector_out_bounds_dim (line 843) | def test_collection_vector_out_bounds_dim(self, dim, enable_milvus_loc...
method test_collection_non_vector_field_dim (line 857) | def test_collection_non_vector_field_dim(self, enable_milvus_local_api):
method test_collection_desc (line 874) | def test_collection_desc(self, enable_milvus_local_api):
method test_collection_none_desc (line 887) | def test_collection_none_desc(self, enable_milvus_local_api):
method test_collection_long_desc (line 900) | def test_collection_long_desc(self, enable_milvus_local_api):
method test_collection_binary (line 915) | def test_collection_binary(self, enable_milvus_local_api):
method test_collection_shards_num_with_default_value (line 929) | def test_collection_shards_num_with_default_value(self, enable_milvus_...
method test_collection_shards_num_with_not_default_value (line 944) | def test_collection_shards_num_with_not_default_value(self, shards_num...
method test_collection_shards_num_invalid (line 959) | def test_collection_shards_num_invalid(self, shards_num, enable_milvus...
method test_collection_shards_num_with_error_type (line 974) | def test_collection_shards_num_with_error_type(self, error_type_shards...
method test_create_collection_maximum_fields (line 988) | def test_create_collection_maximum_fields(self, enable_milvus_local_api):
method test_create_collection_maximum_vector_fields (line 1009) | def test_create_collection_maximum_vector_fields(self, enable_milvus_l...
method test_create_collection_multiple_vector_and_maximum_fields (line 1030) | def test_create_collection_multiple_vector_and_maximum_fields(self, pr...
method test_create_collection_maximum_vector_and_all_fields (line 1059) | def test_create_collection_maximum_vector_and_all_fields(self, primary...
method test_create_collection_over_maximum_fields (line 1087) | def test_create_collection_over_maximum_fields(self, enable_milvus_loc...
method test_create_collection_over_maximum_vector_fields (line 1109) | def test_create_collection_over_maximum_vector_fields(self, enable_mil...
method test_create_collection_multiple_vector_and_over_maximum_all_fields (line 1129) | def test_create_collection_multiple_vector_and_over_maximum_all_fields...
method test_create_collection_over_maximum_vector_and_all_fields (line 1158) | def test_create_collection_over_maximum_vector_and_all_fields(self, en...
method test_collection_multi_sparse_vectors (line 1186) | def test_collection_multi_sparse_vectors(self, enable_milvus_local_api):
class TestCollectionOperation (line 1204) | class TestCollectionOperation(TestcaseBase):
method test_collection_without_connection (line 1212) | def test_collection_without_connection(self, enable_milvus_local_api):
method test_collection_multi_create_drop (line 1229) | def test_collection_multi_create_drop(self, enable_milvus_local_api):
method test_collection_dup_name_drop (line 1247) | def test_collection_dup_name_drop(self, enable_milvus_local_api):
method test_collection_after_drop (line 1266) | def test_collection_after_drop(self, enable_milvus_local_api):
method test_collection_all_datatype_fields (line 1283) | def test_collection_all_datatype_fields(self, enable_milvus_local_api):
method test_load_collection_after_load_partition (line 1308) | def test_load_collection_after_load_partition(self, enable_milvus_loca...
method test_load_collection_release_partition (line 1324) | def test_load_collection_release_partition(self, enable_milvus_local_a...
method test_load_collection_after_release_collection (line 1339) | def test_load_collection_after_release_collection(self, enable_milvus_...
class TestCollectionDataframe (line 1356) | class TestCollectionDataframe(TestcaseBase):
method get_non_df (line 1364) | def get_non_df(self, request):
method test_construct_from_dataframe (line 1370) | def test_construct_from_dataframe(self, enable_milvus_local_api):
method test_construct_from_binary_dataframe (line 1386) | def test_construct_from_binary_dataframe(self, enable_milvus_local_api):
method test_construct_from_none_dataframe (line 1401) | def test_construct_from_none_dataframe(self, enable_milvus_local_api):
method test_construct_from_dataframe_only_column (line 1413) | def test_construct_from_dataframe_only_column(self, enable_milvus_loca...
method test_construct_from_inconsistent_dataframe (line 1427) | def test_construct_from_inconsistent_dataframe(self, enable_milvus_loc...
method test_construct_from_non_dataframe (line 1444) | def test_construct_from_non_dataframe(self, get_non_df, enable_milvus_...
method test_construct_from_data_type_dataframe (line 1457) | def test_construct_from_data_type_dataframe(self, enable_milvus_local_...
method test_construct_from_invalid_field_name (line 1471) | def test_construct_from_invalid_field_name(self, enable_milvus_local_a...
method test_construct_none_primary_field (line 1485) | def test_construct_none_primary_field(self, enable_milvus_local_api):
method test_construct_not_existed_primary_field (line 1499) | def test_construct_not_existed_primary_field(self, enable_milvus_local...
method test_construct_with_none_auto_id (line 1513) | def test_construct_with_none_auto_id(self, enable_milvus_local_api):
method test_construct_auto_id_true_insert (line 1527) | def test_construct_auto_id_true_insert(self, enable_milvus_local_api):
method test_construct_auto_id_true_no_insert (line 1541) | def test_construct_auto_id_true_no_insert(self, enable_milvus_local_api):
method test_construct_none_value_auto_id_true (line 1557) | def test_construct_none_value_auto_id_true(self, enable_milvus_local_a...
method test_construct_auto_id_false (line 1574) | def test_construct_auto_id_false(self, enable_milvus_local_api):
method test_construct_none_value_auto_id_false (line 1589) | def test_construct_none_value_auto_id_false(self, enable_milvus_local_...
method test_construct_auto_id_false_same_values (line 1605) | def test_construct_auto_id_false_same_values(self, enable_milvus_local...
method test_construct_auto_id_false_negative_values (line 1624) | def test_construct_auto_id_false_negative_values(self, enable_milvus_l...
method test_construct_from_dataframe_dup_name (line 1640) | def test_construct_from_dataframe_dup_name(self, enable_milvus_local_a...
class TestCollectionCount (line 1660) | class TestCollectionCount(TestcaseBase):
method test_collection_count_no_vectors (line 1662) | def test_collection_count_no_vectors(self, enable_milvus_local_api):
class TestCollectionCountIP (line 1674) | class TestCollectionCountIP(TestcaseBase):
method insert_count (line 1686) | def insert_count(self, request):
method test_collection_count_after_index_created (line 1690) | def test_collection_count_after_index_created(self, insert_count, enab...
class TestCollectionCountBinary (line 1707) | class TestCollectionCountBinary(TestcaseBase):
method insert_count (line 1720) | def insert_count(self, request):
method test_collection_count_after_index_created_binary (line 1725) | def test_collection_count_after_index_created_binary(self, insert_coun...
method test_binary_collection_with_min_dim (line 1742) | def test_binary_collection_with_min_dim(self, auto_id, enable_milvus_l...
method test_collection_count_no_entities (line 1757) | def test_collection_count_no_entities(self, enable_milvus_local_api):
class TestCollectionMultiCollections (line 1770) | class TestCollectionMultiCollections(TestcaseBase):
method insert_count (line 1783) | def insert_count(self, request):
method test_collection_count_multi_collections_l2 (line 1787) | def test_collection_count_multi_collections_l2(self, insert_count, ena...
method test_collection_count_multi_collections_binary (line 1808) | def test_collection_count_multi_collections_binary(self, insert_count,...
method test_collection_count_multi_collections_mix (line 1829) | def test_collection_count_multi_collections_mix(self, enable_milvus_lo...
class TestCreateCollection (line 1856) | class TestCreateCollection(TestcaseBase):
method auto_id (line 1859) | def auto_id(self, request):
method test_create_collection_multithread (line 1863) | def test_create_collection_multithread(self, enable_milvus_local_api):
method test_create_collection_using_default_value (line 1892) | def test_create_collection_using_default_value(self, auto_id):
class TestCreateCollectionInvalid (line 1916) | class TestCreateCollectionInvalid(TestcaseBase):
method test_create_collection_limit_fields (line 1923) | def test_create_collection_limit_fields(self, enable_milvus_local_api):
method test_create_schema_with_different_language (line 1948) | def test_create_schema_with_different_language(self, invalid_name):
method test_create_collection_with_different_language (line 1965) | def test_create_collection_with_different_language(self, invalid_name):
method test_create_collection_with_invalid_default_value_string (line 1980) | def test_create_collection_with_invalid_default_value_string(self, def...
method test_create_collection_with_invalid_default_value_float (line 1999) | def test_create_collection_with_invalid_default_value_float(self, defa...
method test_create_collection_with_invalid_default_value_int8 (line 2018) | def test_create_collection_with_invalid_default_value_int8(self, defau...
method test_create_collection_with_pk_field_using_default_value (line 2036) | def test_create_collection_with_pk_field_using_default_value(self, ena...
method test_create_collection_with_json_field_using_default_value (line 2067) | def test_create_collection_with_json_field_using_default_value(self, e...
class TestDropCollection (line 2081) | class TestDropCollection(TestcaseBase):
method test_drop_collection_A (line 2089) | def test_drop_collection_A(self, enable_milvus_local_api):
method test_drop_collection_without_connection (line 2103) | def test_drop_collection_without_connection(self, enable_milvus_local_...
method test_drop_collection_not_existed (line 2119) | def test_drop_collection_not_existed(self, enable_milvus_local_api):
method test_create_drop_collection_multithread (line 2136) | def test_create_drop_collection_multithread(self, enable_milvus_local_...
class TestDropCollectionInvalid (line 2165) | class TestDropCollectionInvalid(TestcaseBase):
method test_drop_collection_with_invalid_collection_name (line 2172) | def test_drop_collection_with_invalid_collection_name(self, name, enab...
method test_drop_collection_with_empty_or_None_collection_name (line 2183) | def test_drop_collection_with_empty_or_None_collection_name(self, enab...
class TestHasCollection (line 2196) | class TestHasCollection(TestcaseBase):
method test_has_collection_without_connection (line 2204) | def test_has_collection_without_connection(self, enable_milvus_local_a...
method test_has_collection_not_existed (line 2220) | def test_has_collection_not_existed(self, enable_milvus_local_api):
method test_has_collection_multithread (line 2234) | def test_has_collection_multithread(self, enable_milvus_local_api):
class TestHasCollectionInvalid (line 2259) | class TestHasCollectionInvalid(TestcaseBase):
method test_has_collection_with_invalid_collection_name (line 2262) | def test_has_collection_with_invalid_collection_name(self, name, enabl...
method test_has_collection_with_empty_collection_name (line 2273) | def test_has_collection_with_empty_collection_name(self, enable_milvus...
method test_has_collection_with_none_collection_name (line 2284) | def test_has_collection_with_none_collection_name(self, enable_milvus_...
class TestListCollections (line 2295) | class TestListCollections(TestcaseBase):
method test_list_collections_multi_collections (line 2303) | def test_list_collections_multi_collections(self, enable_milvus_local_...
method test_list_collections_without_connection (line 2321) | def test_list_collections_without_connection(self, enable_milvus_local...
method test_list_collections_multithread (line 2335) | def test_list_collections_multithread(self, enable_milvus_local_api):
class TestLoadCollection (line 2359) | class TestLoadCollection(TestcaseBase):
method test_load_collection_after_index (line 2367) | def test_load_collection_after_index(self, enable_milvus_local_api):
method test_load_collection_after_index_binary (line 2383) | def test_load_collection_after_index_binary(self, enable_milvus_local_...
method test_load_empty_collection (line 2399) | def test_load_empty_collection(self, enable_milvus_local_api):
method test_load_collection_dis_connect (line 2412) | def test_load_collection_dis_connect(self, enable_milvus_local_api):
method test_release_collection_dis_connect (line 2428) | def test_release_collection_dis_connect(self, enable_milvus_local_api):
method test_load_collection_not_existed (line 2444) | def test_load_collection_not_existed(self, enable_milvus_local_api):
method test_release_collection_not_existed (line 2460) | def test_release_collection_not_existed(self, enable_milvus_local_api):
method test_release_collection_not_load (line 2475) | def test_release_collection_not_load(self, enable_milvus_local_api):
method test_load_collection_after_load_release (line 2487) | def test_load_collection_after_load_release(self, enable_milvus_local_...
method test_load_collection_repeatedly (line 2505) | def test_load_collection_repeatedly(self, enable_milvus_local_api):
method test_load_partitions_after_load_collection (line 2521) | def test_load_partitions_after_load_collection(self, enable_milvus_loc...
method test_load_partitions_after_load_release_collection (line 2539) | def test_load_partitions_after_load_release_collection(self, enable_mi...
method test_load_collection_after_release_collection_partition (line 2559) | def test_load_collection_after_release_collection_partition(self, enab...
method test_load_partitions_after_release_collection_partition (line 2582) | def test_load_partitions_after_release_collection_partition(self, enab...
method test_load_collection_after_release_partition (line 2611) | def test_load_collection_after_release_partition(self, enable_milvus_l...
method test_load_partitions_after_release_partition (line 2637) | def test_load_partitions_after_release_partition(self, enable_milvus_l...
method test_load_collection_after_release_partition_collection (line 2661) | def test_load_collection_after_release_partition_collection(self, enab...
method test_load_partitions_after_release_partition_collection (line 2683) | def test_load_partitions_after_release_partition_collection(self, enab...
method test_load_collection_after_release_partitions (line 2708) | def test_load_collection_after_release_partitions(self, enable_milvus_...
method test_load_partitions_after_release_partitions (line 2726) | def test_load_partitions_after_release_partitions(self, enable_milvus_...
method test_load_collection_after_drop_partition_and_release_another (line 2745) | def test_load_collection_after_drop_partition_and_release_another(self...
method test_load_partition_after_drop_partition_and_release_another (line 2769) | def test_load_partition_after_drop_partition_and_release_another(self,...
method test_load_another_partition_after_drop_one_partition (line 2791) | def test_load_another_partition_after_drop_one_partition(self, enable_...
method test_load_collection_after_drop_one_partition (line 2811) | def test_load_collection_after_drop_one_partition(self, enable_milvus_...
method test_load_release_collection (line 2830) | def test_load_release_collection(self, enable_milvus_local_api):
method test_release_collection_after_drop (line 2849) | def test_release_collection_after_drop(self, enable_milvus_local_api):
method test_load_partition_names_empty (line 2866) | def test_load_partition_names_empty(self, enable_milvus_local_api):
method get_non_number_replicas (line 2893) | def get_non_number_replicas(self, request):
method test_load_replica_non_number (line 2901) | def test_load_replica_non_number(self, get_non_number_replicas):
method test_load_replica_invalid_number (line 2921) | def test_load_replica_invalid_number(self, replicas):
method test_load_replica_number_none (line 2942) | def test_load_replica_number_none(self, replicas):
method test_load_replica_greater_than_querynodes (line 2961) | def test_load_replica_greater_than_querynodes(self, enable_milvus_loca...
method test_load_replica_change (line 2979) | def test_load_replica_change(self, enable_milvus_local_api):
method test_load_replica_multi (line 3030) | def test_load_replica_multi(self, enable_milvus_local_api):
method test_load_replica_partitions (line 3062) | def test_load_replica_partitions(self, enable_milvus_local_api):
method test_load_replica_non_shard_leader (line 3098) | def test_load_replica_non_shard_leader(self, enable_milvus_local_api):
method test_load_replica_multiple_shard_leader (line 3148) | def test_load_replica_multiple_shard_leader(self, enable_milvus_local_...
method test_load_replica_sq_count_balance (line 3198) | def test_load_replica_sq_count_balance(self, enable_milvus_local_api):
method test_get_collection_replicas_not_loaded (line 3231) | def test_get_collection_replicas_not_loaded(self, enable_milvus_local_...
method test_count_multi_replicas (line 3247) | def test_count_multi_replicas(self, enable_milvus_local_api):
method test_load_collection_without_creating_index (line 3270) | def test_load_collection_without_creating_index(self, enable_milvus_lo...
class TestDescribeCollection (line 3283) | class TestDescribeCollection(TestcaseBase):
method test_collection_describe (line 3292) | def test_collection_describe(self, enable_milvus_local_api):
class TestReleaseAdvanced (line 3318) | class TestReleaseAdvanced(TestcaseBase):
method test_release_collection_during_searching (line 3320) | def test_release_collection_during_searching(self, enable_milvus_local...
method test_release_partition_during_searching (line 3343) | def test_release_partition_during_searching(self, enable_milvus_local_...
method test_release_indexed_collection_during_searching (line 3370) | def test_release_indexed_collection_during_searching(self, enable_milv...
class TestCollectionString (line 3961) | class TestCollectionString(TestcaseBase):
method test_collection_string_field_is_primary (line 3969) | def test_collection_string_field_is_primary(self, enable_milvus_local_...
method test_collection_with_muti_string_fields (line 3984) | def test_collection_with_muti_string_fields(self, enable_milvus_local_...
method test_collection_only_string_field (line 4003) | def test_collection_only_string_field(self, enable_milvus_local_api):
method test_collection_string_field_with_exceed_max_len (line 4017) | def test_collection_string_field_with_exceed_max_len(self, enable_milv...
method test_collection_invalid_string_field_dtype (line 4037) | def test_collection_invalid_string_field_dtype(self, enable_milvus_loc...
method test_collection_string_field_is_primary_and_auto_id (line 4053) | def test_collection_string_field_is_primary_and_auto_id(self, enable_m...
class TestCollectionJSON (line 4069) | class TestCollectionJSON(TestcaseBase):
method test_collection_json_field_as_primary_key (line 4077) | def test_collection_json_field_as_primary_key(self, auto_id, enable_mi...
method test_collection_json_field_partition_key (line 4103) | def test_collection_json_field_partition_key(self, primary_field, enab...
method test_collection_json_field_supported_primary_key (line 4118) | def test_collection_json_field_supported_primary_key(self, primary_fie...
method test_collection_multiple_json_fields_supported_primary_key (line 4134) | def test_collection_multiple_json_fields_supported_primary_key(self, p...
class TestCollectionARRAY (line 4149) | class TestCollectionARRAY(TestcaseBase):
method test_collection_array_field_element_type_not_exist (line 4157) | def test_collection_array_field_element_type_not_exist(self, enable_mi...
method test_collection_array_field_element_type_invalid (line 4174) | def test_collection_array_field_element_type_invalid(self, element_type):
method test_collection_array_field_no_capacity (line 4193) | def test_collection_array_field_no_capacity(self, enable_milvus_local_...
method test_collection_array_field_invalid_capacity (line 4210) | def test_collection_array_field_invalid_capacity(self, max_capacity):
method test_collection_string_array_without_max_length (line 4228) | def test_collection_string_array_without_max_length(self, enable_milvu...
method test_collection_string_array_max_length_invalid (line 4246) | def test_collection_string_array_max_length_invalid(self, max_length):
method test_collection_array_field_all_datatype (line 4264) | def test_collection_array_field_all_datatype(self, enable_milvus_local...
class TestCollectionMultipleVectorValid (line 4338) | class TestCollectionMultipleVectorValid(TestcaseBase):
method test_create_collection_multiple_vectors_all_supported_field_type (line 4349) | def test_create_collection_multiple_vectors_all_supported_field_type(s...
method test_create_collection_multiple_vectors_different_dim (line 4386) | def test_create_collection_multiple_vectors_different_dim(self, primar...
method test_create_collection_multiple_vectors_maximum_dim (line 4403) | def test_create_collection_multiple_vectors_maximum_dim(self, primary_...
method test_create_collection_multiple_vectors_partition_key (line 4421) | def test_create_collection_multiple_vectors_partition_key(self, primar...
class TestCollectionMultipleVectorInvalid (line 4458) | class TestCollectionMultipleVectorInvalid(TestcaseBase):
method get_invalid_dim (line 4462) | def get_invalid_dim(self, request):
method test_create_collection_multiple_vectors_same_vector_field_name (line 4475) | def test_create_collection_multiple_vectors_same_vector_field_name(sel...
method test_create_collection_multiple_vectors_invalid_part_vector_field_name (line 4508) | def test_create_collection_multiple_vectors_invalid_part_vector_field_...
method test_create_collection_multiple_vectors_invalid_all_vector_field_name (line 4531) | def test_create_collection_multiple_vectors_invalid_all_vector_field_n...
method test_create_collection_multiple_vectors_invalid_dim (line 4553) | def test_create_collection_multiple_vectors_invalid_dim(self, get_inva...
FILE: tests/utils/api_request.py
class Error (line 10) | class Error:
method __init__ (line 11) | def __init__(self, error):
method __str__ (line 15) | def __str__(self):
method __repr__ (line 18) | def __repr__(self):
function api_request_catch (line 25) | def api_request_catch():
function api_request (line 53) | def api_request(_list, **kwargs):
FILE: tests/utils/thread_util.py
class ConcurrencyObj (line 6) | class ConcurrencyObj:
method __init__ (line 7) | def __init__(self, func_obj: Callable, interval=0, stop_flag=False):
method set_stop_flag (line 12) | def set_stop_flag(self, flag=True):
method thread_func (line 15) | def thread_func(self, *args, **kwargs):
method multi_func (line 21) | def multi_func(self, num, *args, **kwargs):
FILE: tests/utils/util_common.py
function gen_experiment_config (line 8) | def gen_experiment_config(yaml):
function findkeys (line 16) | def findkeys(node, kv):
function update_key_value (line 30) | def update_key_value(node, modify_k, modify_v):
function update_key_name (line 43) | def update_key_name(node, modify_k, modify_k_new):
function get_collections (line 58) | def get_collections(file_name="all_collections.json"):
function get_deploy_test_collections (line 69) | def get_deploy_test_collections():
function get_chaos_test_collections (line 80) | def get_chaos_test_collections():
function wait_signal_to_apply_chaos (line 91) | def wait_signal_to_apply_chaos():
FILE: tests/utils/util_k8s.py
function init_k8s_client_config (line 14) | def init_k8s_client_config():
function get_current_namespace (line 29) | def get_current_namespace():
function wait_pods_ready (line 35) | def wait_pods_ready(namespace, label_selector, expected_num=None, timeou...
function get_pod_list (line 88) | def get_pod_list(namespace, label_selector):
function get_pod_ip_name_pairs (line 111) | def get_pod_ip_name_pairs(namespace, label_selector):
function get_querynode_id_pod_pairs (line 133) | def get_querynode_id_pod_pairs(namespace, label_selector):
function get_milvus_instance_name (line 163) | def get_milvus_instance_name(namespace, host="127.0.0.1", port="19530", ...
function get_milvus_deploy_tool (line 201) | def get_milvus_deploy_tool(namespace, milvus_sys):
function export_pod_logs (line 231) | def export_pod_logs(namespace, label_selector, release_name=None):
function read_pod_log (line 268) | def read_pod_log(namespace, label_selector, release_name):
function get_metrics_querynode_sq_req_count (line 292) | def get_metrics_querynode_sq_req_count():
function get_svc_ip (line 318) | def get_svc_ip(namespace, label_selector):
function parse_etcdctl_table_output (line 331) | def parse_etcdctl_table_output(output):
function get_etcd_leader (line 344) | def get_etcd_leader(release_name, deploy_tool="helm"):
function get_etcd_followers (line 370) | def get_etcd_followers(release_name, deploy_tool="helm"):
function find_activate_standby_coord_pod (line 396) | def find_activate_standby_coord_pod(namespace, release_name, coord_type):
function record_time_when_standby_activated (line 424) | def record_time_when_standby_activated(namespace, release_name, coord_ty...
FILE: tests/utils/util_log.py
class TestLog (line 7) | class TestLog:
method __init__ (line 8) | def __init__(self, logger, log_debug, log_file, log_err, log_worker):
FILE: tests/utils/util_pymilvus.py
function create_target_index (line 57) | def create_target_index(index, field_name):
function binary_support (line 61) | def binary_support():
function binary_metrics (line 65) | def binary_metrics():
function structure_metrics (line 69) | def structure_metrics():
function l2 (line 73) | def l2(x, y):
function ip (line 77) | def ip(x, y):
function jaccard (line 81) | def jaccard(x, y):
function hamming (line 87) | def hamming(x, y):
function tanimoto (line 93) | def tanimoto(x, y):
function substructure (line 99) | def substructure(x, y):
function superstructure (line 105) | def superstructure(x, y):
function get_milvus (line 111) | def get_milvus(host, port, uri=None, handler=None, **kwargs):
function reset_build_index_threshold (line 122) | def reset_build_index_threshold(connect):
function disable_flush (line 126) | def disable_flush(connect):
function enable_flush (line 130) | def enable_flush(connect):
function gen_inaccuracy (line 137) | def gen_inaccuracy(num):
function gen_vectors (line 141) | def gen_vectors(num, dim, is_normal=True):
function gen_binary_vectors (line 153) | def gen_binary_vectors(num, dim):
function gen_binary_sub_vectors (line 163) | def gen_binary_sub_vectors(vectors, length):
function gen_binary_super_vectors (line 178) | def gen_binary_super_vectors(vectors, length):
function gen_int_attr (line 191) | def gen_int_attr(row_num):
function gen_float_attr (line 195) | def gen_float_attr(row_num):
function gen_unique_str (line 199) | def gen_unique_str(str_value=None):
function gen_primary_field (line 204) | def gen_primary_field():
function gen_single_filter_fields (line 208) | def gen_single_filter_fields():
function gen_single_vector_fields (line 216) | def gen_single_vector_fields():
function gen_default_fields (line 224) | def gen_default_fields(auto_id=True):
function gen_binary_default_fields (line 236) | def gen_binary_default_fields(auto_id=True):
function gen_entities (line 249) | def gen_entities(nb, start=0, is_normal=False):
function gen_entities_new (line 259) | def gen_entities_new(nb, is_normal=False):
function gen_entities_rows (line 269) | def gen_entities_rows(nb, is_normal=False, _id=True):
function gen_binary_entities (line 292) | def gen_binary_entities(nb):
function gen_binary_entities_new (line 302) | def gen_binary_entities_new(nb):
function gen_binary_entities_rows (line 312) | def gen_binary_entities_rows(nb, _id=True):
function gen_entities_by_fields (line 335) | def gen_entities_by_fields(fields, nb, dim, ids=None):
function assert_equal_entity (line 353) | def assert_equal_entity(a, b):
function gen_search_vectors_params (line 357) | def gen_search_vectors_params(field_name, entities, top_k, nq, search_pa...
function update_query_expr (line 377) | def update_query_expr(src_query, keep_old=True, expr=None):
function gen_default_vector_expr (line 386) | def gen_default_vector_expr(default_query):
function gen_default_term_expr (line 390) | def gen_default_term_expr(keyword="term", field="int64", values=None):
function update_term_expr (line 397) | def update_term_expr(src_term, terms):
function gen_default_range_expr (line 404) | def gen_default_range_expr(keyword="range", field="int64", ranges=None):
function update_range_expr (line 411) | def update_range_expr(src_range, ranges):
function gen_invalid_range (line 418) | def gen_invalid_range():
function gen_valid_ranges (line 428) | def gen_valid_ranges():
function gen_invalid_term (line 439) | def gen_invalid_term():
function add_field_default (line 449) | def add_field_default(default_fields, type=DataType.INT64, field_name=No...
function add_field (line 461) | def add_field(entities, field_name=None):
function add_vector_field (line 475) | def add_vector_field(entities, is_normal=False):
function remove_field (line 497) | def remove_field(entities):
function remove_vector_field (line 502) | def remove_vector_field(entities):
function update_field_name (line 507) | def update_field_name(entities, old_name, new_name):
function update_field_type (line 515) | def update_field_type(entities, old_name, new_name):
function update_field_value (line 523) | def update_field_value(entities, old_type, new_value):
function update_field_name_row (line 532) | def update_field_name_row(entities, old_name, new_name):
function update_field_type_row (line 543) | def update_field_type_row(entities, old_name, new_name):
function add_vector_field (line 551) | def add_vector_field(nb, dimension=default_dim):
function gen_segment_row_limits (line 561) | def gen_segment_row_limits():
function gen_invalid_ips (line 569) | def gen_invalid_ips():
function gen_invalid_uris (line 589) | def gen_invalid_uris():
function gen_invalid_strs (line 618) | def gen_invalid_strs():
function gen_invalid_field_types (line 635) | def gen_invalid_field_types():
function gen_invalid_metric_types (line 647) | def gen_invalid_metric_types():
function gen_invalid_ints (line 660) | def gen_invalid_ints():
function gen_invalid_params (line 676) | def gen_invalid_params():
function gen_invalid_vectors (line 690) | def gen_invalid_vectors():
function gen_invalid_index (line 712) | def gen_invalid_index():
function gen_index (line 734) | def gen_index():
function gen_simple_index (line 762) | def gen_simple_index():
function gen_binary_index (line 773) | def gen_binary_index():
function gen_normal_expressions (line 783) | def gen_normal_expressions():
function assert_equal_vector (line 792) | def assert_equal_vector(v1, v2):
function restart_server (line 799) | def restart_server(helm_release_name):
function compare_list_elements (line 887) | def compare_list_elements(_first, _second):
function get_token (line 898) | def get_token(url):
function get_tags (line 909) | def get_tags(url, token):
function get_master_tags (line 929) | def get_master_tags(tags_list, tag_prefix="master", tag_latest="master-l...
function get_config_digest (line 942) | def get_config_digest(url, token):
function get_latest_tag (line 962) | def get_latest_tag(limit=100, tag_prefix="master", tag_latest="master-la...
class MyThread (line 991) | class MyThread(threading.Thread):
method __init__ (line 992) | def __init__(self, target, args=()):
method run (line 995) | def run(self):
method join (line 1003) | def join(self):
FILE: tests/utils/wrapper.py
function trace (line 9) | def trace(fmt=DEFAULT_FMT, prefix='test', flag=True):
function counter (line 41) | def counter(func):
function snooze (line 62) | def snooze(seconds, name='snooze'):
Condensed preview — 185 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (3,126K chars).
[
{
"path": ".clang-format",
"chars": 1233,
"preview": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE f"
},
{
"path": ".gitignore",
"chars": 1897,
"preview": ".DS_Store\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distributio"
},
{
"path": ".gitmodules",
"chars": 228,
"preview": "[submodule \"thirdparty/milvus-proto\"]\n\tpath = thirdparty/milvus-proto\n\turl = https://github.com/milvus-io/milvus-proto.g"
},
{
"path": ".pylintrc",
"chars": 14015,
"preview": "# This Pylint rcfile contains a best-effort configuration to uphold the\n# best-practices and style described in the Goog"
},
{
"path": "CMakeLists.txt",
"chars": 12348,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "CONTRIBUTING.md",
"chars": 1282,
"preview": "# Contributing Guide\n\nWelcome contributors! This guide will help you get started with contributing to Milvus-lite.\n\nPlea"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 9598,
"preview": "<div align=\"center\">\n <img src=\"https://raw.githubusercontent.com/milvus-io/milvus-lite/refs/heads/main/milvus_lite_l"
},
{
"path": "ci/e2e.groovy",
"chars": 2737,
"preview": "#!/usr/bin/env groovy\n\nint total_timeout_minutes = 60 * 5\n\npipeline {\n options {\n timestamps()\n timeout"
},
{
"path": "ci/entrypoint.sh",
"chars": 126,
"preview": "#!/bin/bash\ngit config --global --add safe.directory /root/milvus-lite/thirdparty/milvus;\ncd python\npython3 -m build --w"
},
{
"path": "ci/get_author_email.sh",
"chars": 496,
"preview": "#!/bin/bash\n# author emaill will be jenkins's email, when the pr branch is not fast forward compared with master branch\n"
},
{
"path": "ci/nightly.groovy",
"chars": 3184,
"preview": "#!/usr/bin/env groovy\n\nint total_timeout_minutes = 60 * 5\n\n\n// When scheduling a job that gets automatically triggered b"
},
{
"path": "ci/rte-build.yaml",
"chars": 1893,
"preview": "\napiVersion: v1\nkind: Pod\nmetadata:\n labels:\n app: milvus-e2e\n task: milvus-e2e-build\n namespace: milvus-ci\nspec"
},
{
"path": "ci/set_docker_mirror.sh",
"chars": 2181,
"preview": "#!/usr/bin/env bash\n\n# Licensed to the LF AI & Data foundation under one\n# or more contributor license agreements. See t"
},
{
"path": "ci/test.sh",
"chars": 1258,
"preview": "#!/bin/bash\n\n\n\n# Function to display usage\nusage() {\n echo \"Usage: $0 -m {nightly|e2e}\"\n exit 1\n}\n\n# Check if the "
},
{
"path": "cmake/Findantlr4-runtime.cmake",
"chars": 159,
"preview": "FIND_PATH(antlr4-cppruntime_INCLUDES antlr4-runtime.h PATH_SUFFIXES antlr4-runtime REQUIRED)\nFIND_LIBRARY(antlr4-cpprunt"
},
{
"path": "cmake/milvus-storage.cmake",
"chars": 1008,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "conanfile.py",
"chars": 3530,
"preview": "from conans import ConanFile\nfrom conan.tools.cmake import CMake\n\n\nclass MilvusLiteConan(ConanFile):\n settings = \"os\""
},
{
"path": "examples/bfloat16_example.py",
"chars": 2192,
"preview": "import random\nimport tensorflow as tf\nfrom pymilvus import (\n connections,\n utility,\n FieldSchema, Collectio"
},
{
"path": "examples/binary_example.py",
"chars": 2317,
"preview": "import random\nimport numpy as np\nfrom pymilvus import (\n connections,\n utility,\n FieldSchema, CollectionSche"
},
{
"path": "examples/bm25.py",
"chars": 3205,
"preview": "from pymilvus import (\n MilvusClient,\n Function,\n FunctionType,\n DataType,\n)\n\nfmt = \"\\n=== {:30} ===\\n\"\ncoll"
},
{
"path": "examples/customize_schema.py",
"chars": 2716,
"preview": "import time\nimport numpy as np\nfrom pymilvus import (\n MilvusClient,\n DataType\n)\n\nfmt = \"\\n=== {:30} ===\\n\"\ndim = "
},
{
"path": "examples/customize_schema_auto_id.py",
"chars": 2549,
"preview": "import numpy as np\nfrom pymilvus import (\n MilvusClient,\n DataType\n)\n\nfmt = \"\\n=== {:30} ===\\n\"\ndim = 8\ncollection"
},
{
"path": "examples/dynamic_field.py",
"chars": 3397,
"preview": "import time\nimport numpy as np\nfrom pymilvus import (\n connections,\n utility,\n FieldSchema, CollectionSchema, D"
},
{
"path": "examples/example_group_by.py",
"chars": 2166,
"preview": "from pymilvus import CollectionSchema, FieldSchema, Collection, connections, DataType, Partition, utility\nimport random\n"
},
{
"path": "examples/float16_example.py",
"chars": 2215,
"preview": "import random\nimport numpy as np\nfrom pymilvus import (\n connections,\n utility,\n FieldSchema, CollectionSche"
},
{
"path": "examples/fuzzy_match.py",
"chars": 2200,
"preview": "from pymilvus import (\n connections,\n FieldSchema, CollectionSchema, DataType,\n Collection,\n)\n\nDIMENSION = 8\nCO"
},
{
"path": "examples/hello_hybrid_bm25.py",
"chars": 6656,
"preview": "# A demo showing hybrid semantic search with dense and full text search with BM25\n# using Milvus.\n#\n# You can optionally"
},
{
"path": "examples/hello_milvus.py",
"chars": 7628,
"preview": "# hello_milvus.py demonstrates the basic operations of PyMilvus, a Python SDK of Milvus.\n# 1. connect to Milvus\n# 2. cre"
},
{
"path": "examples/hello_milvus_array.py",
"chars": 3427,
"preview": "from pymilvus import CollectionSchema, FieldSchema, Collection, connections, DataType, Partition, utility\nimport numpy a"
},
{
"path": "examples/hello_milvus_delete.py",
"chars": 2560,
"preview": "import time\nimport numpy as np\nfrom pymilvus import (\n MilvusClient,\n exceptions\n)\n\nfmt = \"\\n=== {:30} ===\\n\"\ndim "
},
{
"path": "examples/index.py",
"chars": 3280,
"preview": "from milvus_lite.server_manager import server_manager_instance\n\nimport time\nimport numpy as np\nfrom pymilvus import (\n "
},
{
"path": "examples/non_ascii_encode.py",
"chars": 1276,
"preview": "import numpy as np\nfrom pymilvus import MilvusClient, DataType\n\ndimension = 128\ncollection_name = \"books\"\nclient = Milvu"
},
{
"path": "examples/simple.py",
"chars": 2576,
"preview": "import numpy as np\nfrom pymilvus import (\n MilvusClient,\n)\n\nfmt = \"\\n=== {:30} ===\\n\"\ndim = 8\ncollection_name = \"hell"
},
{
"path": "examples/simple_auto_id.py",
"chars": 2082,
"preview": "import numpy as np\nfrom pymilvus import (\n MilvusClient,\n)\n\nfmt = \"\\n=== {:30} ===\\n\"\ndim = 8\ncollection_name = \"hell"
},
{
"path": "examples/sparse.py",
"chars": 3587,
"preview": "from pymilvus import (\n MilvusClient,\n FieldSchema, CollectionSchema, DataType,\n)\nimport random\n\ndef generate_spar"
},
{
"path": "python/pyproject.toml",
"chars": 176,
"preview": "[build-system]\nrequires = [\n \"setuptools>=64.0\",\n \"wheel\"\n]\n\n# Use legacy backend to import local packages in setu"
},
{
"path": "python/requirements.txt",
"chars": 5,
"preview": "tqdm\n"
},
{
"path": "python/setup.py",
"chars": 7290,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "python/src/milvus_lite/__init__.py",
"chars": 963,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "python/src/milvus_lite/cmdline.py",
"chars": 4410,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "python/src/milvus_lite/server.py",
"chars": 5027,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "python/src/milvus_lite/server_manager.py",
"chars": 1914,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "scripts/Dockerfile.manylinux.aarch64",
"chars": 1291,
"preview": "# Copyright (C) 2019-2020 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "scripts/Dockerfile.manylinux.x86_64",
"chars": 1223,
"preview": "# Copyright (C) 2019-2020 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "scripts/README.md",
"chars": 111,
"preview": "# Build Milvus-Lite In Docker\n\n## build commond\n\n```shell\n build.sh ${Dockerfile} ${TAG} {$CONAN_CACHE}\n```\n"
},
{
"path": "scripts/build.sh",
"chars": 1679,
"preview": "#!/bin/bash\n\n# Licensed to the LF AI & Data foundation under one\n# or more contributor license agreements. See the NOTIC"
},
{
"path": "scripts/build_milvus_lite.sh",
"chars": 1151,
"preview": "#!/bin/bash\n\n# Licensed to the LF AI & Data foundation under one\n# or more contributor license agreements. See the NOTIC"
},
{
"path": "src/CMakeLists.txt",
"chars": 2275,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "src/common.h",
"chars": 4122,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/create_collection_task.cpp",
"chars": 15615,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/create_collection_task.h",
"chars": 2070,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/create_index_task.cpp",
"chars": 19900,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/create_index_task.h",
"chars": 2863,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/delete_task.cpp",
"chars": 1546,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/delete_task.h",
"chars": 1312,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/function/bm25_function.cpp",
"chars": 4064,
"preview": "// Copyright (C) 2019-2025 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/function/bm25_function.h",
"chars": 3897,
"preview": "// Copyright (C) 2019-2025 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/function/function.h",
"chars": 4954,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/function/function_executor.cpp",
"chars": 4905,
"preview": "// Copyright (C) 2019-2025 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/function/function_executor.h",
"chars": 2180,
"preview": "// Copyright (C) 2019-2025 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/function/function_util.h",
"chars": 1185,
"preview": "// Copyright (C) 2019-2025 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/hybrid_search_task.cpp",
"chars": 13187,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/hybrid_search_task.h",
"chars": 2884,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/index.cpp",
"chars": 3647,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/index.h",
"chars": 2131,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/insert_task.cpp",
"chars": 10614,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/insert_task.h",
"chars": 1889,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/milvus_id.hpp",
"chars": 2046,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/milvus_local.cpp",
"chars": 13426,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/milvus_local.h",
"chars": 3799,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/milvus_proxy.cpp",
"chars": 20586,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/milvus_proxy.h",
"chars": 3901,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/milvus_service_impl.cpp",
"chars": 11383,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/milvus_service_impl.h",
"chars": 6180,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/parser/Plan.g4",
"chars": 5366,
"preview": "grammar Plan;\n\nexpr:\n\tIntegerConstant\t\t\t\t\t\t\t\t\t\t\t # Integer\n\t| FloatingConstant\t\t\t\t\t\t\t\t\t\t "
},
{
"path": "src/parser/antlr/PlanBaseVisitor.cpp",
"chars": 76,
"preview": "\n// Generated from Plan.g4 by ANTLR 4.13.1\n\n\n#include \"PlanBaseVisitor.h\"\n\n\n"
},
{
"path": "src/parser/antlr/PlanBaseVisitor.h",
"chars": 3772,
"preview": "\n// Generated from Plan.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n#include \"PlanVisitor.h\"\n\n\n/**\n *"
},
{
"path": "src/parser/antlr/PlanLexer.cpp",
"chars": 24914,
"preview": "\n// Generated from Plan.g4 by ANTLR 4.13.1\n\n\n#include \"PlanLexer.h\"\n\n\nusing namespace antlr4;\n\n\n\nusing namespace antlr4;"
},
{
"path": "src/parser/antlr/PlanLexer.h",
"chars": 1727,
"preview": "\n// Generated from Plan.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n\n\n\n\nclass PlanLexer : public ant"
},
{
"path": "src/parser/antlr/PlanParser.cpp",
"chars": 56066,
"preview": "\n// Generated from Plan.g4 by ANTLR 4.13.1\n\n\n#include \"PlanVisitor.h\"\n\n#include \"PlanParser.h\"\n\n\nusing namespace antlrcp"
},
{
"path": "src/parser/antlr/PlanParser.h",
"chars": 12057,
"preview": "\n// Generated from Plan.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n\n\n\n\nclass PlanParser : public an"
},
{
"path": "src/parser/antlr/PlanVisitor.cpp",
"chars": 72,
"preview": "\n// Generated from Plan.g4 by ANTLR 4.13.1\n\n\n#include \"PlanVisitor.h\"\n\n\n"
},
{
"path": "src/parser/antlr/PlanVisitor.h",
"chars": 2720,
"preview": "\n// Generated from Plan.g4 by ANTLR 4.13.1\n\n#pragma once\n\n\n#include \"antlr4-runtime.h\"\n#include \"PlanParser.h\"\n\n\n\n/**\n *"
},
{
"path": "src/parser/parser.cc",
"chars": 2103,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/parser/parser.h",
"chars": 71807,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/parser/utils.h",
"chars": 29370,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/query_task.cpp",
"chars": 8310,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/query_task.h",
"chars": 1791,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/re_scorer.cpp",
"chars": 6369,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/re_scorer.h",
"chars": 2911,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/retrieve_result.h",
"chars": 1312,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/schema_util.cpp",
"chars": 29974,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/schema_util.h",
"chars": 6418,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/search_result.h",
"chars": 1523,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/search_task.cpp",
"chars": 11481,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/search_task.h",
"chars": 2287,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/segcore_wrapper.cpp",
"chars": 11433,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/segcore_wrapper.h",
"chars": 4277,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/server.cpp",
"chars": 2778,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/status.h",
"chars": 7500,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/storage/bm25_stats.cpp",
"chars": 7834,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/storage/bm25_stats.h",
"chars": 3081,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/storage/collection_data.cpp",
"chars": 5817,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/storage/collection_data.h",
"chars": 3089,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/storage/collection_meta.cpp",
"chars": 7690,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/storage/collection_meta.h",
"chars": 7600,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/storage/storage.cpp",
"chars": 10060,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/storage/storage.h",
"chars": 4731,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/string_util.hpp",
"chars": 3050,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/timer.cpp",
"chars": 977,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/timer.h",
"chars": 2684,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/type.h",
"chars": 830,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/unittest/CMakeLists.txt",
"chars": 2019,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "src/unittest/bm25_function_test.cpp",
"chars": 2790,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/unittest/grpc_server_test.cpp",
"chars": 14411,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/unittest/milvus_local_test.cpp",
"chars": 6693,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/unittest/milvus_proxy_test.cpp",
"chars": 9505,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/unittest/run_examples.py",
"chars": 1320,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "src/unittest/storage_test.cpp",
"chars": 2993,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/unittest/test_util.cpp",
"chars": 12338,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/unittest/test_util.h",
"chars": 3256,
"preview": "// Copyright (C) 2019-2024 Zilliz. All rights reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "src/upsert_task.cpp",
"chars": 1891,
"preview": "#include \"upsert_task.h\"\n#include <cstdint>\n#include <string>\n#include \"schema.pb.h\"\n#include \"status.h\"\n\nnamespace milv"
},
{
"path": "src/upsert_task.h",
"chars": 724,
"preview": "#pragma once\n\n#include \"common.h\"\n#include \"pb/milvus.pb.h\"\n#include \"pb/schema.pb.h\"\n#include \"status.h\"\n\nnamespace mil"
},
{
"path": "tests/base/client_base.py",
"chars": 21773,
"preview": "from numpy.core.fromnumeric import _partition_dispatcher\nimport pytest\nimport sys\nfrom pymilvus import DefaultConfig\nfro"
},
{
"path": "tests/base/collection_wrapper.py",
"chars": 18888,
"preview": "import sys\nimport time\nimport timeout_decorator\nfrom numpy import NaN\n\nfrom pymilvus import Collection\n\nsys.path.append("
},
{
"path": "tests/base/connections_wrapper.py",
"chars": 3820,
"preview": "from pymilvus import Connections\nfrom pymilvus import DefaultConfig\nfrom pymilvus import MilvusClient\nimport sys\n\nsys.pa"
},
{
"path": "tests/base/database_wrapper.py",
"chars": 1727,
"preview": "import sys\n\nfrom pymilvus import db\n\nfrom utils.api_request import api_request\nfrom check.func_check import ResponseChec"
},
{
"path": "tests/base/high_level_api_wrapper.py",
"chars": 36910,
"preview": "import sys\nimport time\nimport timeout_decorator\nfrom numpy import NaN\n\nfrom pymilvus import Collection\nfrom pymilvus imp"
},
{
"path": "tests/base/index_wrapper.py",
"chars": 2017,
"preview": "import sys\nfrom pymilvus import Index\n\nsys.path.append(\"..\")\nfrom check.func_check import ResponseChecker\nfrom utils.api"
},
{
"path": "tests/base/partition_wrapper.py",
"chars": 6809,
"preview": "import sys\nfrom numpy import NaN\n\nfrom pymilvus import Partition\n\nsys.path.append(\"..\")\nfrom check.func_check import Res"
},
{
"path": "tests/base/schema_wrapper.py",
"chars": 3353,
"preview": "import sys\n\nsys.path.append(\"..\")\nfrom check.func_check import ResponseChecker\nfrom utils.api_request import api_request"
},
{
"path": "tests/base/utility_wrapper.py",
"chars": 31524,
"preview": "from datetime import datetime\nimport time\nfrom pymilvus import utility\nimport sys\n\nsys.path.append(\"..\")\nfrom check.func"
},
{
"path": "tests/check/func_check.py",
"chars": 27809,
"preview": "from pymilvus.client.types import CompactionPlans\nfrom pymilvus import Role\n\nfrom utils.util_log import test_log as log\n"
},
{
"path": "tests/check/param_check.py",
"chars": 7976,
"preview": "import sys\nimport operator\nfrom common import common_type as ct\n\nsys.path.append(\"..\")\nfrom utils.util_log import test_l"
},
{
"path": "tests/common/bulk_insert_data.py",
"chars": 49892,
"preview": "import copy\nimport json\nimport os\nimport time\n\nimport numpy as np\nimport jax.numpy as jnp\nimport pandas as pd\nimport ran"
},
{
"path": "tests/common/code_mapping.py",
"chars": 853,
"preview": "from enum import Enum\nfrom pymilvus import ExceptionsMessage\n\n\nclass ErrorCode(Enum):\n ErrorOk = 0\n Error = 1\n\n\nEr"
},
{
"path": "tests/common/common_func.py",
"chars": 96939,
"preview": "import os\nimport random\nimport math\nimport string\nimport json\nimport time\nimport uuid\nfrom functools import singledispat"
},
{
"path": "tests/common/common_type.py",
"chars": 10461,
"preview": "\"\"\" Initialized parameters \"\"\"\nport = 19530\nepsilon = 0.000001\nnamespace = \"milvus\"\ndefault_flush_interval = 1\nbig_flush"
},
{
"path": "tests/common/constants.py",
"chars": 950,
"preview": "# constants for old pymilvus API test\n\nimport utils.util_pymilvus as utils\n\ndefault_fields = utils.gen_default_fields()\n"
},
{
"path": "tests/common/cus_resource_opts.py",
"chars": 4884,
"preview": "from __future__ import print_function\n\nimport os\n\nfrom kubernetes import client, config\nfrom kubernetes.client.rest impo"
},
{
"path": "tests/common/milvus_sys.py",
"chars": 3929,
"preview": "import ujson\nimport json\nfrom pymilvus.grpc_gen import milvus_pb2 as milvus_types\nfrom pymilvus import connections\n\nsys_"
},
{
"path": "tests/common/minio_comm.py",
"chars": 1389,
"preview": "import os\nfrom minio import Minio\nfrom minio.error import S3Error\nfrom utils.util_log import test_log as log\n\n\ndef copy_"
},
{
"path": "tests/config/log_config.py",
"chars": 1468,
"preview": "import os\nfrom pathlib import Path\n\nclass LogConfig:\n def __init__(self):\n self.log_debug = \"\"\n self.lo"
},
{
"path": "tests/conftest.py",
"chars": 17041,
"preview": "import logging\n\nimport pytest\nimport functools\nimport socket\n\nimport common.common_type as ct\nimport common.common_func "
},
{
"path": "tests/customize/README.md",
"chars": 109,
"preview": "Verify user is able to easily customize Milvus deployment with various configuration items.\n\nTo be updated..."
},
{
"path": "tests/customize/milvus_operator.py",
"chars": 6651,
"preview": "import json\nimport os\nimport time\nfrom benedict import benedict\nfrom utils.util_log import test_log as log\nfrom common.c"
},
{
"path": "tests/customize/template/default.yaml",
"chars": 581,
"preview": "apiVersion: milvus.io/v1beta1\nkind: Milvus\nmetadata:\n name: my-release\n namespace: chaos-testing\n labels:\n app: mi"
},
{
"path": "tests/customize/template/minimum.yaml",
"chars": 1176,
"preview": "apiVersion: milvus.io/v1alpha1\nkind: MilvusCluster\nmetadata:\n name: my-release\n labels:\n app: milvus\nspec:\n compon"
},
{
"path": "tests/customize/test_customize_segment_size.py",
"chars": 5546,
"preview": "import pytest\nimport time\n\nfrom pymilvus import connections\nfrom utils.util_log import test_log as log\nfrom base.collect"
},
{
"path": "tests/customize/test_simd_compat.py",
"chars": 6013,
"preview": "import pytest\nimport time\n\nfrom pymilvus import connections\nfrom utils.util_log import test_log as log\nfrom base.collect"
},
{
"path": "tests/data/train_embeddings.csv",
"chars": 1139866,
"preview": "path,label,vector\n./train/brain_coral/n01917289_1783.JPEG,brain_coral,\"[0.001221718150191009, 0.019385283812880516, 0.00"
},
{
"path": "tests/milvus_lite/test_milvus_lite_close.py",
"chars": 5232,
"preview": "import pytest\nimport numpy as np\nfrom base.client_base import TestcaseBase\nfrom common.common_type import CaseLabel, Che"
},
{
"path": "tests/milvus_lite/test_milvus_lite_collection.py",
"chars": 61001,
"preview": "import multiprocessing\nimport numbers\nimport random\nimport numpy\nimport threading\nimport pytest\nimport pandas as pd\nimpo"
},
{
"path": "tests/milvus_lite/test_milvus_lite_delete.py",
"chars": 12950,
"preview": "import multiprocessing\nimport numbers\nimport random\nimport numpy\nimport threading\nimport pytest\nimport pandas as pd\nimpo"
},
{
"path": "tests/milvus_lite/test_milvus_lite_index.py",
"chars": 33287,
"preview": "import multiprocessing\nimport numbers\nimport random\nimport numpy\nimport threading\nimport pytest\nimport pandas as pd\nimpo"
},
{
"path": "tests/milvus_lite/test_milvus_lite_init.py",
"chars": 2380,
"preview": "import pytest\nimport numpy as np\nfrom base.client_base import TestcaseBase\nfrom common.common_type import CaseLabel, Che"
},
{
"path": "tests/milvus_lite/test_milvus_lite_insert.py",
"chars": 56107,
"preview": "import multiprocessing\nimport numbers\nimport random\nimport numpy\nimport threading\nimport pytest\nimport pandas as pd\nimpo"
},
{
"path": "tests/milvus_lite/test_milvus_lite_migrate.py",
"chars": 4428,
"preview": "import json\nimport random\nimport time\nfrom multiprocessing import Process\nimport numpy as np\nfrom pymilvus import (\n "
},
{
"path": "tests/milvus_lite/test_milvus_lite_query.py",
"chars": 23748,
"preview": "import multiprocessing\nimport numbers\nimport random\nimport numpy\nimport threading\nimport pytest\nimport pandas as pd\nimpo"
},
{
"path": "tests/milvus_lite/test_milvus_lite_search.py",
"chars": 125564,
"preview": "import math\nimport multiprocessing\nimport numbers\nimport random\nimport numpy\nimport threading\n\nimport numpy as np\nimport"
},
{
"path": "tests/milvus_lite/test_milvus_lite_sparse.py",
"chars": 4583,
"preview": "import time\n\nimport pytest\n\nfrom base.client_base import TestcaseBase\nfrom common.common_type import CaseLabel, CheckTas"
},
{
"path": "tests/milvus_lite/test_milvus_lite_stability.py",
"chars": 5798,
"preview": "import logging\nimport random\nimport string\nimport time\nimport numpy as np\nimport pytest\n\nfrom base.client_base import Te"
},
{
"path": "tests/pytest.ini",
"chars": 305,
"preview": "[pytest]\n\n\naddopts = --host localhost --html=/tmp/ci_logs/report.html --self-contained-html -v\n# python3 -W ignore -m p"
},
{
"path": "tests/requirements.txt",
"chars": 1026,
"preview": "--extra-index-url https://test.pypi.org/simple/\npytest-cov==2.8.1\nrequests==2.26.0\nscikit-learn==1.1.3\ntimeout_decorator"
},
{
"path": "tests/run.sh",
"chars": 24,
"preview": "#/bin/bash\n\n\npytest . $@"
},
{
"path": "tests/test_bm25.py",
"chars": 9834,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "tests/test_delete.py",
"chars": 4659,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "tests/test_dump_tool.py",
"chars": 7963,
"preview": "# Copyright (C) 2019-2023 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "tests/test_hybrid_search.py",
"chars": 14145,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "tests/test_query.py",
"chars": 5076,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "tests/test_schema.py",
"chars": 4593,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "tests/test_search.py",
"chars": 10034,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "tests/test_sparse.py",
"chars": 8410,
"preview": "# Copyright (C) 2019-2024 Zilliz. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\""
},
{
"path": "tests/testcases/test_collection.py",
"chars": 243488,
"preview": "import random\n\nimport numpy\nimport pandas as pd\nimport pytest\n\nfrom pymilvus import DataType\nfrom base.client_base impor"
},
{
"path": "tests/utils/api_request.py",
"chars": 2270,
"preview": "import traceback\nimport copy\nimport os\nfrom utils.util_log import test_log as log\n\n# enable_traceback = os.getenv('ENABL"
},
{
"path": "tests/utils/thread_util.py",
"chars": 804,
"preview": "import logging\nimport threading\nfrom typing import Callable\n\n\nclass ConcurrencyObj:\n def __init__(self, func_obj: Cal"
},
{
"path": "tests/utils/util_common.py",
"chars": 4049,
"preview": "import glob\nimport time\nfrom yaml import full_load\nimport json\nimport pandas as pd\nfrom utils.util_log import test_log a"
},
{
"path": "tests/utils/util_k8s.py",
"chars": 18299,
"preview": "import json\nimport os.path\nimport time\nimport pyetcd\nimport requests\nfrom pymilvus import connections\nfrom kubernetes im"
},
{
"path": "tests/utils/util_log.py",
"chars": 1917,
"preview": "import logging\nimport sys\n\nfrom config.log_config import log_config\n\n\nclass TestLog:\n def __init__(self, logger, log_"
},
{
"path": "tests/utils/util_pymilvus.py",
"chars": 29596,
"preview": "import json\nimport random\nimport string\nimport threading\nimport traceback\nimport time\nimport copy\nimport numpy as np\nimp"
},
{
"path": "tests/utils/wrapper.py",
"chars": 2281,
"preview": "import time\nfrom datetime import datetime\nimport functools\nfrom utils.util_log import test_log as log\n\nDEFAULT_FMT = '[{"
},
{
"path": "thirdparty/knowhere-android.patch",
"chars": 77912,
"preview": "diff --git a/CMakeLists.txt b/CMakeLists.txt\nindex bd495fcd..a46918b0 100644\n--- a/CMakeLists.txt\n+++ b/CMakeLists.txt\n@"
},
{
"path": "thirdparty/knowhere.patch",
"chars": 1667,
"preview": "diff --git a/CMakeLists.txt b/CMakeLists.txt\nindex b7870adf..0d8fc16c 100644\n--- a/CMakeLists.txt\n+++ b/CMakeLists.txt\n@"
},
{
"path": "thirdparty/milvus.patch",
"chars": 12918,
"preview": "diff --git a/internal/core/cmake/Utils.cmake b/internal/core/cmake/Utils.cmake\nindex 8facf7cccd..f0d75bf8be 100644\n--- a"
}
]
About this extraction
This page contains the full source code of the milvus-io/milvus-lite GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 185 files (2.9 MB), approximately 770.0k tokens, and a symbol index with 1557 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.