Showing preview only (217K chars total). Download the full file or copy to clipboard to get everything.
Repository: stephan-hof/pyrocksdb
Branch: master
Commit: 23e304b0d1cf
Files: 48
Total size: 203.9 KB
Directory structure:
gitextract_d2ye60kk/
├── .gitignore
├── LICENSE.md
├── MANIFEST.in
├── README.rst
├── docs/
│ ├── Makefile
│ ├── api/
│ │ ├── backup.rst
│ │ ├── database.rst
│ │ ├── index.rst
│ │ ├── interfaces.rst
│ │ └── options.rst
│ ├── changelog.rst
│ ├── conf.py
│ ├── index.rst
│ ├── installation.rst
│ └── tutorial/
│ └── index.rst
├── rocksdb/
│ ├── __init__.py
│ ├── _rocksdb.pyx
│ ├── backup.pxd
│ ├── cache.pxd
│ ├── comparator.pxd
│ ├── cpp/
│ │ ├── comparator_wrapper.hpp
│ │ ├── filter_policy_wrapper.hpp
│ │ ├── memtable_factories.hpp
│ │ ├── merge_operator_wrapper.hpp
│ │ ├── slice_transform_wrapper.hpp
│ │ ├── utils.hpp
│ │ └── write_batch_iter_helper.hpp
│ ├── db.pxd
│ ├── env.pxd
│ ├── errors.py
│ ├── filter_policy.pxd
│ ├── interfaces.py
│ ├── iterator.pxd
│ ├── logger.pxd
│ ├── memtablerep.pxd
│ ├── merge_operator.pxd
│ ├── options.pxd
│ ├── slice_.pxd
│ ├── slice_transform.pxd
│ ├── snapshot.pxd
│ ├── status.pxd
│ ├── std_memory.pxd
│ ├── table_factory.pxd
│ ├── tests/
│ │ ├── __init__.py
│ │ ├── test_db.py
│ │ └── test_options.py
│ └── universal_compaction.pxd
└── setup.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
/build/
/docs/_build/
================================================
FILE: LICENSE.md
================================================
Copyright (c) 2014, Stephan Hofmockel
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
* Neither the name of the author nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================
FILE: MANIFEST.in
================================================
include rocksdb/cpp/*.hpp
recursive-include rocksdb *.pxd
recursive-include rocksdb *.pyx
================================================
FILE: README.rst
================================================
pyrocksdb
=========
Python bindings for RocksDB.
See http://pyrocksdb.readthedocs.org for a more comprehensive install and usage description.
Quick Install
-------------
Quick install for debian/ubuntu like linux distributions.
.. code-block:: bash
$ apt-get install build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev
$ git clone https://github.com/facebook/rocksdb.git
$ cd rocksdb
$ make shared_lib
$ export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:`pwd`/include
$ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:`pwd`
$ export LIBRARY_PATH=${LIBRARY_PATH}:`pwd`
$ cd ../
$ apt-get install python-virtualenv python-dev
$ virtualenv pyrocks_test
$ cd pyrocks_test
$ . bin/active
$ pip install pyrocksdb
Quick Usage Guide
-----------------
.. code-block:: pycon
>>> import rocksdb
>>> db = rocksdb.DB("test.db", rocksdb.Options(create_if_missing=True))
>>> db.put(b'a', b'data')
>>> print db.get(b'a')
b'data'
================================================
FILE: docs/Makefile
================================================
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
clean:
rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/rocksdb.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/rocksdb.qhc"
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/rocksdb"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/rocksdb"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
================================================
FILE: docs/api/backup.rst
================================================
Backup and Restore
******************
BackupEngine
============
.. py:class:: rocksdb.BackupEngine
.. py:method:: __init__(backup_dir)
Creates a object to manage backup of a single database.
:param unicode backup_dir: Where to keep the backup files.
Has to be different than db.db_name.
For example db.db_name + '/backups'.
.. py:method:: create_backup(db, flush_before_backup=False)
Triggers the creation of a backup.
:param db: Database object to backup.
:type db: :py:class:`rocksdb.DB`
:param bool flush_before_backup: If ``True`` the current memtable is flushed.
.. py:method:: restore_backup(backup_id, db_dir, wal_dir)
Restores the backup from the given id.
:param int backup_id: id of the backup to restore.
:param unicode db_dir: Target directory to restore backup.
:param unicode wal_dir: Target directory to restore backuped WAL files.
.. py:method:: restore_latest_backup(db_dir, wal_dir)
Restores the latest backup.
:param unicode db_dir: see :py:meth:`restore_backup`
:param unicode wal_dir: see :py:meth:`restore_backup`
.. py:method:: stop_backup()
Can be called from another thread to stop the current backup process.
.. py:method:: purge_old_backups(num_backups_to_keep)
Deletes all backups (oldest first) until "num_backups_to_keep" are left.
:param int num_backups_to_keep: Number of backupfiles to keep.
.. py:method:: delete_backup(backup_id)
:param int backup_id: Delete the backup with the given id.
.. py:method:: get_backup_info()
Returns information about all backups.
It returns a list of dict's where each dict as the following keys.
``backup_id``
(int): id of this backup.
``timestamp``
(int): Seconds since epoch, when the backup was created.
``size``
(int): Size in bytes of the backup.
================================================
FILE: docs/api/database.rst
================================================
Database interactions
*********************
Database object
===============
.. py:class:: rocksdb.DB
.. py:method:: __init__(db_name, Options opts, read_only=False)
:param unicode db_name: Name of the database to open
:param opts: Options for this specific database
:type opts: :py:class:`rocksdb.Options`
:param bool read_only: If ``True`` the database is opened read-only.
All DB calls which modify data will raise an
Exception.
.. py:method:: put(key, value, sync=False, disable_wal=False)
Set the database entry for "key" to "value".
:param bytes key: Name for this entry
:param bytes value: Data for this entry
:param bool sync:
If ``True``, the write will be flushed from the operating system
buffer cache (by calling WritableFile::Sync()) before the write
is considered complete. If this flag is true, writes will be
slower.
If this flag is ``False``, and the machine crashes, some recent
writes may be lost. Note that if it is just the process that
crashes (i.e., the machine does not reboot), no writes will be
lost even if ``sync == False``.
In other words, a DB write with ``sync == False`` has similar
crash semantics as the "write()" system call. A DB write
with ``sync == True`` has similar crash semantics to a "write()"
system call followed by "fdatasync()".
:param bool disable_wal:
If ``True``, writes will not first go to the write ahead log,
and the write may got lost after a crash.
.. py:method:: delete(key, sync=False, disable_wal=False)
Remove the database entry for "key".
:param bytes key: Name to delete
:param sync: See :py:meth:`rocksdb.DB.put`
:param disable_wal: See :py:meth:`rocksdb.DB.put`
:raises rocksdb.errors.NotFound: If the key did not exists
.. py:method:: merge(key, value, sync=False, disable_wal=False)
Merge the database entry for "key" with "value".
The semantics of this operation is determined by the user provided
merge_operator when opening DB.
See :py:meth:`rocksdb.DB.put` for the parameters
:raises:
:py:exc:`rocksdb.errors.NotSupported` if this is called and
no :py:attr:`rocksdb.Options.merge_operator` was set at creation
.. py:method:: write(batch, sync=False, disable_wal=False)
Apply the specified updates to the database.
:param rocksdb.WriteBatch batch: Batch to apply
:param sync: See :py:meth:`rocksdb.DB.put`
:param disable_wal: See :py:meth:`rocksdb.DB.put`
.. py:method:: get(key, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all")
:param bytes key: Name to get
:param bool verify_checksums:
If ``True``, all data read from underlying storage will be
verified against corresponding checksums.
:param bool fill_cache:
Should the "data block", "index block" or "filter block"
read for this iteration be cached in memory?
Callers may wish to set this field to ``False`` for bulk scans.
:param snapshot:
If not ``None``, read as of the supplied snapshot
(which must belong to the DB that is being read and which must
not have been released). Is it ``None`` a implicit snapshot of the
state at the beginning of this read operation is used
:type snapshot: :py:class:`rocksdb.Snapshot`
:param string read_tier:
Specify if this read request should process data that ALREADY
resides on a particular cache. If the required data is not
found at the specified cache,
then :py:exc:`rocksdb.errors.Incomplete` is raised.
| Use ``all`` if a fetch from disk is allowed.
| Use ``cache`` if only data from cache is allowed.
:returns: ``None`` if not found, else the value for this key
.. py:method:: multi_get(keys, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all")
:param keys: Keys to fetch
:type keys: list of bytes
For the other params see :py:meth:`rocksdb.DB.get`
:returns:
A ``dict`` where the value is either ``bytes`` or ``None`` if not found
:raises: If the fetch for a single key fails
.. note::
keys will not be "de-duplicated".
Duplicate keys will return duplicate values in order.
.. py:method:: key_may_exist(key, fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all")
If the key definitely does not exist in the database, then this method
returns ``False``, else ``True``. If the caller wants to obtain value
when the key is found in memory, fetch should be set to ``True``.
This check is potentially lighter-weight than invoking DB::get().
One way to make this lighter weight is to avoid doing any IOs.
:param bytes key: Key to check
:param bool fetch: Obtain also the value if found
For the other params see :py:meth:`rocksdb.DB.get`
:returns:
* ``(True, None)`` if key is found but value not in memory
* ``(True, None)`` if key is found and ``fetch=False``
* ``(True, <data>)`` if key is found and value in memory and ``fetch=True``
* ``(False, None)`` if key is not found
.. py:method:: iterkeys(fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all")
Iterate over the keys
For other params see :py:meth:`rocksdb.DB.get`
:returns:
A iterator object which is not valid yet.
Call first one of the seek methods of the iterator to position it
:rtype: :py:class:`rocksdb.BaseIterator`
.. py:method:: itervalues(fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all")
Iterate over the values
For other params see :py:meth:`rocksdb.DB.get`
:returns:
A iterator object which is not valid yet.
Call first one of the seek methods of the iterator to position it
:rtype: :py:class:`rocksdb.BaseIterator`
.. py:method:: iteritems(fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier="all")
Iterate over the items
For other params see :py:meth:`rocksdb.DB.get`
:returns:
A iterator object which is not valid yet.
Call first one of the seek methods of the iterator to position it
:rtype: :py:class:`rocksdb.BaseIterator`
.. py:method:: snapshot()
Return a handle to the current DB state.
Iterators created with this handle will all observe a stable snapshot
of the current DB state.
:rtype: :py:class:`rocksdb.Snapshot`
.. py:method:: get_property(prop)
DB implementations can export properties about their state
via this method. If "property" is a valid property understood by this
DB implementation, a byte string with its value is returned.
Otherwise ``None``
Valid property names include:
* ``b"rocksdb.num-files-at-level<N>"``: return the number of files at level <N>,
where <N> is an ASCII representation of a level number (e.g. "0").
* ``b"rocksdb.stats"``: returns a multi-line byte string that describes statistics
about the internal operation of the DB.
* ``b"rocksdb.sstables"``: returns a multi-line byte string that describes all
of the sstables that make up the db contents.
* ``b"rocksdb.num-immutable-mem-table"``: Number of immutable mem tables.
* ``b"rocksdb.mem-table-flush-pending"``: Returns ``1`` if mem table flush is pending, otherwise ``0``.
* ``b"rocksdb.compaction-pending"``: Returns ``1`` if a compaction is pending, otherweise ``0``.
* ``b"rocksdb.background-errors"``: Returns accumulated background errors encountered.
* ``b"rocksdb.cur-size-active-mem-table"``: Returns current size of the active memtable.
.. py:method:: get_live_files_metadata()
Returns a list of all table files.
It returns a list of dict's were each dict has the following keys.
``name``
Name of the file
``level``
Level at which this file resides
``size``
File size in bytes
``smallestkey``
Smallest user defined key in the file
``largestkey``
Largest user defined key in the file
``smallest_seqno``
smallest seqno in file
``largest_seqno``
largest seqno in file
.. py:method:: compact_range(begin=None, end=None, ** options)
Compact the underlying storage for the key range [begin,end].
The actual compaction interval might be superset of [begin, end].
In particular, deleted and overwritten versions are discarded,
and the data is rearranged to reduce the cost of operations
needed to access the data.
This operation should typically only be invoked by users who understand
the underlying implementation.
``begin == None`` is treated as a key before all keys in the database.
``end == None`` is treated as a key after all keys in the database.
Therefore the following call will compact the entire database: ``db.compact_range()``.
Note that after the entire database is compacted, all data are pushed
down to the last level containing any data. If the total data size
after compaction is reduced, that level might not be appropriate for
hosting all the files. In this case, client could set change_level
to ``True``, to move the files back to the minimum level capable of holding
the data set or a given level (specified by non-negative target_level).
:param bytes begin: Key where to start compaction.
If ``None`` start at the beginning of the database.
:param bytes end: Key where to end compaction.
If ``None`` end at the last key of the database.
:param bool change_level: If ``True``, compacted files will be moved to
the minimum level capable of holding the data
or given level (specified by non-negative target_level).
If ``False`` you may end with a bigger level
than configured. Default is ``False``.
:param int target_level: If change_level is true and target_level have non-negative
value, compacted files will be moved to target_level.
Default is ``-1``.
:param string bottommost_level_compaction:
For level based compaction, we can configure if we want to
skip/force bottommost level compaction. By default level based
compaction will only compact the bottommost level if there is a
compaction filter. It can be set to the following values.
``skip``
Skip bottommost level compaction
``if_compaction_filter``
Only compact bottommost level if there is a compaction filter.
This is the default.
``force``
Always compact bottommost level
.. py:attribute:: options
Returns the associated :py:class:`rocksdb.Options` instance.
.. note::
Changes to this object have no effect anymore.
Consider this as read-only
Iterator
========
.. py:class:: rocksdb.BaseIterator
Base class for all iterators in this module. After creation a iterator is
invalid. Call one of the seek methods first before starting iteration
.. py:method:: seek_to_first()
Position at the first key in the source
.. py:method:: seek_to_last()
Position at the last key in the source
.. py:method:: seek(key)
:param bytes key: Position at the first key in the source that at or past
Methods to support the python iterator protocol
.. py:method:: __iter__()
.. py:method:: __next__()
.. py:method:: __reversed__()
Snapshot
========
.. py:class:: rocksdb.Snapshot
Opaque handler for a single Snapshot.
Snapshot is released if nobody holds a reference on it.
Retrieved via :py:meth:`rocksdb.DB.snapshot`
WriteBatch
==========
.. py:class:: rocksdb.WriteBatch
WriteBatch holds a collection of updates to apply atomically to a DB.
The updates are applied in the order in which they are added
to the WriteBatch. For example, the value of "key" will be "v3"
after the following batch is written::
batch = rocksdb.WriteBatch()
batch.put(b"key", b"v1")
batch.delete(b"key")
batch.put(b"key", b"v2")
batch.put(b"key", b"v3")
.. py:method:: __init__(data=None)
Creates a WriteBatch.
:param bytes data:
A serialized version of a previous WriteBatch. As retrieved
from a previous .data() call. If ``None`` a empty WriteBatch is
generated
.. py:method:: put(key, value)
Store the mapping "key->value" in the database.
:param bytes key: Name of the entry to store
:param bytes value: Data of this entry
.. py:method:: merge(key, value)
Merge "value" with the existing value of "key" in the database.
:param bytes key: Name of the entry to merge
:param bytes value: Data to merge
.. py:method:: delete(key)
If the database contains a mapping for "key", erase it. Else do nothing.
:param bytes key: Key to erase
.. py:method:: clear()
Clear all updates buffered in this batch.
.. note::
Don't call this method if there is an outstanding iterator.
Calling :py:meth:`rocksdb.WriteBatch.clear()` with outstanding
iterator, leads to SEGFAULT.
.. py:method:: data()
Retrieve the serialized version of this batch.
:rtype: ``bytes``
.. py:method:: count()
Returns the number of updates in the batch
:rtype: int
.. py:method:: __iter__()
Returns an iterator over the current contents of the write batch.
If you add new items to the batch, they are not visible for this
iterator. Create a new one if you need to see them.
.. note::
Calling :py:meth:`rocksdb.WriteBatch.clear()` on the write batch
invalidates the iterator. Using a iterator where its corresponding
write batch has been cleared, leads to SEGFAULT.
:rtype: :py:class:`rocksdb.WriteBatchIterator`
WriteBatchIterator
==================
.. py:class:: rocksdb.WriteBatchIterator
.. py:method:: __iter__()
Returns self.
.. py:method:: __next__()
Returns the next item inside the corresponding write batch.
The return value is a tuple of always size three.
First item (Name of the operation):
* ``"Put"``
* ``"Merge"``
* ``"Delete"``
Second item (key):
Key for this operation.
Third item (value):
The value for this operation. Empty for ``"Delete"``.
Repair DB
=========
.. py:function:: repair_db(db_name, opts)
:param unicode db_name: Name of the database to open
:param opts: Options for this specific database
:type opts: :py:class:`rocksdb.Options`
If a DB cannot be opened, you may attempt to call this method to
resurrect as much of the contents of the database as possible.
Some data may be lost, so be careful when calling this function
on a database that contains important information.
Errors
======
.. py:exception:: rocksdb.errors.NotFound
.. py:exception:: rocksdb.errors.Corruption
.. py:exception:: rocksdb.errors.NotSupported
.. py:exception:: rocksdb.errors.InvalidArgument
.. py:exception:: rocksdb.errors.RocksIOError
.. py:exception:: rocksdb.errors.MergeInProgress
.. py:exception:: rocksdb.errors.Incomplete
================================================
FILE: docs/api/index.rst
================================================
Python driver for RocksDB
=========================
.. py:module:: rocksdb
.. toctree::
Options <options>
Database <database>
Interfaces <interfaces>
Backup <backup>
================================================
FILE: docs/api/interfaces.rst
================================================
Interfaces
**********
Comparator
==========
.. py:class:: rocksdb.interfaces.Comparator
A Comparator object provides a total order across slices that are
used as keys in an sstable or a database. A Comparator implementation
must be thread-safe since rocksdb may invoke its methods concurrently
from multiple threads.
.. py:method:: compare(a, b)
Three-way comparison.
:param bytes a: First field to compare
:param bytes b: Second field to compare
:returns: * -1 if a < b
* 0 if a == b
* 1 if a > b
:rtype: ``int``
.. py:method:: name()
The name of the comparator. Used to check for comparator
mismatches (i.e., a DB created with one comparator is
accessed using a different comparator).
The client of this package should switch to a new name whenever
the comparator implementation changes in a way that will cause
the relative ordering of any two keys to change.
Names starting with "rocksdb." are reserved and should not be used
by any clients of this package.
:rtype: ``bytes``
Merge Operator
==============
Essentially, a MergeOperator specifies the SEMANTICS of a merge, which only
client knows. It could be numeric addition, list append, string
concatenation, edit data structure, whatever.
The library, on the other hand, is concerned with the exercise of this
interface, at the right time (during get, iteration, compaction...)
To use merge, the client needs to provide an object implementing one of
the following interfaces:
* AssociativeMergeOperator - for most simple semantics (always take
two values, and merge them into one value, which is then put back
into rocksdb).
numeric addition and string concatenation are examples.
* MergeOperator - the generic class for all the more complex operations.
One method (FullMerge) to merge a Put/Delete value with a merge operand.
Another method (PartialMerge) that merges two operands together.
This is especially useful if your key values have a complex structure but
you would still like to support client-specific incremental updates.
AssociativeMergeOperator is simpler to implement.
MergeOperator is simply more powerful.
See this page for more details
https://github.com/facebook/rocksdb/wiki/Merge-Operator
AssociativeMergeOperator
------------------------
.. py:class:: rocksdb.interfaces.AssociativeMergeOperator
.. py:method:: merge(key, existing_value, value)
Gives the client a way to express the read -> modify -> write semantics
:param bytes key: The key that's associated with this merge operation
:param bytes existing_value: The current value in the db.
``None`` indicates the key does not exist
before this op
:param bytes value: The value to update/merge the existing_value with
:returns: ``True`` and the new value on success.
All values passed in will be client-specific values.
So if this method returns false, it is because client
specified bad data or there was internal corruption.
The client should assume that this will be treated as an
error by the library.
:rtype: ``(bool, bytes)``
.. py:method:: name()
The name of the MergeOperator. Used to check for MergeOperator mismatches.
For example a DB created with one MergeOperator is accessed using a
different MergeOperator.
:rtype: ``bytes``
MergeOperator
-------------
.. py:class:: rocksdb.interfaces.MergeOperator
.. py:method:: full_merge(key, existing_value, operand_list)
Gives the client a way to express the read -> modify -> write semantics
:param bytes key: The key that's associated with this merge operation.
Client could multiplex the merge operator based on it
if the key space is partitioned and different subspaces
refer to different types of data which have different
merge operation semantics
:param bytes existing_value: The current value in the db.
``None`` indicates the key does not exist
before this op
:param operand_list: The sequence of merge operations to apply.
:type operand_list: list of bytes
:returns: ``True`` and the new value on success.
All values passed in will be client-specific values.
So if this method returns false, it is because client
specified bad data or there was internal corruption.
The client should assume that this will be treated as an
error by the library.
:rtype: ``(bool, bytes)``
.. py:method:: partial_merge(key, left_operand, right_operand)
This function performs merge(left_op, right_op)
when both the operands are themselves merge operation types
that you would have passed to a DB::Merge() call in the same order.
For example DB::Merge(key,left_op), followed by DB::Merge(key,right_op)).
PartialMerge should combine them into a single merge operation that is
returned together with ``True``
This new value should be constructed such that a call to
DB::Merge(key, new_value) would yield the same result as a call
to DB::Merge(key, left_op) followed by DB::Merge(key, right_op).
If it is impossible or infeasible to combine the two operations,
return ``(False, None)`` The library will internally keep track of the
operations, and apply them in the correct order once a base-value
(a Put/Delete/End-of-Database) is seen.
:param bytes key: the key that is associated with this merge operation.
:param bytes left_operand: First operand to merge
:param bytes right_operand: Second operand to merge
:rtype: ``(bool, bytes)``
.. note::
Presently there is no way to differentiate between error/corruption
and simply "return false". For now, the client should simply return
false in any case it cannot perform partial-merge, regardless of reason.
If there is corruption in the data, handle it in the FullMerge() function,
and return false there.
.. py:method:: name()
The name of the MergeOperator. Used to check for MergeOperator mismatches.
For example a DB created with one MergeOperator is accessed using a
different MergeOperator.
:rtype: ``bytes``
FilterPolicy
============
.. py:class:: rocksdb.interfaces.FilterPolicy
.. py:method:: create_filter(keys)
Create a bytestring which can act as a filter for keys.
:param keys: list of keys (potentially with duplicates)
that are ordered according to the user supplied
comparator.
:type keys: list of bytes
:returns: A filter that summarizes keys
:rtype: ``bytes``
.. py:method:: key_may_match(key, filter)
Check if the key is maybe in the filter.
:param bytes key: Key for a single entry inside the database
:param bytes filter: Contains the data returned by a preceding call
to create_filter on this class
:returns: This method must return ``True`` if the key was in the list
of keys passed to create_filter().
This method may return ``True`` or ``False`` if the key was
not on the list, but it should aim to return ``False`` with
a high probability.
:rtype: ``bool``
.. py:method:: name()
Return the name of this policy. Note that if the filter encoding
changes in an incompatible way, the name returned by this method
must be changed. Otherwise, old incompatible filters may be
passed to methods of this type.
:rtype: ``bytes``
SliceTransform
==============
.. py:class:: rocksdb.interfaces.SliceTransform
SliceTransform is currently used to implement the 'prefix-API' of rocksdb.
https://github.com/facebook/rocksdb/wiki/Proposal-for-prefix-API
.. py:method:: transform(src)
:param bytes src: Full key to extract the prefix from.
:returns: A tuple of two interges ``(offset, size)``.
Where the first integer is the offset within the ``src``
and the second the size of the prefix after the offset.
Which means the prefix is generted by ``src[offset:offset+size]``
:rtype: ``(int, int)``
.. py:method:: in_domain(src)
Decide if a prefix can be extraced from ``src``.
Only if this method returns ``True`` :py:meth:`transform` will be
called.
:param bytes src: Full key to check.
:rtype: ``bool``
.. py:method:: in_range(prefix)
Checks if prefix is a valid prefix
:param bytes prefix: Prefix to check.
:returns: ``True`` if ``prefix`` is a valid prefix.
:rtype: ``bool``
.. py:method:: name()
Return the name of this transformation.
:rtype: ``bytes``
================================================
FILE: docs/api/options.rst
================================================
Options creation
****************
Options object
==============
.. py:class:: rocksdb.Options
.. IMPORTANT::
The default values mentioned here, describe the values of the
C++ library only. This wrapper does not set any default value
itself. So as soon as the rocksdb developers change a default value
this document could be outdated. So if you really depend on a default
value, double check it with the according version of the C++ library.
| Most recent default values should be here
| https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h
| https://github.com/facebook/rocksdb/blob/master/util/options.cc
.. py:method:: __init__(**kwargs)
All options mentioned below can also be passed as keyword-arguments in
the constructor. For example::
import rocksdb
opts = rocksdb.Options(create_if_missing=True)
# is the same as
opts = rocksdb.Options()
opts.create_if_missing = True
.. py:attribute:: create_if_missing
If ``True``, the database will be created if it is missing.
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: error_if_exists
If ``True``, an error is raised if the database already exists.
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: paranoid_checks
If ``True``, the implementation will do aggressive checking of the
data it is processing and will stop early if it detects any
errors. This may have unforeseen ramifications: for example, a
corruption of one DB entry may cause a large number of entries to
become unreadable or for the entire DB to become unopenable.
If any of the writes to the database fails (Put, Delete, Merge, Write),
the database will switch to read-only mode and fail all other
Write operations.
| *Type:* ``bool``
| *Default:* ``True``
.. py:attribute:: write_buffer_size
Amount of data to build up in memory (backed by an unsorted log
on disk) before converting to a sorted on-disk file.
Larger values increase performance, especially during bulk loads.
Up to max_write_buffer_number write buffers may be held in memory
at the same time, so you may wish to adjust this parameter to control
memory usage. Also, a larger write buffer will result in a longer recovery
time the next time the database is opened.
| *Type:* ``int``
| *Default:* ``4194304``
.. py:attribute:: max_write_buffer_number
The maximum number of write buffers that are built up in memory.
The default is 2, so that when 1 write buffer is being flushed to
storage, new writes can continue to the other write buffer.
| *Type:* ``int``
| *Default:* ``2``
.. py:attribute:: min_write_buffer_number_to_merge
The minimum number of write buffers that will be merged together
before writing to storage. If set to 1, then
all write buffers are fushed to L0 as individual files and this increases
read amplification because a get request has to check in all of these
files. Also, an in-memory merge may result in writing lesser
data to storage if there are duplicate records in each of these
individual write buffers.
| *Type:* ``int``
| *Default:* ``1``
.. py:attribute:: max_open_files
Number of open files that can be used by the DB. You may need to
increase this if your database has a large working set. Value -1 means
files opened are always kept open. You can estimate number of
files based on target_file_size_base and target_file_size_multiplier
for level-based compaction.
For universal-style compaction, you can usually set it to -1.
| *Type:* ``int``
| *Default:* ``5000``
.. py:attribute:: compression
Compress blocks using the specified compression algorithm.
This parameter can be changed dynamically.
| *Type:* Member of :py:class:`rocksdb.CompressionType`
| *Default:* :py:attr:`rocksdb.CompressionType.snappy_compression`
.. py:attribute:: num_levels
Number of levels for this database
| *Type:* ``int``
| *Default:* ``7``
.. py:attribute:: level0_file_num_compaction_trigger
Number of files to trigger level-0 compaction. A value <0 means that
level-0 compaction will not be triggered by number of files at all.
| *Type:* ``int``
| *Default:* ``4``
.. py:attribute:: level0_slowdown_writes_trigger
Soft limit on number of level-0 files. We start slowing down writes at this
point. A value <0 means that no writing slow down will be triggered by
number of files in level-0.
| *Type:* ``int``
| *Default:* ``20``
.. py:attribute:: level0_stop_writes_trigger
Maximum number of level-0 files. We stop writes at this point.
| *Type:* ``int``
| *Default:* ``24``
.. py:attribute:: max_mem_compaction_level
Maximum level to which a new compacted memtable is pushed if it
does not create overlap. We try to push to level 2 to avoid the
relatively expensive level 0=>1 compactions and to avoid some
expensive manifest file operations. We do not push all the way to
the largest level since that can generate a lot of wasted disk
space if the same key space is being repeatedly overwritten.
| *Type:* ``int``
| *Default:* ``2``
.. py:attribute:: target_file_size_base
| Target file size for compaction.
| target_file_size_base is per-file size for level-1.
| Target file size for level L can be calculated by
| target_file_size_base * (target_file_size_multiplier ^ (L-1)).
For example, if target_file_size_base is 2MB and
target_file_size_multiplier is 10, then each file on level-1 will
be 2MB, and each file on level 2 will be 20MB,
and each file on level-3 will be 200MB.
| *Type:* ``int``
| *Default:* ``2097152``
.. py:attribute:: target_file_size_multiplier
| by default target_file_size_multiplier is 1, which means
| by default files in different levels will have similar size.
| *Type:* ``int``
| *Default:* ``1``
.. py:attribute:: max_bytes_for_level_base
Control maximum total data size for a level.
*max_bytes_for_level_base* is the max total for level-1.
Maximum number of bytes for level L can be calculated as
(*max_bytes_for_level_base*) * (*max_bytes_for_level_multiplier* ^ (L-1))
For example, if *max_bytes_for_level_base* is 20MB, and if
*max_bytes_for_level_multiplier* is 10, total data size for level-1
will be 20MB, total file size for level-2 will be 200MB,
and total file size for level-3 will be 2GB.
| *Type:* ``int``
| *Default:* ``10485760``
.. py:attribute:: max_bytes_for_level_multiplier
See :py:attr:`max_bytes_for_level_base`
| *Type:* ``int``
| *Default:* ``10``
.. py:attribute:: max_bytes_for_level_multiplier_additional
Different max-size multipliers for different levels.
These are multiplied by max_bytes_for_level_multiplier to arrive
at the max-size of each level.
| *Type:* ``[int]``
| *Default:* ``[1, 1, 1, 1, 1, 1, 1]``
.. py:attribute:: expanded_compaction_factor
Maximum number of bytes in all compacted files. We avoid expanding
the lower level file set of a compaction if it would make the
total compaction cover more than
(expanded_compaction_factor * targetFileSizeLevel()) many bytes.
| *Type:* ``int``
| *Default:* ``25``
.. py:attribute:: source_compaction_factor
Maximum number of bytes in all source files to be compacted in a
single compaction run. We avoid picking too many files in the
source level so that we do not exceed the total source bytes
for compaction to exceed
(source_compaction_factor * targetFileSizeLevel()) many bytes.
If 1 pick maxfilesize amount of data as the source of
a compaction.
| *Type:* ``int``
| *Default:* ``1``
.. py:attribute:: max_grandparent_overlap_factor
Control maximum bytes of overlaps in grandparent (i.e., level+2) before we
stop building a single file in a level->level+1 compaction.
| *Type:* ``int``
| *Default:* ``10``
.. py:attribute:: disable_data_sync
If true, then the contents of data files are not synced
to stable storage. Their contents remain in the OS buffers till the
OS decides to flush them. This option is good for bulk-loading
of data. Once the bulk-loading is complete, please issue a
sync to the OS to flush all dirty buffesrs to stable storage.
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: use_fsync
If true, then every store to stable storage will issue a fsync.
If false, then every store to stable storage will issue a fdatasync.
This parameter should be set to true while storing data to
filesystem like ext3 that can lose files after a reboot.
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: db_log_dir
This specifies the info LOG dir.
If it is empty, the log files will be in the same dir as data.
If it is non empty, the log files will be in the specified dir,
and the db data dir's absolute path will be used as the log file
name's prefix.
| *Type:* ``unicode``
| *Default:* ``""``
.. py:attribute:: wal_dir
This specifies the absolute dir path for write-ahead logs (WAL).
If it is empty, the log files will be in the same dir as data,
dbname is used as the data dir by default.
If it is non empty, the log files will be in kept the specified dir.
When destroying the db, all log files in wal_dir and the dir itself is deleted
| *Type:* ``unicode``
| *Default:* ``""``
.. py:attribute:: delete_obsolete_files_period_micros
The periodicity when obsolete files get deleted. The default
value is 6 hours. The files that get out of scope by compaction
process will still get automatically delete on every compaction,
regardless of this setting
| *Type:* ``int``
| *Default:* ``21600000000``
.. py:attribute:: max_background_compactions
Maximum number of concurrent background jobs, submitted to
the default LOW priority thread pool
| *Type:* ``int``
| *Default:* ``1``
.. py:attribute:: max_background_flushes
Maximum number of concurrent background memtable flush jobs, submitted to
the HIGH priority thread pool.
By default, all background jobs (major compaction and memtable flush) go
to the LOW priority pool. If this option is set to a positive number,
memtable flush jobs will be submitted to the HIGH priority pool.
It is important when the same Env is shared by multiple db instances.
Without a separate pool, long running major compaction jobs could
potentially block memtable flush jobs of other db instances, leading to
unnecessary Put stalls.
| *Type:* ``int``
| *Default:* ``1``
.. py:attribute:: max_log_file_size
Specify the maximal size of the info log file. If the log file
is larger than `max_log_file_size`, a new info log file will
be created.
If max_log_file_size == 0, all logs will be written to one
log file.
| *Type:* ``int``
| *Default:* ``0``
.. py:attribute:: log_file_time_to_roll
Time for the info log file to roll (in seconds).
If specified with non-zero value, log file will be rolled
if it has been active longer than `log_file_time_to_roll`.
A value of ``0`` means disabled.
| *Type:* ``int``
| *Default:* ``0``
.. py:attribute:: keep_log_file_num
Maximal info log files to be kept.
| *Type:* ``int``
| *Default:* ``1000``
.. py:attribute:: soft_rate_limit
Puts are delayed 0-1 ms when any level has a compaction score that exceeds
soft_rate_limit. This is ignored when == 0.0.
CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not
hold, RocksDB will set soft_rate_limit = hard_rate_limit.
A value of ``0`` means disabled.
| *Type:* ``float``
| *Default:* ``0``
.. py:attribute:: hard_rate_limit
Puts are delayed 1ms at a time when any level has a compaction score that
exceeds hard_rate_limit. This is ignored when <= 1.0.
A value fo ``0`` means disabled.
| *Type:* ``float``
| *Default:* ``0``
.. py:attribute:: rate_limit_delay_max_milliseconds
Max time a put will be stalled when hard_rate_limit is enforced. If 0, then
there is no limit.
| *Type:* ``int``
| *Default:* ``1000``
.. py:attribute:: max_manifest_file_size
manifest file is rolled over on reaching this limit.
The older manifest file be deleted.
The default value is MAX_INT so that roll-over does not take place.
| *Type:* ``int``
| *Default:* ``(2**64) - 1``
.. py:attribute:: table_cache_numshardbits
Number of shards used for table cache.
| *Type:* ``int``
| *Default:* ``4``
.. py:attribute:: arena_block_size
size of one block in arena memory allocation.
If <= 0, a proper value is automatically calculated (usually 1/10 of
writer_buffer_size).
| *Type:* ``int``
| *Default:* ``0``
.. py:attribute:: disable_auto_compactions
Disable automatic compactions. Manual compactions can still
be issued on this database.
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: wal_ttl_seconds, wal_size_limit_mb
The following two fields affect how archived logs will be deleted.
1. If both set to 0, logs will be deleted asap and will not get into
the archive.
2. If wal_ttl_seconds is 0 and wal_size_limit_mb is not 0,
WAL files will be checked every 10 min and if total size is greater
then wal_size_limit_mb, they will be deleted starting with the
earliest until size_limit is met. All empty files will be deleted.
3. If wal_ttl_seconds is not 0 and wal_size_limit_mb is 0, then
WAL files will be checked every wal_ttl_secondsi / 2 and those that
are older than wal_ttl_seconds will be deleted.
4. If both are not 0, WAL files will be checked every 10 min and both
checks will be performed with ttl being first.
| *Type:* ``int``
| *Default:* ``0``
.. py:attribute:: manifest_preallocation_size
Number of bytes to preallocate (via fallocate) the manifest
files. Default is 4mb, which is reasonable to reduce random IO
as well as prevent overallocation for mounts that preallocate
large amounts of data (such as xfs's allocsize option).
| *Type:* ``int``
| *Default:* ``4194304``
.. py:attribute:: purge_redundant_kvs_while_flush
Purge duplicate/deleted keys when a memtable is flushed to storage.
| *Type:* ``bool``
| *Default:* ``True``
.. py:attribute:: allow_os_buffer
Data being read from file storage may be buffered in the OS
| *Type:* ``bool``
| *Default:* ``True``
.. py:attribute:: allow_mmap_reads
Allow the OS to mmap file for reading sst tables
| *Type:* ``bool``
| *Default:* ``True``
.. py:attribute:: allow_mmap_writes
Allow the OS to mmap file for writing
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: is_fd_close_on_exec
Disable child process inherit open files
| *Type:* ``bool``
| *Default:* ``True``
.. py:attribute:: skip_log_error_on_recovery
Skip log corruption error on recovery
(If client is ok with losing most recent changes)
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: stats_dump_period_sec
If not zero, dump rocksdb.stats to LOG every stats_dump_period_sec
| *Type:* ``int``
| *Default:* ``3600``
.. py:attribute:: advise_random_on_open
If set true, will hint the underlying file system that the file
access pattern is random, when a sst file is opened.
| *Type:* ``bool``
| *Default:* ``True``
.. py:attribute:: use_adaptive_mutex
Use adaptive mutex, which spins in the user space before resorting
to kernel. This could reduce context switch when the mutex is not
heavily contended. However, if the mutex is hot, we could end up
wasting spin time.
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: bytes_per_sync
Allows OS to incrementally sync files to disk while they are being
written, asynchronously, in the background.
Issue one request for every bytes_per_sync written. 0 turns it off.
| *Type:* ``int``
| *Default:* ``0``
.. py:attribute:: verify_checksums_in_compaction
If ``True``, compaction will verify checksum on every read that
happens as part of compaction.
| *Type:* ``bool``
| *Default:* ``True``
.. py:attribute:: compaction_style
The compaction style. Could be set to ``"level"`` to use level-style
compaction. For universal-style compaction use ``"universal"``.
| *Type:* ``string``
| *Default:* ``level``
.. py:attribute:: compaction_options_universal
Options to use for universal-style compaction. They make only sense if
:py:attr:`rocksdb.Options.compaction_style` is set to ``"universal"``.
It is a dict with the following keys.
* ``size_ratio``:
Percentage flexibilty while comparing file size.
If the candidate file(s) size is 1% smaller than the next file's size,
then include next file into this candidate set.
Default: ``1``
* ``min_merge_width``:
The minimum number of files in a single compaction run.
Default: ``2``
* ``max_merge_width``:
The maximum number of files in a single compaction run.
Default: ``UINT_MAX``
* ``max_size_amplification_percent``:
The size amplification is defined as the amount (in percentage) of
additional storage needed to store a single byte of data in the database.
For example, a size amplification of 2% means that a database that
contains 100 bytes of user-data may occupy upto 102 bytes of
physical storage. By this definition, a fully compacted database has
a size amplification of 0%. Rocksdb uses the following heuristic
to calculate size amplification: it assumes that all files excluding
the earliest file contribute to the size amplification.
Default: ``200``, which means that a 100 byte database could require upto
300 bytes of storage.
* ``compression_size_percent``:
If this option is set to be -1 (the default value), all the output
files will follow compression type specified.
If this option is not negative, we will try to make sure compressed
size is just above this value. In normal cases, at least this
percentage of data will be compressed.
When we are compacting to a new file, here is the criteria whether
it needs to be compressed: assuming here are the list of files sorted
by generation time: ``A1...An B1...Bm C1...Ct``
where ``A1`` is the newest and ``Ct`` is the oldest, and we are going
to compact ``B1...Bm``, we calculate the total size of all the files
as total_size, as well as the total size of ``C1...Ct`` as
``total_C``, the compaction output file will be compressed if
``total_C / total_size < this percentage``.
Default: -1
* ``stop_style``:
The algorithm used to stop picking files into a single compaction.
Can be either ``"similar_size"`` or ``"total_size"``.
* ``similar_size``: Pick files of similar size.
* ``total_size``: Total size of picked files is greater than next file.
Default: ``"total_size"``
For setting options, just assign a dict with the fields to set.
It is allowed to omit keys in this dict. Missing keys are just not set
to the underlying options object.
This example just changes the stop_style and leaves the other options
untouched. ::
opts = rocksdb.Options()
opts.compaction_options_universal = {'stop_style': 'similar_size'}
.. py:attribute:: filter_deletes
Use KeyMayExist API to filter deletes when this is true.
If KeyMayExist returns false, i.e. the key definitely does not exist, then
the delete is a noop. KeyMayExist only incurs in-memory look up.
This optimization avoids writing the delete to storage when appropriate.
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: max_sequential_skip_in_iterations
An iteration->Next() sequentially skips over keys with the same
user-key unless this option is set. This number specifies the number
of keys (with the same userkey) that will be sequentially
skipped before a reseek is issued.
| *Type:* ``int``
| *Default:* ``8``
.. py:attribute:: memtable_factory
This is a factory that provides MemTableRep objects.
Right now you can assing instances of the following classes.
* :py:class:`rocksdb.VectorMemtableFactory`
* :py:class:`rocksdb.SkipListMemtableFactory`
* :py:class:`rocksdb.HashSkipListMemtableFactory`
* :py:class:`rocksdb.HashLinkListMemtableFactory`
*Default:* :py:class:`rocksdb.SkipListMemtableFactory`
.. py:attribute:: table_factory
Factory for the files forming the persisten data storage.
Sometimes they are also named SST-Files. Right now you can assign
instances of the following classes.
* :py:class:`rocksdb.BlockBasedTableFactory`
* :py:class:`rocksdb.PlainTableFactory`
* :py:class:`rocksdb.TotalOrderPlainTableFactory`
*Default:* :py:class:`rocksdb.BlockBasedTableFactory`
.. py:attribute:: inplace_update_support
Allows thread-safe inplace updates. Requires Updates if
* key exists in current memtable
* new sizeof(new_value) <= sizeof(old_value)
* old_value for that key is a put i.e. kTypeValue
| *Type:* ``bool``
| *Default:* ``False``
.. py:attribute:: inplace_update_num_locks
| Number of locks used for inplace update.
| Default: 10000, if inplace_update_support = true, else 0.
| *Type:* ``int``
| *Default:* ``10000``
.. py:attribute:: comparator
Comparator used to define the order of keys in the table.
A python comparator must implement the :py:class:`rocksdb.interfaces.Comparator`
interface.
*Requires*: The client must ensure that the comparator supplied
here has the same name and orders keys *exactly* the same as the
comparator provided to previous open calls on the same DB.
*Default:* :py:class:`rocksdb.BytewiseComparator`
.. py:attribute:: merge_operator
The client must provide a merge operator if Merge operation
needs to be accessed. Calling Merge on a DB without a merge operator
would result in :py:exc:`rocksdb.errors.NotSupported`. The client must
ensure that the merge operator supplied here has the same name and
*exactly* the same semantics as the merge operator provided to
previous open calls on the same DB. The only exception is reserved
for upgrade, where a DB previously without a merge operator is
introduced to Merge operation for the first time. It's necessary to
specify a merge operator when openning the DB in this case.
A python merge operator must implement the
:py:class:`rocksdb.interfaces.MergeOperator` or
:py:class:`rocksdb.interfaces.AssociativeMergeOperator`
interface.
*Default:* ``None``
.. py:attribute:: prefix_extractor
If not ``None``, use the specified function to determine the
prefixes for keys. These prefixes will be placed in the filter.
Depending on the workload, this can reduce the number of read-IOP
cost for scans when a prefix is passed to the calls generating an
iterator (:py:meth:`rocksdb.DB.iterkeys` ...).
A python prefix_extractor must implement the
:py:class:`rocksdb.interfaces.SliceTransform` interface
For prefix filtering to work properly, "prefix_extractor" and "comparator"
must be such that the following properties hold:
1. ``key.starts_with(prefix(key))``
2. ``compare(prefix(key), key) <= 0``
3. ``If compare(k1, k2) <= 0, then compare(prefix(k1), prefix(k2)) <= 0``
4. ``prefix(prefix(key)) == prefix(key)``
*Default:* ``None``
.. py:attribute:: row_cache
A global cache for table-level rows. If ``None`` this cache is not used.
Otherwise it must be an instance of :py:class:`rocksdb.LRUCache`
*Default:* ``None``
CompressionTypes
================
.. py:class:: rocksdb.CompressionType
Defines the support compression types
.. py:attribute:: no_compression
.. py:attribute:: snappy_compression
.. py:attribute:: zlib_compression
.. py:attribute:: bzip2_compression
.. py:attribute:: lz4_compression
.. py:attribute:: lz4hc_compression
BytewiseComparator
==================
.. py:class:: rocksdb.BytewiseComparator
Wraps the rocksdb Bytewise Comparator, it uses lexicographic byte-wise
ordering
BloomFilterPolicy
=================
.. py:class:: rocksdb.BloomFilterPolicy
Wraps the rocksdb BloomFilter Policy
.. py:method:: __init__(bits_per_key)
:param int bits_per_key:
Specifies the approximately number of bits per key.
A good value for bits_per_key is 10, which yields a filter with
~ 1% false positive rate.
LRUCache
========
.. py:class:: rocksdb.LRUCache
Wraps the rocksdb LRUCache
.. py:method:: __init__(capacity, shard_bits=None)
Create a new cache with a fixed size capacity (in bytes).
The cache is sharded to 2^numShardBits shards, by hash of the key.
The total capacity is divided and evenly assigned to each shard.
.. _table_factories_label:
TableFactories
==============
Currently RocksDB supports two types of tables: plain table and block-based table.
Instances of this classes can assigned to :py:attr:`rocksdb.Options.table_factory`
* *Block-based table:* This is the default table type that RocksDB inherited from
LevelDB. It was designed for storing data in hard disk or flash device.
* *Plain table:* It is one of RocksDB's SST file format optimized
for low query latency on pure-memory or really low-latency media.
Tutorial of rocksdb table formats is available here:
https://github.com/facebook/rocksdb/wiki/A-Tutorial-of-RocksDB-SST-formats
.. py:class:: rocksdb.BlockBasedTableFactory
Wraps BlockBasedTableFactory of RocksDB.
.. py:method:: __init__(index_type='binary_search', hash_index_allow_collision=True, checksum='crc32', block_cache, block_cache_compressed, filter_policy=None, no_block_cache=False, block_size=None, block_size_deviation=None, block_restart_interval=None, whole_key_filtering=None):
:param string index_type:
* ``binary_search`` a space efficient index block that is optimized
for binary-search-based index.
* ``hash_search`` the hash index. If enabled, will do hash lookup
when `Options.prefix_extractor` is provided.
:param bool hash_index_allow_collision:
Influence the behavior when ``hash_search`` is used.
If ``False``, stores a precise prefix to block range mapping.
If ``True``, does not store prefix and allows prefix hash collision
(less memory consumption)
:param string checksum:
Use the specified checksum type. Newly created table files will be
protected with this checksum type. Old table files will still be readable,
even though they have different checksum type.
Can be either ``crc32`` or ``xxhash``.
:param block_cache:
Control over blocks (user data is stored in a set of blocks, and
a block is the unit of reading from disk).
If ``None``, rocksdb will automatically create and use an 8MB internal cache.
If not ``None`` use the specified cache for blocks. In that case it must
be an instance of :py:class:`rocksdb.LRUCache`
:param block_cache_compressed:
If ``None``, rocksdb will not use a compressed block cache.
If not ``None`` use the specified cache for compressed blocks. In that
case it must be an instance of :py:class:`rocksdb.LRUCache`
:param filter_policy:
If not ``None`` use the specified filter policy to reduce disk reads.
A python filter policy must implement the
:py:class:`rocksdb.interfaces.FilterPolicy` interface.
Recommended is a instance of :py:class:`rocksdb.BloomFilterPolicy`
:param bool no_block_cache:
Disable block cache. If this is set to true,
then no block cache should be used, and the block_cache should
point to ``None``
:param int block_size:
If set to ``None`` the rocksdb default of ``4096`` is used.
Approximate size of user data packed per block. Note that the
block size specified here corresponds to uncompressed data. The
actual size of the unit read from disk may be smaller if
compression is enabled. This parameter can be changed dynamically.
:param int block_size_deviation:
If set to ``None`` the rocksdb default of ``10`` is used.
This is used to close a block before it reaches the configured
'block_size'. If the percentage of free space in the current block is less
than this specified number and adding a new record to the block will
exceed the configured block size, then this block will be closed and the
new record will be written to the next block.
:param int block_restart_interval:
If set to ``None`` the rocksdb default of ``16`` is used.
Number of keys between restart points for delta encoding of keys.
This parameter can be changed dynamically. Most clients should
leave this parameter alone.
:param bool whole_key_filtering:
If set to ``None`` the rocksdb default of ``True`` is used.
If ``True``, place whole keys in the filter (not just prefixes).
This must generally be true for gets to be efficient.
.. py:class:: rocksdb.PlainTableFactory
Plain Table with prefix-only seek. It wraps rocksdb PlainTableFactory.
For this factory, you need to set :py:attr:`rocksdb.Options.prefix_extractor`
properly to make it work. Look-up will start with prefix hash lookup for
key prefix. Inside the hash bucket found, a binary search is executed for
hash conflicts. Finally, a linear search is used.
.. py:method:: __init__(user_key_len=0, bloom_bits_per_key=10, hash_table_ratio=0.75, index_sparseness=10, huge_page_tlb_size=0, encoding_type='plain', full_scan_mode=False, store_index_in_file=False)
:param int user_key_len:
Plain table has optimization for fix-sized keys, which can be
specified via user_key_len.
Alternatively, you can pass `0` if your keys have variable lengths.
:param int bloom_bits_per_key:
The number of bits used for bloom filer per prefix.
You may disable it by passing `0`.
:param float hash_table_ratio:
The desired utilization of the hash table used for prefix hashing.
hash_table_ratio = number of prefixes / #buckets in the hash table.
:param int index_sparseness:
Inside each prefix, need to build one index record for how
many keys for binary search inside each hash bucket.
For encoding type ``prefix``, the value will be used when
writing to determine an interval to rewrite the full key.
It will also be used as a suggestion and satisfied when possible.
:param int huge_page_tlb_size:
If <=0, allocate hash indexes and blooms from malloc.
Otherwise from huge page TLB.
The user needs to reserve huge pages for it to be allocated, like:
``sysctl -w vm.nr_hugepages=20``
See linux doc Documentation/vm/hugetlbpage.txt
:param string encoding_type:
How to encode the keys. The value will determine how to encode keys
when writing to a new SST file. This value will be stored
inside the SST file which will be used when reading from the
file, which makes it possible for users to choose different
encoding type when reopening a DB. Files with different
encoding types can co-exist in the same DB and can be read.
* ``plain``: Always write full keys without any special encoding.
* ``prefix``: Find opportunity to write the same prefix once for multiple rows.
In some cases, when a key follows a previous key with the same prefix,
instead of writing out the full key, it just writes out the size of the
shared prefix, as well as other bytes, to save some bytes.
When using this option, the user is required to use the same prefix
extractor to make sure the same prefix will be extracted from the same key.
The Name() value of the prefix extractor will be stored in the file.
When reopening the file, the name of the options.prefix_extractor given
will be bitwise compared to the prefix extractors stored in the file.
An error will be returned if the two don't match.
:param bool full_scan_mode:
Mode for reading the whole file one record by one without using the index.
:param bool store_index_in_file:
Compute plain table index and bloom filter during file building
and store it in file. When reading file, index will be mmaped
instead of recomputation.
.. _memtable_factories_label:
MemtableFactories
=================
RocksDB has different classes to represent the in-memory buffer for the current
operations. You have to assing instances of the following classes to
:py:attr:`rocksdb.Options.memtable_factory`.
This page has a comparison the most popular ones.
https://github.com/facebook/rocksdb/wiki/Hash-based-memtable-implementations
.. py:class:: rocksdb.VectorMemtableFactory
This creates MemTableReps that are backed by an std::vector.
On iteration, the vector is sorted. This is useful for workloads where
iteration is very rare and writes are generally not issued after reads begin.
.. py:method:: __init__(count=0)
:param int count:
Passed to the constructor of the underlying std::vector of each
VectorRep. On initialization, the underlying array will be at
least count bytes reserved for usage.
.. py:class:: rocksdb.SkipListMemtableFactory
This uses a skip list to store keys.
.. py:method:: __init__()
.. py:class:: rocksdb.HashSkipListMemtableFactory
This class contains a fixed array of buckets, each pointing
to a skiplist (null if the bucket is empty).
.. note::
:py:attr:`rocksdb.Options.prefix_extractor` must be set, otherwise
rocksdb fails back to skip-list.
.. py:method:: __init__(bucket_count = 1000000, skiplist_height = 4, skiplist_branching_factor = 4)
:param int bucket_count: number of fixed array buckets
:param int skiplist_height: the max height of the skiplist
:param int skiplist_branching_factor:
probabilistic size ratio between adjacent link lists in the skiplist
.. py:class:: rocksdb.HashLinkListMemtableFactory
The factory is to create memtables with a hashed linked list.
It contains a fixed array of buckets, each pointing to a sorted single
linked list (null if the bucket is empty).
.. note::
:py:attr:`rocksdb.Options.prefix_extractor` must be set, otherwise
rocksdb fails back to skip-list.
.. py:method:: __init__(bucket_count=50000)
:param int bucket: number of fixed array buckets
================================================
FILE: docs/changelog.rst
================================================
Changelog
*********
Version 0.5
-----------
Version 0.4
-----------
This version works with RocksDB v3.12.
* Added :py:func:`repair_db`.
* Added :py:meth:`rocksdb.Options.row_cache`
* Publish to pypi.
Backward Incompatible Changes:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
* Changed API of :py:meth:`rocksdb.DB.compact_range`.
* Only allow keyword arguments.
* Changed ``reduce_level`` to ``change_level``.
* Add new argument called ``bottommost_level_compaction``.
Version 0.3
-----------
This version works with RocksDB version v3.11.
Backward Incompatible Changes:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
**Prefix Seeks:**
According to this page https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes,
all the prefix related parameters on ``ReadOptions`` are removed.
Rocksdb realizes now if ``Options.prefix_extractor`` is set and uses then
prefix-seeks automatically. This means the following changes on pyrocksdb.
* DB.iterkeys, DB.itervalues, DB.iteritems have *no* ``prefix`` parameter anymore.
* DB.get, DB.multi_get, DB.key_may_exist, DB.iterkeys, DB.itervalues, DB.iteritems
have *no* ``prefix_seek`` parameter anymore.
Which means all the iterators walk now always to the *end* of the database.
So if you need to stay within a prefix, write your own code to ensure that.
For DB.iterkeys and DB.iteritems ``itertools.takewhile`` is a possible solution. ::
from itertools import takewhile
it = self.db.iterkeys()
it.seek(b'00002')
print list(takewhile(lambda key: key.startswith(b'00002'), it))
it = self.db.iteritems()
it.seek(b'00002')
print dict(takewhile(lambda item: item[0].startswith(b'00002'), it))
**SST Table Builders:**
* Removed ``NewTotalOrderPlainTableFactory``, because rocksdb drops it too.
**Changed Options:**
In newer versions of rocksdb a bunch of options were moved or removed.
* Rename ``bloom_bits_per_prefix`` of :py:class:`rocksdb.PlainTableFactory` to ``bloom_bits_per_key``
* Removed ``Options.db_stats_log_interval``.
* Removed ``Options.disable_seek_compaction``
* Moved ``Options.no_block_cache`` to ``BlockBasedTableFactory``
* Moved ``Options.block_size`` to ``BlockBasedTableFactory``
* Moved ``Options.block_size_deviation`` to ``BlockBasedTableFactory``
* Moved ``Options.block_restart_interval`` to ``BlockBasedTableFactory``
* Moved ``Options.whole_key_filtering`` to ``BlockBasedTableFactory``
* Removed ``Options.table_cache_remove_scan_count_limit``
* Removed rm_scan_count_limit from ``LRUCache``
New:
^^^^
* Make CompactRange available: :py:meth:`rocksdb.DB.compact_range`
* Add init options to :py:class:`rocksdb.BlockBasedTableFactory`
* Add more option to :py:class:`rocksdb.PlainTableFactory`
* Add :py:class:`rocksdb.WriteBatchIterator`
* add :py:attr:`rocksdb.CompressionType.lz4_compression`
* add :py:attr:`rocksdb.CompressionType.lz4hc_compression`
Version 0.2
-----------
This version works with RocksDB version 2.8.fb. Now you have access to the more
advanced options of rocksdb. Like changing the memtable or SST representation.
It is also possible now to enable *Universal Style Compaction*.
* Fixed `issue 3 <https://github.com/stephan-hof/pyrocksdb/pull/3>`_.
Which fixed the change of prefix_extractor from raw-pointer to smart-pointer.
* Support the new :py:attr:`rocksdb.Options.verify_checksums_in_compaction` option.
* Add :py:attr:`rocksdb.Options.table_factory` option. So you could use the new
'PlainTableFactories' which are optimized for in-memory-databases.
* https://github.com/facebook/rocksdb/wiki/PlainTable-Format
* https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database%3F
* Add :py:attr:`rocksdb.Options.memtable_factory` option.
* Add options :py:attr:`rocksdb.Options.compaction_style` and
:py:attr:`rocksdb.Options.compaction_options_universal` to change the
compaction style.
* Update documentation to the new default values
* allow_mmap_reads=true
* allow_mmap_writes=false
* max_background_flushes=1
* max_open_files=5000
* paranoid_checks=true
* disable_seek_compaction=true
* level0_stop_writes_trigger=24
* level0_slowdown_writes_trigger=20
* Document new property names for :py:meth:`rocksdb.DB.get_property`.
Version 0.1
-----------
Initial version. Works with rocksdb version 2.7.fb.
================================================
FILE: docs/conf.py
================================================
# -*- coding: utf-8 -*-
#
# pyrocksdb documentation build configuration file, created by
# sphinx-quickstart on Tue Dec 31 12:50:54 2013.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys
import os
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.todo',
'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'pyrocksdb'
copyright = u'2014, sh'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.4'
# The full version, including alpha/beta/rc tags.
release = '0.4'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
# html_theme = 'default'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'pyrocksdbdoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'pyrocksdb.tex', u'pyrocksdb Documentation',
u'sh', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'pyrocksdb', u'pyrocksdb Documentation',
[u'sh'], 1)
]
# If true, show URL addresses after external links.
#man_show_urls = False
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'pyrocksdb', u'pyrocksdb Documentation',
u'sh', 'pyrocksdb', 'One line description of project.',
'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
================================================
FILE: docs/index.rst
================================================
Welcome to pyrocksdb's documentation!
=====================================
Overview
--------
Python bindings to the C++ interface of http://rocksdb.org/ using cython::
import rocksdb
db = rocksdb.DB("test.db", rocksdb.Options(create_if_missing=True))
db.put(b"a", b"b")
print db.get(b"a")
Tested with python2.7 and python3.4 and RocksDB version 3.12
.. toctree::
:maxdepth: 2
Instructions how to install <installation>
Tutorial <tutorial/index>
API <api/index>
Changelog <changelog>
Contributing
------------
Source can be found on `github <https://github.com/stephan-hof/pyrocksdb>`_.
Feel free to fork and send pull-requests or create issues on the
`github issue tracker <https://github.com/stephan-hof/pyrocksdb/issues>`_
RoadMap/TODO
------------
No plans so far. Please submit wishes to the github issues.
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
================================================
FILE: docs/installation.rst
================================================
Installing
**********
.. highlight:: bash
Building rocksdb
----------------
Briefly describes how to build rocksdb under an ordinary debian/ubuntu.
For more details consider https://github.com/facebook/rocksdb/blob/master/INSTALL.md
.. code-block:: bash
apt-get install build-essential
apt-get install libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev
git clone https://github.com/facebook/rocksdb.git
cd rocksdb
make shared_lib
Systemwide rocksdb
^^^^^^^^^^^^^^^^^^
The following command installs the shared library in ``/usr/lib/`` and the
header files in ``/usr/include/rocksdb/``::
make install-shared INSTALL_PATH=/usr
To uninstall use::
make uninstall INSTALL_PATH=/usr
Local rocksdb
^^^^^^^^^^^^^
If you don't like the system wide installation, or you don't have the
permissions, it is possible to set the following environment variables.
These varialbes are picked up by the compiler, linker and loader
.. code-block:: bash
export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:`pwd`/include
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:`pwd`
export LIBRARY_PATH=${LIBRARY_PATH}:`pwd`
Building pyrocksdb
------------------
.. code-block:: bash
apt-get install python-virtualenv python-dev
virtualenv pyrocks_test
cd pyrocks_test
. bin/active
pip install "Cython>=0.20"
pip install git+git://github.com/stephan-hof/pyrocksdb.git
================================================
FILE: docs/tutorial/index.rst
================================================
Basic Usage of pyrocksdb
************************
Open
====
The most basic open call is ::
import rocksdb
db = rocksdb.DB("test.db", rocksdb.Options(create_if_missing=True))
A more production ready open can look like this ::
import rocksdb
opts = rocksdb.Options()
opts.create_if_missing = True
opts.max_open_files = 300000
opts.write_buffer_size = 67108864
opts.max_write_buffer_number = 3
opts.target_file_size_base = 67108864
opts.table_factory = rocksdb.BlockBasedTableFactory(
filter_policy=rocksdb.BloomFilterPolicy(10),
block_cache=rocksdb.LRUCache(2 * (1024 ** 3)),
block_cache_compressed=rocksdb.LRUCache(500 * (1024 ** 2)))
db = rocksdb.DB("test.db", opts)
It assings a cache of 2.5G, uses a bloom filter for faster lookups and keeps
more data (64 MB) in memory before writting a .sst file.
About Bytes And Unicode
========================
RocksDB stores all data as uninterpreted *byte strings*.
pyrocksdb behaves the same and uses nearly everywhere byte strings too.
In python2 this is the ``str`` type. In python3 the ``bytes`` type.
Since the default string type for string literals differs between python 2 and 3,
it is strongly recommended to use an explicit ``b`` prefix for all byte string
literals in both python2 and python3 code.
For example ``b'this is a byte string'``. This avoids ambiguity and ensures
that your code keeps working as intended if you switch between python2 and python3.
The only place where you can pass unicode objects are filesytem paths like
* Directory name of the database itself :py:meth:`rocksdb.DB.__init__`
* :py:attr:`rocksdb.Options.wal_dir`
* :py:attr:`rocksdb.Options.db_log_dir`
To encode this path name, `sys.getfilesystemencoding()` encoding is used.
Access
======
Store, Get, Delete is straight forward ::
# Store
db.put(b"key", b"value")
# Get
db.get(b"key")
# Delete
db.delete(b"key")
It is also possible to gather modifications and
apply them in a single operation ::
batch = rocksdb.WriteBatch()
batch.put(b"key", b"v1")
batch.delete(b"key")
batch.put(b"key", b"v2")
batch.put(b"key", b"v3")
db.write(batch)
Fetch of multiple values at once ::
db.put(b"key1", b"v1")
db.put(b"key2", b"v2")
ret = db.multi_get([b"key1", b"key2", b"key3"])
# prints b"v1"
print ret[b"key1"]
# prints None
print ret[b"key3"]
Iteration
=========
Iterators behave slightly different than expected. Per default they are not
valid. So you have to call one of its seek methods first ::
db.put(b"key1", b"v1")
db.put(b"key2", b"v2")
db.put(b"key3", b"v3")
it = db.iterkeys()
it.seek_to_first()
# prints [b'key1', b'key2', b'key3']
print list(it)
it.seek_to_last()
# prints [b'key3']
print list(it)
it.seek(b'key2')
# prints [b'key2', b'key3']
print list(it)
There are also methods to iterate over values/items ::
it = db.itervalues()
it.seek_to_first()
# prints [b'v1', b'v2', b'v3']
print list(it)
it = db.iteritems()
it.seek_to_first()
# prints [(b'key1', b'v1'), (b'key2, b'v2'), (b'key3', b'v3')]
print list(it)
Reversed iteration ::
it = db.iteritems()
it.seek_to_last()
# prints [(b'key3', b'v3'), (b'key2', b'v2'), (b'key1', b'v1')]
print list(reversed(it))
Snapshots
=========
Snapshots are nice to get a consistent view on the database ::
self.db.put(b"a", b"1")
self.db.put(b"b", b"2")
snapshot = self.db.snapshot()
self.db.put(b"a", b"2")
self.db.delete(b"b")
it = self.db.iteritems()
it.seek_to_first()
# prints {b'a': b'2'}
print dict(it)
it = self.db.iteritems(snapshot=snapshot)
it.seek_to_first()
# prints {b'a': b'1', b'b': b'2'}
print dict(it)
MergeOperator
=============
Merge operators are useful for efficient read-modify-write operations.
For more details see `Merge Operator <https://github.com/facebook/rocksdb/wiki/Merge-Operator>`_
A python merge operator must either implement the
:py:class:`rocksdb.interfaces.AssociativeMergeOperator` or
:py:class:`rocksdb.interfaces.MergeOperator` interface.
The following example python merge operator implements a counter ::
class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator):
def merge(self, key, existing_value, value):
if existing_value:
s = int(existing_value) + int(value)
return (True, str(s).encode('ascii'))
return (True, value)
def name(self):
return b'AssocCounter'
opts = rocksdb.Options()
opts.create_if_missing = True
opts.merge_operator = AssocCounter()
db = rocksdb.DB('test.db', opts)
db.merge(b"a", b"1")
db.merge(b"a", b"1")
# prints b'2'
print db.get(b"a")
PrefixExtractor
===============
According to `Prefix API <https://github.com/facebook/rocksdb/wiki/Proposal-for-prefix-API>`_
a prefix_extractor can reduce IO for scans within a prefix range.
A python prefix extractor must implement the :py:class:`rocksdb.interfaces.SliceTransform` interface.
The following example presents a prefix extractor of a static size.
So always the first 5 bytes are used as the prefix ::
class StaticPrefix(rocksdb.interfaces.SliceTransform):
def name(self):
return b'static'
def transform(self, src):
return (0, 5)
def in_domain(self, src):
return len(src) >= 5
def in_range(self, dst):
return len(dst) == 5
opts = rocksdb.Options()
opts.create_if_missing=True
opts.prefix_extractor = StaticPrefix()
db = rocksdb.DB('test.db', opts)
db.put(b'00001.x', b'x')
db.put(b'00001.y', b'y')
db.put(b'00001.z', b'z')
db.put(b'00002.x', b'x')
db.put(b'00002.y', b'y')
db.put(b'00002.z', b'z')
db.put(b'00003.x', b'x')
db.put(b'00003.y', b'y')
db.put(b'00003.z', b'z')
prefix = b'00002'
it = db.iteritems()
it.seek(prefix)
# prints {b'00002.z': b'z', b'00002.y': b'y', b'00002.x': b'x'}
print dict(itertools.takewhile(lambda item: item[0].startswith(prefix), it))
Backup And Restore
==================
Backup and Restore is done with a separate :py:class:`rocksdb.BackupEngine` object.
A backup can only be created on a living database object. ::
import rocksdb
db = rocksdb.DB("test.db", rocksdb.Options(create_if_missing=True))
db.put(b'a', b'v1')
db.put(b'b', b'v2')
db.put(b'c', b'v3')
Backup is created like this.
You can choose any path for the backup destination except the db path itself.
If ``flush_before_backup`` is ``True`` the current memtable is flushed to disk
before backup. ::
backup = rocksdb.BackupEngine("test.db/backups")
backup.create_backup(db, flush_before_backup=True)
Restore is done like this.
The two arguments are the db_dir and wal_dir, which are mostly the same. ::
backup = rocksdb.BackupEngine("test.db/backups")
backup.restore_latest_backup("test.db", "test.db")
Change Memtable Or SST Implementations
======================================
As noted here :ref:`memtable_factories_label`, RocksDB offers different implementations for the memtable
representation. Per default :py:class:`rocksdb.SkipListMemtableFactory` is used,
but changing it to a different one is veary easy.
Here is an example for HashSkipList-MemtableFactory.
Keep in mind: To use the hashed based MemtableFactories you must set
:py:attr:`rocksdb.Options.prefix_extractor`.
In this example all keys have a static prefix of len 5. ::
class StaticPrefix(rocksdb.interfaces.SliceTransform):
def name(self):
return b'static'
def transform(self, src):
return (0, 5)
def in_domain(self, src):
return len(src) >= 5
def in_range(self, dst):
return len(dst) == 5
opts = rocksdb.Options()
opts.prefix_extractor = StaticPrefix()
opts.memtable_factory = rocksdb.HashSkipListMemtableFactory()
opts.create_if_missing = True
db = rocksdb.DB("test.db", opts)
db.put(b'00001.x', b'x')
db.put(b'00001.y', b'y')
db.put(b'00002.x', b'x')
For initial bulk loads the Vector-MemtableFactory makes sense. ::
opts = rocksdb.Options()
opts.memtable_factory = rocksdb.VectorMemtableFactory()
opts.create_if_missing = True
db = rocksdb.DB("test.db", opts)
As noted here :ref:`table_factories_label`, it is also possible to change the
representation of the final data files.
Here is an example how to use a 'PlainTable'. ::
opts = rocksdb.Options()
opts.table_factory = rocksdb.PlainTableFactory()
opts.create_if_missing = True
db = rocksdb.DB("test.db", opts)
Change Compaction Style
=======================
RocksDB has a compaction algorithm called *universal*. This style typically
results in lower write amplification but higher space amplification than
Level Style Compaction. See here for more details,
https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide#multi-threaded-compactions
Here is an example to switch to *universal style compaction*. ::
opts = rocksdb.Options()
opts.compaction_style = "universal"
opts.compaction_options_universal = {"min_merge_width": 3}
See here for more options on *universal style compaction*,
:py:attr:`rocksdb.Options.compaction_options_universal`
Iterate Over WriteBatch
=======================
In same cases you need to know, what operations happened on a WriteBatch.
The pyrocksdb WriteBatch supports the iterator protocol, see this example. ::
batch = rocksdb.WriteBatch()
batch.put(b"key1", b"v1")
batch.delete(b'a')
batch.merge(b'xxx', b'value')
for op, key, value in batch:
print op, key, value
# prints the following three lines
# Put key1 v1
# Delete a
# Merge xxx value
================================================
FILE: rocksdb/__init__.py
================================================
from ._rocksdb import *
================================================
FILE: rocksdb/_rocksdb.pyx
================================================
import cython
from libcpp.string cimport string
from libcpp.deque cimport deque
from libcpp.vector cimport vector
from cpython cimport bool as py_bool
from libcpp cimport bool as cpp_bool
from libc.stdint cimport uint32_t
from cython.operator cimport dereference as deref
from cpython.bytes cimport PyBytes_AsString
from cpython.bytes cimport PyBytes_Size
from cpython.bytes cimport PyBytes_FromString
from cpython.bytes cimport PyBytes_FromStringAndSize
from cpython.unicode cimport PyUnicode_Decode
from std_memory cimport shared_ptr
cimport options
cimport merge_operator
cimport filter_policy
cimport comparator
cimport slice_transform
cimport cache
cimport logger
cimport snapshot
cimport db
cimport iterator
cimport backup
cimport env
cimport table_factory
cimport memtablerep
cimport universal_compaction
# Enums are the only exception for direct imports
# Their name als already unique enough
from universal_compaction cimport kCompactionStopStyleSimilarSize
from universal_compaction cimport kCompactionStopStyleTotalSize
from options cimport kCompactionStyleLevel
from options cimport kCompactionStyleUniversal
from slice_ cimport Slice
from status cimport Status
import sys
from interfaces import MergeOperator as IMergeOperator
from interfaces import AssociativeMergeOperator as IAssociativeMergeOperator
from interfaces import FilterPolicy as IFilterPolicy
from interfaces import Comparator as IComparator
from interfaces import SliceTransform as ISliceTransform
import traceback
import errors
ctypedef const filter_policy.FilterPolicy ConstFilterPolicy
cdef extern from "cpp/utils.hpp" namespace "py_rocks":
cdef const Slice* vector_data(vector[Slice]&)
# Prepare python for threaded usage.
# Python callbacks (merge, comparator)
# could be executed in a rocksdb background thread (eg. compaction).
cdef extern from "Python.h":
void PyEval_InitThreads()
PyEval_InitThreads()
## Here comes the stuff to wrap the status to exception
cdef check_status(const Status& st):
if st.ok():
return
if st.IsNotFound():
raise errors.NotFound(st.ToString())
if st.IsCorruption():
raise errors.Corruption(st.ToString())
if st.IsNotSupported():
raise errors.NotSupported(st.ToString())
if st.IsInvalidArgument():
raise errors.InvalidArgument(st.ToString())
if st.IsIOError():
raise errors.RocksIOError(st.ToString())
if st.IsMergeInProgress():
raise errors.MergeInProgress(st.ToString())
if st.IsIncomplete():
raise errors.Incomplete(st.ToString())
raise Exception("Unknown error: %s" % st.ToString())
######################################################
cdef string bytes_to_string(path) except *:
return string(PyBytes_AsString(path), PyBytes_Size(path))
cdef string_to_bytes(string ob):
return PyBytes_FromStringAndSize(ob.c_str(), ob.size())
cdef Slice bytes_to_slice(ob) except *:
return Slice(PyBytes_AsString(ob), PyBytes_Size(ob))
cdef slice_to_bytes(Slice sl):
return PyBytes_FromStringAndSize(sl.data(), sl.size())
## only for filsystem paths
cdef string path_to_string(object path) except *:
if isinstance(path, bytes):
return bytes_to_string(path)
if isinstance(path, unicode):
path = path.encode(sys.getfilesystemencoding())
return bytes_to_string(path)
else:
raise TypeError("Wrong type for path: %s" % path)
cdef object string_to_path(string path):
fs_encoding = sys.getfilesystemencoding().encode('ascii')
return PyUnicode_Decode(path.c_str(), path.size(), fs_encoding, "replace")
## Here comes the stuff for the comparator
@cython.internal
cdef class PyComparator(object):
cdef object get_ob(self):
return None
cdef const comparator.Comparator* get_comparator(self):
return NULL
cdef set_info_log(self, shared_ptr[logger.Logger] info_log):
pass
@cython.internal
cdef class PyGenericComparator(PyComparator):
cdef comparator.ComparatorWrapper* comparator_ptr
cdef object ob
def __cinit__(self, object ob):
self.comparator_ptr = NULL
if not isinstance(ob, IComparator):
raise TypeError("%s is not of type %s" % (ob, IComparator))
self.ob = ob
self.comparator_ptr = new comparator.ComparatorWrapper(
bytes_to_string(ob.name()),
<void*>ob,
compare_callback)
def __dealloc__(self):
if not self.comparator_ptr == NULL:
del self.comparator_ptr
cdef object get_ob(self):
return self.ob
cdef const comparator.Comparator* get_comparator(self):
return <comparator.Comparator*> self.comparator_ptr
cdef set_info_log(self, shared_ptr[logger.Logger] info_log):
self.comparator_ptr.set_info_log(info_log)
@cython.internal
cdef class PyBytewiseComparator(PyComparator):
cdef const comparator.Comparator* comparator_ptr
def __cinit__(self):
self.comparator_ptr = comparator.BytewiseComparator()
def name(self):
return PyBytes_FromString(self.comparator_ptr.Name())
def compare(self, a, b):
return self.comparator_ptr.Compare(
bytes_to_slice(a),
bytes_to_slice(b))
cdef object get_ob(self):
return self
cdef const comparator.Comparator* get_comparator(self):
return self.comparator_ptr
cdef int compare_callback(
void* ctx,
logger.Logger* log,
string& error_msg,
const Slice& a,
const Slice& b) with gil:
try:
return (<object>ctx).compare(slice_to_bytes(a), slice_to_bytes(b))
except BaseException as error:
tb = traceback.format_exc()
logger.Log(log, "Error in compare callback: %s", <bytes>tb)
error_msg.assign(<bytes>str(error))
BytewiseComparator = PyBytewiseComparator
#########################################
## Here comes the stuff for the filter policy
@cython.internal
cdef class PyFilterPolicy(object):
cdef object get_ob(self):
return None
cdef shared_ptr[ConstFilterPolicy] get_policy(self):
return shared_ptr[ConstFilterPolicy]()
cdef set_info_log(self, shared_ptr[logger.Logger] info_log):
pass
@cython.internal
cdef class PyGenericFilterPolicy(PyFilterPolicy):
cdef shared_ptr[filter_policy.FilterPolicyWrapper] policy
cdef object ob
def __cinit__(self, object ob):
if not isinstance(ob, IFilterPolicy):
raise TypeError("%s is not of type %s" % (ob, IFilterPolicy))
self.ob = ob
self.policy.reset(new filter_policy.FilterPolicyWrapper(
bytes_to_string(ob.name()),
<void*>ob,
create_filter_callback,
key_may_match_callback))
cdef object get_ob(self):
return self.ob
cdef shared_ptr[ConstFilterPolicy] get_policy(self):
return <shared_ptr[ConstFilterPolicy]>(self.policy)
cdef set_info_log(self, shared_ptr[logger.Logger] info_log):
self.policy.get().set_info_log(info_log)
cdef void create_filter_callback(
void* ctx,
logger.Logger* log,
string& error_msg,
const Slice* keys,
int n,
string* dst) with gil:
try:
ret = (<object>ctx).create_filter(
[slice_to_bytes(keys[i]) for i in range(n)])
dst.append(bytes_to_string(ret))
except BaseException as error:
tb = traceback.format_exc()
logger.Log(log, "Error in create filter callback: %s", <bytes>tb)
error_msg.assign(<bytes>str(error))
cdef cpp_bool key_may_match_callback(
void* ctx,
logger.Logger* log,
string& error_msg,
const Slice& key,
const Slice& filt) with gil:
try:
return (<object>ctx).key_may_match(
slice_to_bytes(key),
slice_to_bytes(filt))
except BaseException as error:
tb = traceback.format_exc()
logger.Log(log, "Error in key_mach_match callback: %s", <bytes>tb)
error_msg.assign(<bytes>str(error))
@cython.internal
cdef class PyBloomFilterPolicy(PyFilterPolicy):
cdef shared_ptr[ConstFilterPolicy] policy
def __cinit__(self, int bits_per_key):
self.policy.reset(filter_policy.NewBloomFilterPolicy(bits_per_key))
def name(self):
return PyBytes_FromString(self.policy.get().Name())
def create_filter(self, keys):
cdef string dst
cdef vector[Slice] c_keys
for key in keys:
c_keys.push_back(bytes_to_slice(key))
self.policy.get().CreateFilter(
vector_data(c_keys),
c_keys.size(),
cython.address(dst))
return string_to_bytes(dst)
def key_may_match(self, key, filter_):
return self.policy.get().KeyMayMatch(
bytes_to_slice(key),
bytes_to_slice(filter_))
cdef object get_ob(self):
return self
cdef shared_ptr[ConstFilterPolicy] get_policy(self):
return self.policy
BloomFilterPolicy = PyBloomFilterPolicy
#############################################
## Here comes the stuff for the merge operator
@cython.internal
cdef class PyMergeOperator(object):
cdef shared_ptr[merge_operator.MergeOperator] merge_op
cdef object ob
def __cinit__(self, object ob):
if isinstance(ob, IAssociativeMergeOperator):
self.ob = ob
self.merge_op.reset(
<merge_operator.MergeOperator*>
new merge_operator.AssociativeMergeOperatorWrapper(
bytes_to_string(ob.name()),
<void*>(ob),
merge_callback))
elif isinstance(ob, IMergeOperator):
self.ob = ob
self.merge_op.reset(
<merge_operator.MergeOperator*>
new merge_operator.MergeOperatorWrapper(
bytes_to_string(ob.name()),
<void*>ob,
<void*>ob,
full_merge_callback,
partial_merge_callback))
else:
msg = "%s is not of this types %s"
msg %= (ob, (IAssociativeMergeOperator, IMergeOperator))
raise TypeError(msg)
cdef object get_ob(self):
return self.ob
cdef shared_ptr[merge_operator.MergeOperator] get_operator(self):
return self.merge_op
cdef cpp_bool merge_callback(
void* ctx,
const Slice& key,
const Slice* existing_value,
const Slice& value,
string* new_value,
logger.Logger* log) with gil:
if existing_value == NULL:
py_existing_value = None
else:
py_existing_value = slice_to_bytes(deref(existing_value))
try:
ret = (<object>ctx).merge(
slice_to_bytes(key),
py_existing_value,
slice_to_bytes(value))
if ret[0]:
new_value.assign(bytes_to_string(ret[1]))
return True
return False
except:
tb = traceback.format_exc()
logger.Log(log, "Error in merge_callback: %s", <bytes>tb)
return False
cdef cpp_bool full_merge_callback(
void* ctx,
const Slice& key,
const Slice* existing_value,
const deque[string]& op_list,
string* new_value,
logger.Logger* log) with gil:
if existing_value == NULL:
py_existing_value = None
else:
py_existing_value = slice_to_bytes(deref(existing_value))
try:
ret = (<object>ctx).full_merge(
slice_to_bytes(key),
py_existing_value,
[string_to_bytes(op_list[i]) for i in range(op_list.size())])
if ret[0]:
new_value.assign(bytes_to_string(ret[1]))
return True
return False
except:
tb = traceback.format_exc()
logger.Log(log, "Error in full_merge_callback: %s", <bytes>tb)
return False
cdef cpp_bool partial_merge_callback(
void* ctx,
const Slice& key,
const Slice& left_op,
const Slice& right_op,
string* new_value,
logger.Logger* log) with gil:
try:
ret = (<object>ctx).partial_merge(
slice_to_bytes(key),
slice_to_bytes(left_op),
slice_to_bytes(right_op))
if ret[0]:
new_value.assign(bytes_to_string(ret[1]))
return True
return False
except:
tb = traceback.format_exc()
logger.Log(log, "Error in partial_merge_callback: %s", <bytes>tb)
return False
##############################################
#### Here comes the Cache stuff
@cython.internal
cdef class PyCache(object):
cdef shared_ptr[cache.Cache] get_cache(self):
return shared_ptr[cache.Cache]()
@cython.internal
cdef class PyLRUCache(PyCache):
cdef shared_ptr[cache.Cache] cache_ob
def __cinit__(self, capacity, shard_bits=None):
if shard_bits is not None:
self.cache_ob = cache.NewLRUCache(capacity, shard_bits)
else:
self.cache_ob = cache.NewLRUCache(capacity)
cdef shared_ptr[cache.Cache] get_cache(self):
return self.cache_ob
LRUCache = PyLRUCache
###############################
### Here comes the stuff for SliceTransform
@cython.internal
cdef class PySliceTransform(object):
cdef shared_ptr[slice_transform.SliceTransform] transfomer
cdef object ob
def __cinit__(self, object ob):
if not isinstance(ob, ISliceTransform):
raise TypeError("%s is not of type %s" % (ob, ISliceTransform))
self.ob = ob
self.transfomer.reset(
<slice_transform.SliceTransform*>
new slice_transform.SliceTransformWrapper(
bytes_to_string(ob.name()),
<void*>ob,
slice_transform_callback,
slice_in_domain_callback,
slice_in_range_callback))
cdef object get_ob(self):
return self.ob
cdef shared_ptr[slice_transform.SliceTransform] get_transformer(self):
return self.transfomer
cdef set_info_log(self, shared_ptr[logger.Logger] info_log):
cdef slice_transform.SliceTransformWrapper* ptr
ptr = <slice_transform.SliceTransformWrapper*> self.transfomer.get()
ptr.set_info_log(info_log)
cdef Slice slice_transform_callback(
void* ctx,
logger.Logger* log,
string& error_msg,
const Slice& src) with gil:
cdef size_t offset
cdef size_t size
try:
ret = (<object>ctx).transform(slice_to_bytes(src))
offset = ret[0]
size = ret[1]
if (offset + size) > src.size():
msg = "offset(%i) + size(%i) is bigger than slice(%i)"
raise Exception(msg % (offset, size, src.size()))
return Slice(src.data() + offset, size)
except BaseException as error:
tb = traceback.format_exc()
logger.Log(log, "Error in slice transfrom callback: %s", <bytes>tb)
error_msg.assign(<bytes>str(error))
cdef cpp_bool slice_in_domain_callback(
void* ctx,
logger.Logger* log,
string& error_msg,
const Slice& src) with gil:
try:
return (<object>ctx).in_domain(slice_to_bytes(src))
except BaseException as error:
tb = traceback.format_exc()
logger.Log(log, "Error in slice transfrom callback: %s", <bytes>tb)
error_msg.assign(<bytes>str(error))
cdef cpp_bool slice_in_range_callback(
void* ctx,
logger.Logger* log,
string& error_msg,
const Slice& src) with gil:
try:
return (<object>ctx).in_range(slice_to_bytes(src))
except BaseException as error:
tb = traceback.format_exc()
logger.Log(log, "Error in slice transfrom callback: %s", <bytes>tb)
error_msg.assign(<bytes>str(error))
###########################################
## Here are the TableFactories
@cython.internal
cdef class PyTableFactory(object):
cdef shared_ptr[table_factory.TableFactory] factory
cdef shared_ptr[table_factory.TableFactory] get_table_factory(self):
return self.factory
cdef set_info_log(self, shared_ptr[logger.Logger] info_log):
pass
cdef class BlockBasedTableFactory(PyTableFactory):
cdef PyFilterPolicy py_filter_policy
def __init__(self,
index_type='binary_search',
py_bool hash_index_allow_collision=True,
checksum='crc32',
PyCache block_cache=None,
PyCache block_cache_compressed=None,
filter_policy=None,
no_block_cache=False,
block_size=None,
block_size_deviation=None,
block_restart_interval=None,
whole_key_filtering=None):
cdef table_factory.BlockBasedTableOptions table_options
if index_type == 'binary_search':
table_options.index_type = table_factory.kBinarySearch
elif index_type == 'hash_search':
table_options.index_type = table_factory.kHashSearch
else:
raise ValueError("Unknown index_type: %s" % index_type)
if hash_index_allow_collision:
table_options.hash_index_allow_collision = True
else:
table_options.hash_index_allow_collision = False
if checksum == 'crc32':
table_options.checksum = table_factory.kCRC32c
elif checksum == 'xxhash':
table_options.checksum = table_factory.kxxHash
else:
raise ValueError("Unknown checksum: %s" % checksum)
if no_block_cache:
table_options.no_block_cache = True
else:
table_options.no_block_cache = False
# If the following options are None use the rocksdb default.
if block_size is not None:
table_options.block_size = block_size
if block_size_deviation is not None:
table_options.block_size_deviation = block_size_deviation
if block_restart_interval is not None:
table_options.block_restart_interval = block_restart_interval
if whole_key_filtering is not None:
if whole_key_filtering:
table_options.whole_key_filtering = True
else:
table_options.whole_key_filtering = False
if block_cache is not None:
table_options.block_cache = block_cache.get_cache()
if block_cache_compressed is not None:
table_options.block_cache_compressed = block_cache_compressed.get_cache()
# Set the filter_policy
self.py_filter_policy = None
if filter_policy is not None:
if isinstance(filter_policy, PyFilterPolicy):
if (<PyFilterPolicy?>filter_policy).get_policy().get() == NULL:
raise Exception("Cannot set filter policy: %s" % filter_policy)
self.py_filter_policy = filter_policy
else:
self.py_filter_policy = PyGenericFilterPolicy(filter_policy)
table_options.filter_policy = self.py_filter_policy.get_policy()
self.factory.reset(table_factory.NewBlockBasedTableFactory(table_options))
cdef set_info_log(self, shared_ptr[logger.Logger] info_log):
if self.py_filter_policy is not None:
self.py_filter_policy.set_info_log(info_log)
cdef class PlainTableFactory(PyTableFactory):
def __init__(
self,
user_key_len=0,
bloom_bits_per_key=10,
hash_table_ratio=0.75,
index_sparseness=10,
huge_page_tlb_size=0,
encoding_type='plain',
py_bool full_scan_mode=False,
py_bool store_index_in_file=False):
cdef table_factory.PlainTableOptions table_options
table_options.user_key_len = user_key_len
table_options.bloom_bits_per_key = bloom_bits_per_key
table_options.hash_table_ratio = hash_table_ratio
table_options.index_sparseness = index_sparseness
table_options.huge_page_tlb_size = huge_page_tlb_size
if encoding_type == 'plain':
table_options.encoding_type = table_factory.kPlain
elif encoding_type == 'prefix':
table_options.encoding_type = table_factory.kPrefix
else:
raise ValueError("Unknown encoding_type: %s" % encoding_type)
table_options.full_scan_mode = full_scan_mode
table_options.store_index_in_file = store_index_in_file
self.factory.reset( table_factory.NewPlainTableFactory(table_options))
#############################################
### Here are the MemtableFactories
@cython.internal
cdef class PyMemtableFactory(object):
cdef shared_ptr[memtablerep.MemTableRepFactory] factory
cdef shared_ptr[memtablerep.MemTableRepFactory] get_memtable_factory(self):
return self.factory
cdef class SkipListMemtableFactory(PyMemtableFactory):
def __init__(self):
self.factory.reset(memtablerep.NewSkipListFactory())
cdef class VectorMemtableFactory(PyMemtableFactory):
def __init__(self, count=0):
self.factory.reset(memtablerep.NewVectorRepFactory(count))
cdef class HashSkipListMemtableFactory(PyMemtableFactory):
def __init__(
self,
bucket_count=1000000,
skiplist_height=4,
skiplist_branching_factor=4):
self.factory.reset(
memtablerep.NewHashSkipListRepFactory(
bucket_count,
skiplist_height,
skiplist_branching_factor))
cdef class HashLinkListMemtableFactory(PyMemtableFactory):
def __init__(self, bucket_count=50000):
self.factory.reset(memtablerep.NewHashLinkListRepFactory(bucket_count))
##################################
cdef class CompressionType(object):
no_compression = u'no_compression'
snappy_compression = u'snappy_compression'
zlib_compression = u'zlib_compression'
bzip2_compression = u'bzip2_compression'
lz4_compression = u'lz4_compression'
lz4hc_compression = u'lz4hc_compression'
cdef class Options(object):
cdef options.Options* opts
cdef PyComparator py_comparator
cdef PyMergeOperator py_merge_operator
cdef PySliceTransform py_prefix_extractor
cdef PyTableFactory py_table_factory
cdef PyMemtableFactory py_memtable_factory
cdef PyCache py_row_cache
# Used to protect sharing of Options with many DB-objects
cdef cpp_bool in_use
def __cinit__(self):
self.opts = NULL
self.opts = new options.Options()
self.in_use = False
def __dealloc__(self):
if not self.opts == NULL:
del self.opts
def __init__(self, **kwargs):
self.py_comparator = BytewiseComparator()
self.py_merge_operator = None
self.py_prefix_extractor = None
self.py_table_factory = None
self.py_memtable_factory = None
self.py_row_cache = None
for key, value in kwargs.items():
setattr(self, key, value)
property create_if_missing:
def __get__(self):
return self.opts.create_if_missing
def __set__(self, value):
self.opts.create_if_missing = value
property error_if_exists:
def __get__(self):
return self.opts.error_if_exists
def __set__(self, value):
self.opts.error_if_exists = value
property paranoid_checks:
def __get__(self):
return self.opts.paranoid_checks
def __set__(self, value):
self.opts.paranoid_checks = value
property write_buffer_size:
def __get__(self):
return self.opts.write_buffer_size
def __set__(self, value):
self.opts.write_buffer_size = value
property max_write_buffer_number:
def __get__(self):
return self.opts.max_write_buffer_number
def __set__(self, value):
self.opts.max_write_buffer_number = value
property min_write_buffer_number_to_merge:
def __get__(self):
return self.opts.min_write_buffer_number_to_merge
def __set__(self, value):
self.opts.min_write_buffer_number_to_merge = value
property max_open_files:
def __get__(self):
return self.opts.max_open_files
def __set__(self, value):
self.opts.max_open_files = value
property compression:
def __get__(self):
if self.opts.compression == options.kNoCompression:
return CompressionType.no_compression
elif self.opts.compression == options.kSnappyCompression:
return CompressionType.snappy_compression
elif self.opts.compression == options.kZlibCompression:
return CompressionType.zlib_compression
elif self.opts.compression == options.kBZip2Compression:
return CompressionType.bzip2_compression
elif self.opts.compression == options.kLZ4Compression:
return CompressionType.lz4_compression
elif self.opts.compression == options.kLZ4HCCompression:
return CompressionType.lz4hc_compression
else:
raise Exception("Unknonw type: %s" % self.opts.compression)
def __set__(self, value):
if value == CompressionType.no_compression:
self.opts.compression = options.kNoCompression
elif value == CompressionType.snappy_compression:
self.opts.compression = options.kSnappyCompression
elif value == CompressionType.zlib_compression:
self.opts.compression = options.kZlibCompression
elif value == CompressionType.bzip2_compression:
self.opts.compression = options.kBZip2Compression
elif value == CompressionType.lz4_compression:
self.opts.compression = options.kLZ4Compression
elif value == CompressionType.lz4hc_compression:
self.opts.compression = options.kLZ4HCCompression
else:
raise TypeError("Unknown compression: %s" % value)
property num_levels:
def __get__(self):
return self.opts.num_levels
def __set__(self, value):
self.opts.num_levels = value
property level0_file_num_compaction_trigger:
def __get__(self):
return self.opts.level0_file_num_compaction_trigger
def __set__(self, value):
self.opts.level0_file_num_compaction_trigger = value
property level0_slowdown_writes_trigger:
def __get__(self):
return self.opts.level0_slowdown_writes_trigger
def __set__(self, value):
self.opts.level0_slowdown_writes_trigger = value
property level0_stop_writes_trigger:
def __get__(self):
return self.opts.level0_stop_writes_trigger
def __set__(self, value):
self.opts.level0_stop_writes_trigger = value
property max_mem_compaction_level:
def __get__(self):
return self.opts.max_mem_compaction_level
def __set__(self, value):
self.opts.max_mem_compaction_level = value
property target_file_size_base:
def __get__(self):
return self.opts.target_file_size_base
def __set__(self, value):
self.opts.target_file_size_base = value
property target_file_size_multiplier:
def __get__(self):
return self.opts.target_file_size_multiplier
def __set__(self, value):
self.opts.target_file_size_multiplier = value
property max_bytes_for_level_base:
def __get__(self):
return self.opts.max_bytes_for_level_base
def __set__(self, value):
self.opts.max_bytes_for_level_base = value
property max_bytes_for_level_multiplier:
def __get__(self):
return self.opts.max_bytes_for_level_multiplier
def __set__(self, value):
self.opts.max_bytes_for_level_multiplier = value
property max_bytes_for_level_multiplier_additional:
def __get__(self):
return self.opts.max_bytes_for_level_multiplier_additional
def __set__(self, value):
self.opts.max_bytes_for_level_multiplier_additional = value
property expanded_compaction_factor:
def __get__(self):
return self.opts.expanded_compaction_factor
def __set__(self, value):
self.opts.expanded_compaction_factor = value
property source_compaction_factor:
def __get__(self):
return self.opts.source_compaction_factor
def __set__(self, value):
self.opts.source_compaction_factor = value
property max_grandparent_overlap_factor:
def __get__(self):
return self.opts.max_grandparent_overlap_factor
def __set__(self, value):
self.opts.max_grandparent_overlap_factor = value
property disable_data_sync:
def __get__(self):
return self.opts.disableDataSync
def __set__(self, value):
self.opts.disableDataSync = value
property use_fsync:
def __get__(self):
return self.opts.use_fsync
def __set__(self, value):
self.opts.use_fsync = value
property db_log_dir:
def __get__(self):
return string_to_path(self.opts.db_log_dir)
def __set__(self, value):
self.opts.db_log_dir = path_to_string(value)
property wal_dir:
def __get__(self):
return string_to_path(self.opts.wal_dir)
def __set__(self, value):
self.opts.wal_dir = path_to_string(value)
property delete_obsolete_files_period_micros:
def __get__(self):
return self.opts.delete_obsolete_files_period_micros
def __set__(self, value):
self.opts.delete_obsolete_files_period_micros = value
property max_background_compactions:
def __get__(self):
return self.opts.max_background_compactions
def __set__(self, value):
self.opts.max_background_compactions = value
property max_background_flushes:
def __get__(self):
return self.opts.max_background_flushes
def __set__(self, value):
self.opts.max_background_flushes = value
property max_log_file_size:
def __get__(self):
return self.opts.max_log_file_size
def __set__(self, value):
self.opts.max_log_file_size = value
property log_file_time_to_roll:
def __get__(self):
return self.opts.log_file_time_to_roll
def __set__(self, value):
self.opts.log_file_time_to_roll = value
property keep_log_file_num:
def __get__(self):
return self.opts.keep_log_file_num
def __set__(self, value):
self.opts.keep_log_file_num = value
property soft_rate_limit:
def __get__(self):
return self.opts.soft_rate_limit
def __set__(self, value):
self.opts.soft_rate_limit = value
property hard_rate_limit:
def __get__(self):
return self.opts.hard_rate_limit
def __set__(self, value):
self.opts.hard_rate_limit = value
property rate_limit_delay_max_milliseconds:
def __get__(self):
return self.opts.rate_limit_delay_max_milliseconds
def __set__(self, value):
self.opts.rate_limit_delay_max_milliseconds = value
property max_manifest_file_size:
def __get__(self):
return self.opts.max_manifest_file_size
def __set__(self, value):
self.opts.max_manifest_file_size = value
property table_cache_numshardbits:
def __get__(self):
return self.opts.table_cache_numshardbits
def __set__(self, value):
self.opts.table_cache_numshardbits = value
property arena_block_size:
def __get__(self):
return self.opts.arena_block_size
def __set__(self, value):
self.opts.arena_block_size = value
property disable_auto_compactions:
def __get__(self):
return self.opts.disable_auto_compactions
def __set__(self, value):
self.opts.disable_auto_compactions = value
property wal_ttl_seconds:
def __get__(self):
return self.opts.WAL_ttl_seconds
def __set__(self, value):
self.opts.WAL_ttl_seconds = value
property wal_size_limit_mb:
def __get__(self):
return self.opts.WAL_size_limit_MB
def __set__(self, value):
self.opts.WAL_size_limit_MB = value
property manifest_preallocation_size:
def __get__(self):
return self.opts.manifest_preallocation_size
def __set__(self, value):
self.opts.manifest_preallocation_size = value
property purge_redundant_kvs_while_flush:
def __get__(self):
return self.opts.purge_redundant_kvs_while_flush
def __set__(self, value):
self.opts.purge_redundant_kvs_while_flush = value
property allow_os_buffer:
def __get__(self):
return self.opts.allow_os_buffer
def __set__(self, value):
self.opts.allow_os_buffer = value
property allow_mmap_reads:
def __get__(self):
return self.opts.allow_mmap_reads
def __set__(self, value):
self.opts.allow_mmap_reads = value
property allow_mmap_writes:
def __get__(self):
return self.opts.allow_mmap_writes
def __set__(self, value):
self.opts.allow_mmap_writes = value
property is_fd_close_on_exec:
def __get__(self):
return self.opts.is_fd_close_on_exec
def __set__(self, value):
self.opts.is_fd_close_on_exec = value
property skip_log_error_on_recovery:
def __get__(self):
return self.opts.skip_log_error_on_recovery
def __set__(self, value):
self.opts.skip_log_error_on_recovery = value
property stats_dump_period_sec:
def __get__(self):
return self.opts.stats_dump_period_sec
def __set__(self, value):
self.opts.stats_dump_period_sec = value
property advise_random_on_open:
def __get__(self):
return self.opts.advise_random_on_open
def __set__(self, value):
self.opts.advise_random_on_open = value
property use_adaptive_mutex:
def __get__(self):
return self.opts.use_adaptive_mutex
def __set__(self, value):
self.opts.use_adaptive_mutex = value
property bytes_per_sync:
def __get__(self):
return self.opts.bytes_per_sync
def __set__(self, value):
self.opts.bytes_per_sync = value
property verify_checksums_in_compaction:
def __get__(self):
return self.opts.verify_checksums_in_compaction
def __set__(self, value):
self.opts.verify_checksums_in_compaction = value
property compaction_style:
def __get__(self):
if self.opts.compaction_style == kCompactionStyleLevel:
return 'level'
if self.opts.compaction_style == kCompactionStyleUniversal:
return 'universal'
raise Exception("Unknown compaction_style")
def __set__(self, str value):
if value == 'level':
self.opts.compaction_style = kCompactionStyleLevel
elif value == 'universal':
self.opts.compaction_style = kCompactionStyleUniversal
else:
raise Exception("Unknown compaction style")
property compaction_options_universal:
def __get__(self):
cdef universal_compaction.CompactionOptionsUniversal uopts
cdef dict ret_ob = {}
uopts = self.opts.compaction_options_universal
ret_ob['size_ratio'] = uopts.size_ratio
ret_ob['min_merge_width'] = uopts.min_merge_width
ret_ob['max_merge_width'] = uopts.max_merge_width
ret_ob['max_size_amplification_percent'] = uopts.max_size_amplification_percent
ret_ob['compression_size_percent'] = uopts.compression_size_percent
if uopts.stop_style == kCompactionStopStyleSimilarSize:
ret_ob['stop_style'] = 'similar_size'
elif uopts.stop_style == kCompactionStopStyleTotalSize:
ret_ob['stop_style'] = 'total_size'
else:
raise Exception("Unknown compaction style")
return ret_ob
def __set__(self, dict value):
cdef universal_compaction.CompactionOptionsUniversal* uopts
uopts = cython.address(self.opts.compaction_options_universal)
if 'size_ratio' in value:
uopts.size_ratio = value['size_ratio']
if 'min_merge_width' in value:
uopts.min_merge_width = value['min_merge_width']
if 'max_merge_width' in value:
uopts.max_merge_width = value['max_merge_width']
if 'max_size_amplification_percent' in value:
uopts.max_size_amplification_percent = value['max_size_amplification_percent']
if 'compression_size_percent' in value:
uopts.compression_size_percent = value['compression_size_percent']
if 'stop_style' in value:
if value['stop_style'] == 'similar_size':
uopts.stop_style = kCompactionStopStyleSimilarSize
elif value['stop_style'] == 'total_size':
uopts.stop_style = kCompactionStopStyleTotalSize
else:
raise Exception("Unknown compaction style")
property filter_deletes:
def __get__(self):
return self.opts.filter_deletes
def __set__(self, value):
self.opts.filter_deletes = value
property max_sequential_skip_in_iterations:
def __get__(self):
return self.opts.max_sequential_skip_in_iterations
def __set__(self, value):
self.opts.max_sequential_skip_in_iterations = value
property inplace_update_support:
def __get__(self):
return self.opts.inplace_update_support
def __set__(self, value):
self.opts.inplace_update_support = value
property table_factory:
def __get__(self):
return self.py_table_factory
def __set__(self, PyTableFactory value):
self.py_table_factory = value
self.opts.table_factory = value.get_table_factory()
property memtable_factory:
def __get__(self):
return self.py_memtable_factory
def __set__(self, PyMemtableFactory value):
self.py_memtable_factory = value
self.opts.memtable_factory = value.get_memtable_factory()
property inplace_update_num_locks:
def __get__(self):
return self.opts.inplace_update_num_locks
def __set__(self, value):
self.opts.inplace_update_num_locks = value
property comparator:
def __get__(self):
return self.py_comparator.get_ob()
def __set__(self, value):
if isinstance(value, PyComparator):
if (<PyComparator?>value).get_comparator() == NULL:
raise Exception("Cannot set %s as comparator" % value)
else:
self.py_comparator = value
else:
self.py_comparator = PyGenericComparator(value)
self.opts.comparator = self.py_comparator.get_comparator()
property merge_operator:
def __get__(self):
if self.py_merge_operator is None:
return None
return self.py_merge_operator.get_ob()
def __set__(self, value):
self.py_merge_operator = PyMergeOperator(value)
self.opts.merge_operator = self.py_merge_operator.get_operator()
property prefix_extractor:
def __get__(self):
if self.py_prefix_extractor is None:
return None
return self.py_prefix_extractor.get_ob()
def __set__(self, value):
self.py_prefix_extractor = PySliceTransform(value)
self.opts.prefix_extractor = self.py_prefix_extractor.get_transformer()
property row_cache:
def __get__(self):
return self.py_row_cache
def __set__(self, value):
if value is None:
self.py_row_cache = None
self.opts.row_cache.reset()
elif not isinstance(value, PyCache):
raise Exception("row_cache must be a Cache object")
else:
self.py_row_cache = value
self.opts.row_cache = self.py_row_cache.get_cache()
# Forward declaration
cdef class Snapshot
cdef class KeysIterator
cdef class ValuesIterator
cdef class ItemsIterator
cdef class ReversedIterator
# Forward declaration
cdef class WriteBatchIterator
cdef class WriteBatch(object):
cdef db.WriteBatch* batch
def __cinit__(self, data=None):
self.batch = NULL
if data is not None:
self.batch = new db.WriteBatch(bytes_to_string(data))
else:
self.batch = new db.WriteBatch()
def __dealloc__(self):
if not self.batch == NULL:
del self.batch
def put(self, key, value):
self.batch.Put(bytes_to_slice(key), bytes_to_slice(value))
def merge(self, key, value):
self.batch.Merge(bytes_to_slice(key), bytes_to_slice(value))
def delete(self, key):
self.batch.Delete(bytes_to_slice(key))
def clear(self):
self.batch.Clear()
def data(self):
return string_to_bytes(self.batch.Data())
def count(self):
return self.batch.Count()
def __iter__(self):
return WriteBatchIterator(self)
@cython.internal
cdef class WriteBatchIterator(object):
# Need a reference to the WriteBatch.
# The BatchItems are only pointers to the memory in WriteBatch.
cdef WriteBatch batch
cdef vector[db.BatchItem] items
cdef size_t pos
def __init__(self, WriteBatch batch):
cdef Status st
self.batch = batch
self.pos = 0
st = db.get_batch_items(batch.batch, cython.address(self.items))
check_status(st)
def __iter__(self):
return self
def __next__(self):
if self.pos == self.items.size():
raise StopIteration()
cdef str op
if self.items[self.pos].op == db.BatchItemOpPut:
op = "Put"
elif self.items[self.pos].op == db.BatchItemOpMerge:
op = "Merge"
elif self.items[self.pos].op == db.BatchItemOpDelte:
op = "Delete"
ret = (
op,
slice_to_bytes(self.items[self.pos].key),
slice_to_bytes(self.items[self.pos].value))
self.pos += 1
return ret
@cython.no_gc_clear
cdef class DB(object):
cdef Options opts
cdef db.DB* db
def __cinit__(self, db_name, Options opts, read_only=False):
cdef Status st
cdef string db_path
self.db = NULL
self.opts = None
if opts.in_use:
raise Exception("Options object is already used by another DB")
db_path = path_to_string(db_name)
if read_only:
with nogil:
st = db.DB_OpenForReadOnly(
deref(opts.opts),
db_path,
cython.address(self.db),
False)
else:
with nogil:
st = db.DB_Open(
deref(opts.opts),
db_path,
cython.address(self.db))
check_status(st)
# Inject the loggers into the python callbacks
cdef shared_ptr[logger.Logger] info_log = self.db.GetOptions().info_log
if opts.py_comparator is not None:
opts.py_comparator.set_info_log(info_log)
if opts.py_table_factory is not None:
opts.py_table_factory.set_info_log(info_log)
if opts.prefix_extractor is not None:
opts.py_prefix_extractor.set_info_log(info_log)
self.opts = opts
self.opts.in_use = True
def __dealloc__(self):
if not self.db == NULL:
with nogil:
del self.db
if self.opts is not None:
self.opts.in_use = False
def put(self, key, value, sync=False, disable_wal=False):
cdef Status st
cdef options.WriteOptions opts
opts.sync = sync
opts.disableWAL = disable_wal
cdef Slice c_key = bytes_to_slice(key)
cdef Slice c_value = bytes_to_slice(value)
with nogil:
st = self.db.Put(opts, c_key, c_value)
check_status(st)
def delete(self, key, sync=False, disable_wal=False):
cdef Status st
cdef options.WriteOptions opts
opts.sync = sync
opts.disableWAL = disable_wal
cdef Slice c_key = bytes_to_slice(key)
with nogil:
st = self.db.Delete(opts, c_key)
check_status(st)
def merge(self, key, value, sync=False, disable_wal=False):
cdef Status st
cdef options.WriteOptions opts
opts.sync = sync
opts.disableWAL = disable_wal
cdef Slice c_key = bytes_to_slice(key)
cdef Slice c_value = bytes_to_slice(value)
with nogil:
st = self.db.Merge(opts, c_key, c_value)
check_status(st)
def write(self, WriteBatch batch, sync=False, disable_wal=False):
cdef Status st
cdef options.WriteOptions opts
opts.sync = sync
opts.disableWAL = disable_wal
with nogil:
st = self.db.Write(opts, batch.batch)
check_status(st)
def get(self, key, *args, **kwargs):
cdef string res
cdef Status st
cdef options.ReadOptions opts
opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))
cdef Slice c_key = bytes_to_slice(key)
with nogil:
st = self.db.Get(opts, c_key, cython.address(res))
if st.ok():
return string_to_bytes(res)
elif st.IsNotFound():
return None
else:
check_status(st)
def multi_get(self, keys, *args, **kwargs):
cdef vector[string] values
values.resize(len(keys))
cdef vector[Slice] c_keys
for key in keys:
c_keys.push_back(bytes_to_slice(key))
cdef options.ReadOptions opts
opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))
cdef vector[Status] res
with nogil:
res = self.db.MultiGet(
opts,
c_keys,
cython.address(values))
cdef dict ret_dict = {}
for index in range(len(keys)):
if res[index].ok():
ret_dict[keys[index]] = string_to_bytes(values[index])
elif res[index].IsNotFound():
ret_dict[keys[index]] = None
else:
check_status(res[index])
return ret_dict
def key_may_exist(self, key, fetch=False, *args, **kwargs):
cdef string value
cdef cpp_bool value_found
cdef cpp_bool exists
cdef options.ReadOptions opts
cdef Slice c_key
opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))
c_key = bytes_to_slice(key)
exists = False
if fetch:
value_found = False
with nogil:
exists = self.db.KeyMayExist(
opts,
c_key,
cython.address(value),
cython.address(value_found))
if exists:
if value_found:
return (True, string_to_bytes(value))
else:
return (True, None)
else:
return (False, None)
else:
with nogil:
exists = self.db.KeyMayExist(
opts,
c_key,
cython.address(value))
return (exists, None)
def iterkeys(self, *args, **kwargs):
cdef options.ReadOptions opts
cdef KeysIterator it
opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))
it = KeysIterator(self)
with nogil:
it.ptr = self.db.NewIterator(opts)
return it
def itervalues(self, *args, **kwargs):
cdef options.ReadOptions opts
cdef ValuesIterator it
opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))
it = ValuesIterator(self)
with nogil:
it.ptr = self.db.NewIterator(opts)
return it
def iteritems(self, *args, **kwargs):
cdef options.ReadOptions opts
cdef ItemsIterator it
opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))
it = ItemsIterator(self)
with nogil:
it.ptr = self.db.NewIterator(opts)
return it
def snapshot(self):
return Snapshot(self)
def get_property(self, prop):
cdef string value
cdef Slice c_prop = bytes_to_slice(prop)
cdef cpp_bool ret = False
with nogil:
ret = self.db.GetProperty(c_prop, cython.address(value))
if ret:
return string_to_bytes(value)
else:
return None
def get_live_files_metadata(self):
cdef vector[db.LiveFileMetaData] metadata
with nogil:
self.db.GetLiveFilesMetaData(cython.address(metadata))
ret = []
for ob in metadata:
t = {}
t['name'] = string_to_path(ob.name)
t['level'] = ob.level
t['size'] = ob.size
t['smallestkey'] = string_to_bytes(ob.smallestkey)
t['largestkey'] = string_to_bytes(ob.largestkey)
t['smallest_seqno'] = ob.smallest_seqno
t['largest_seqno'] = ob.largest_seqno
ret.append(t)
return ret
def compact_range(self, begin=None, end=None, **py_options):
cdef options.CompactRangeOptions c_options
c_options.change_level = py_options.get('change_level', False)
c_options.target_level = py_options.get('target_level', -1)
blc = py_options.get('bottommost_level_compaction', 'if_compaction_filter')
if blc == 'skip':
c_options.bottommost_level_compaction = options.blc_skip
elif blc == 'if_compaction_filter':
c_options.bottommost_level_compaction = options.blc_is_filter
elif blc == 'force':
c_options.bottommost_level_compaction = options.blc_force
else:
raise ValueError("bottommost_level_compaction is not valid")
cdef Status st
cdef Slice begin_val
cdef Slice end_val
cdef Slice* begin_ptr
cdef Slice* end_ptr
begin_ptr = NULL
end_ptr = NULL
if begin is not None:
begin_val = bytes_to_slice(begin)
begin_ptr = cython.address(begin_val)
if end is not None:
end_val = bytes_to_slice(end)
end_ptr = cython.address(end_val)
st = self.db.CompactRange(c_options, begin_ptr, end_ptr)
check_status(st)
@staticmethod
def __parse_read_opts(
verify_checksums=False,
fill_cache=True,
snapshot=None,
read_tier="all"):
# TODO: Is this really effiencet ?
return locals()
cdef options.ReadOptions build_read_opts(self, dict py_opts):
cdef options.ReadOptions opts
opts.verify_checksums = py_opts['verify_checksums']
opts.fill_cache = py_opts['fill_cache']
if py_opts['snapshot'] is not None:
opts.snapshot = (<Snapshot?>(py_opts['snapshot'])).ptr
if py_opts['read_tier'] == "all":
opts.read_tier = options.kReadAllTier
elif py_opts['read_tier'] == 'cache':
opts.read_tier = options.kBlockCacheTier
else:
raise ValueError("Invalid read_tier")
return opts
property options:
def __get__(self):
return self.opts
def repair_db(db_name, Options opts):
cdef Status st
cdef string db_path
db_path = path_to_string(db_name)
st = db.RepairDB(db_path, deref(opts.opts))
check_status(st)
@cython.no_gc_clear
@cython.internal
cdef class Snapshot(object):
cdef const snapshot.Snapshot* ptr
cdef DB db
def __cinit__(self, DB db):
self.db = db
self.ptr = NULL
with nogil:
self.ptr = db.db.GetSnapshot()
def __dealloc__(self):
if not self.ptr == NULL:
with nogil:
self.db.db.ReleaseSnapshot(self.ptr)
@cython.internal
cdef class BaseIterator(object):
cdef iterator.Iterator* ptr
cdef DB db
def __cinit__(self, DB db):
self.db = db
self.ptr = NULL
def __dealloc__(self):
if not self.ptr == NULL:
del self.ptr
def __iter__(self):
return self
def __next__(self):
if not self.ptr.Valid():
raise StopIteration()
cdef object ret = self.get_ob()
with nogil:
self.ptr.Next()
check_status(self.ptr.status())
return ret
def __reversed__(self):
return ReversedIterator(self)
cpdef seek_to_first(self):
with nogil:
self.ptr.SeekToFirst()
check_status(self.ptr.status())
cpdef seek_to_last(self):
with nogil:
self.ptr.SeekToLast()
check_status(self.ptr.status())
cpdef seek(self, key):
cdef Slice c_key = bytes_to_slice(key)
with nogil:
self.ptr.Seek(c_key)
check_status(self.ptr.status())
cdef object get_ob(self):
return None
@cython.internal
cdef class KeysIterator(BaseIterator):
cdef object get_ob(self):
cdef Slice c_key
with nogil:
c_key = self.ptr.key()
check_status(self.ptr.status())
return slice_to_bytes(c_key)
@cython.internal
cdef class ValuesIterator(BaseIterator):
cdef object get_ob(self):
cdef Slice c_value
with nogil:
c_value = self.ptr.value()
check_status(self.ptr.status())
return slice_to_bytes(c_value)
@cython.internal
cdef class ItemsIterator(BaseIterator):
cdef object get_ob(self):
cdef Slice c_key
cdef Slice c_value
with nogil:
c_key = self.ptr.key()
c_value = self.ptr.value()
check_status(self.ptr.status())
return (slice_to_bytes(c_key), slice_to_bytes(c_value))
@cython.internal
cdef class ReversedIterator(object):
cdef BaseIterator it
def __cinit__(self, BaseIterator it):
self.it = it
def seek_to_first(self):
self.it.seek_to_first()
def seek_to_last(self):
self.it.seek_to_last()
def seek(self, key):
self.it.seek(key)
def __iter__(self):
return self
def __reversed__(self):
return self.it
def __next__(self):
if not self.it.ptr.Valid():
raise StopIteration()
cdef object ret = self.it.get_ob()
with nogil:
self.it.ptr.Prev()
check_status(self.it.ptr.status())
return ret
cdef class BackupEngine(object):
cdef backup.BackupEngine* engine
def __cinit__(self, backup_dir):
cdef Status st
cdef string c_backup_dir
self.engine = NULL
c_backup_dir = path_to_string(backup_dir)
st = backup.BackupEngine_Open(
env.Env_Default(),
backup.BackupableDBOptions(c_backup_dir),
cython.address(self.engine))
check_status(st)
def __dealloc__(self):
if not self.engine == NULL:
with nogil:
del self.engine
def create_backup(self, DB db, flush_before_backup=False):
cdef Status st
cdef cpp_bool c_flush_before_backup
c_flush_before_backup = flush_before_backup
with nogil:
st = self.engine.CreateNewBackup(db.db, c_flush_before_backup)
check_status(st)
def restore_backup(self, backup_id, db_dir, wal_dir):
cdef Status st
cdef backup.BackupID c_backup_id
cdef string c_db_dir
cdef string c_wal_dir
c_backup_id = backup_id
c_db_dir = path_to_string(db_dir)
c_wal_dir = path_to_string(wal_dir)
with nogil:
st = self.engine.RestoreDBFromBackup(
c_backup_id,
c_db_dir,
c_wal_dir)
check_status(st)
def restore_latest_backup(self, db_dir, wal_dir):
cdef Status st
cdef string c_db_dir
cdef string c_wal_dir
c_db_dir = path_to_string(db_dir)
c_wal_dir = path_to_string(wal_dir)
with nogil:
st = self.engine.RestoreDBFromLatestBackup(c_db_dir, c_wal_dir)
check_status(st)
def stop_backup(self):
with nogil:
self.engine.StopBackup()
def purge_old_backups(self, num_backups_to_keep):
cdef Status st
cdef uint32_t c_num_backups_to_keep
c_num_backups_to_keep = num_backups_to_keep
with nogil:
st = self.engine.PurgeOldBackups(c_num_backups_to_keep)
check_status(st)
def delete_backup(self, backup_id):
cdef Status st
cdef backup.BackupID c_backup_id
c_backup_id = backup_id
with nogil:
st = self.engine.DeleteBackup(c_backup_id)
check_status(st)
def get_backup_info(self):
cdef vector[backup.BackupInfo] backup_info
with nogil:
self.engine.GetBackupInfo(cython.address(backup_info))
ret = []
for ob in backup_info:
t = {}
t['backup_id'] = ob.backup_id
t['timestamp'] = ob.timestamp
t['size'] = ob.size
ret.append(t)
return ret
================================================
FILE: rocksdb/backup.pxd
================================================
from libcpp cimport bool as cpp_bool
from libcpp.string cimport string
from libcpp.vector cimport vector
from libc.stdint cimport uint32_t
from libc.stdint cimport int64_t
from libc.stdint cimport uint64_t
from status cimport Status
from db cimport DB
from env cimport Env
cdef extern from "rocksdb/utilities/backupable_db.h" namespace "rocksdb":
ctypedef uint32_t BackupID
cdef cppclass BackupableDBOptions:
BackupableDBOptions(const string& backup_dir)
cdef struct BackupInfo:
BackupID backup_id
int64_t timestamp
uint64_t size
cdef cppclass BackupEngine:
Status CreateNewBackup(DB*, cpp_bool) nogil except+
Status PurgeOldBackups(uint32_t) nogil except+
Status DeleteBackup(BackupID) nogil except+
void StopBackup() nogil except+
void GetBackupInfo(vector[BackupInfo]*) nogil except+
Status RestoreDBFromBackup(BackupID, string&, string&) nogil except+
Status RestoreDBFromLatestBackup(string&, string&) nogil except+
cdef Status BackupEngine_Open "rocksdb::BackupEngine::Open"(
Env*,
BackupableDBOptions&,
BackupEngine**)
================================================
FILE: rocksdb/cache.pxd
================================================
from std_memory cimport shared_ptr
cdef extern from "rocksdb/cache.h" namespace "rocksdb":
cdef cppclass Cache:
pass
cdef extern shared_ptr[Cache] NewLRUCache(size_t)
cdef extern shared_ptr[Cache] NewLRUCache(size_t, int)
================================================
FILE: rocksdb/comparator.pxd
================================================
from libcpp.string cimport string
from slice_ cimport Slice
from logger cimport Logger
from std_memory cimport shared_ptr
cdef extern from "rocksdb/comparator.h" namespace "rocksdb":
cdef cppclass Comparator:
const char* Name()
int Compare(const Slice&, const Slice&) const
cdef extern const Comparator* BytewiseComparator() nogil except +
ctypedef int (*compare_func)(
void*,
Logger*,
string&,
const Slice&,
const Slice&)
cdef extern from "cpp/comparator_wrapper.hpp" namespace "py_rocks":
cdef cppclass ComparatorWrapper:
ComparatorWrapper(string, void*, compare_func) nogil except +
void set_info_log(shared_ptr[Logger]) nogil except+
================================================
FILE: rocksdb/cpp/comparator_wrapper.hpp
================================================
#include "rocksdb/comparator.h"
#include "rocksdb/env.h"
#include <stdexcept>
using std::string;
using rocksdb::Comparator;
using rocksdb::Slice;
using rocksdb::Logger;
namespace py_rocks {
class ComparatorWrapper: public Comparator {
public:
typedef int (*compare_func)(
void*,
Logger*,
string&,
const Slice&,
const Slice&);
ComparatorWrapper(
string name,
void* compare_context,
compare_func compare_callback):
name(name),
compare_context(compare_context),
compare_callback(compare_callback)
{}
virtual int Compare(const Slice& a, const Slice& b) const {
string error_msg;
int val;
val = this->compare_callback(
this->compare_context,
this->info_log.get(),
error_msg,
a,
b);
if (error_msg.size()) {
throw std::runtime_error(error_msg.c_str());
}
return val;
}
virtual const char* Name() const {
return this->name.c_str();
}
virtual void FindShortestSeparator(string*, const Slice&) const {}
virtual void FindShortSuccessor(string*) const {}
void set_info_log(std::shared_ptr<Logger> info_log) {
this->info_log = info_log;
}
private:
string name;
void* compare_context;
compare_func compare_callback;
std::shared_ptr<Logger> info_log;
};
}
================================================
FILE: rocksdb/cpp/filter_policy_wrapper.hpp
================================================
#include "rocksdb/filter_policy.h"
#include "rocksdb/env.h"
#include <stdexcept>
using std::string;
using rocksdb::FilterPolicy;
using rocksdb::Slice;
using rocksdb::Logger;
namespace py_rocks {
class FilterPolicyWrapper: public FilterPolicy {
public:
typedef void (*create_filter_func)(
void* ctx,
Logger*,
string&,
const Slice* keys,
int n,
string* dst);
typedef bool (*key_may_match_func)(
void* ctx,
Logger*,
string&,
const Slice& key,
const Slice& filter);
FilterPolicyWrapper(
string name,
void* ctx,
create_filter_func create_filter_callback,
key_may_match_func key_may_match_callback):
name(name),
ctx(ctx),
create_filter_callback(create_filter_callback),
key_may_match_callback(key_may_match_callback)
{}
virtual void
CreateFilter(const Slice* keys, int n, std::string* dst) const {
string error_msg;
this->create_filter_callback(
this->ctx,
this->info_log.get(),
error_msg,
keys,
n,
dst);
if (error_msg.size()) {
throw std::runtime_error(error_msg.c_str());
}
}
virtual bool
KeyMayMatch(const Slice& key, const Slice& filter) const {
string error_msg;
bool val;
val = this->key_may_match_callback(
this->ctx,
this->info_log.get(),
error_msg,
key,
filter);
if (error_msg.size()) {
throw std::runtime_error(error_msg.c_str());
}
return val;
}
virtual const char* Name() const {
return this->name.c_str();
}
void set_info_log(std::shared_ptr<Logger> info_log) {
this->info_log = info_log;
}
private:
string name;
void* ctx;
create_filter_func create_filter_callback;
key_may_match_func key_may_match_callback;
std::shared_ptr<Logger> info_log;
};
}
================================================
FILE: rocksdb/cpp/memtable_factories.hpp
================================================
#include "rocksdb/memtablerep.h"
using rocksdb::MemTableRepFactory;
using rocksdb::VectorRepFactory;
using rocksdb::SkipListFactory;
namespace py_rocks {
MemTableRepFactory* NewVectorRepFactory(size_t count = 0) {
return new VectorRepFactory(count);
}
MemTableRepFactory* NewSkipListFactory() {
return new SkipListFactory();
}
}
================================================
FILE: rocksdb/cpp/merge_operator_wrapper.hpp
================================================
#include "rocksdb/merge_operator.h"
using std::string;
using std::deque;
using rocksdb::Slice;
using rocksdb::Logger;
using rocksdb::MergeOperator;
using rocksdb::AssociativeMergeOperator;
namespace py_rocks {
class AssociativeMergeOperatorWrapper: public AssociativeMergeOperator {
public:
typedef bool (*merge_func)(
void*,
const Slice& key,
const Slice* existing_value,
const Slice& value,
std::string* new_value,
Logger* logger);
AssociativeMergeOperatorWrapper(
string name,
void* merge_context,
merge_func merge_callback):
name(name),
merge_context(merge_context),
merge_callback(merge_callback)
{}
virtual bool Merge(
const Slice& key,
const Slice* existing_value,
const Slice& value,
std::string* new_value,
Logger* logger) const
{
return this->merge_callback(
this->merge_context,
key,
existing_value,
value,
new_value,
logger);
}
virtual const char* Name() const {
return this->name.c_str();
}
private:
string name;
void* merge_context;
merge_func merge_callback;
};
class MergeOperatorWrapper: public MergeOperator {
public:
typedef bool (*full_merge_func)(
void* ctx,
const Slice& key,
const Slice* existing_value,
const deque<string>& operand_list,
string* new_value,
Logger* logger);
typedef bool (*partial_merge_func)(
void* ctx,
const Slice& key,
const Slice& left_op,
const Slice& right_op,
string* new_value,
Logger* logger);
MergeOperatorWrapper(
string name,
void* full_merge_context,
void* partial_merge_context,
full_merge_func full_merge_callback,
partial_merge_func partial_merge_callback):
name(name),
full_merge_context(full_merge_context),
partial_merge_context(partial_merge_context),
full_merge_callback(full_merge_callback),
partial_merge_callback(partial_merge_callback)
{}
virtual bool FullMerge(
const Slice& key,
const Slice* existing_value,
const deque<string>& operand_list,
string* new_value,
Logger* logger) const
{
return this->full_merge_callback(
this->full_merge_context,
key,
existing_value,
operand_list,
new_value,
logger);
}
virtual bool PartialMerge (
const Slice& key,
const Slice& left_operand,
const Slice& right_operand,
string* new_value,
Logger* logger) const
{
return this->partial_merge_callback(
this->partial_merge_context,
key,
left_operand,
right_operand,
new_value,
logger);
}
virtual const char* Name() const {
return this->name.c_str();
}
private:
string name;
void* full_merge_context;
void* partial_merge_context;
full_merge_func full_merge_callback;
partial_merge_func partial_merge_callback;
};
}
================================================
FILE: rocksdb/cpp/slice_transform_wrapper.hpp
================================================
#include <string>
#include "rocksdb/slice_transform.h"
#include "rocksdb/env.h"
#include <stdexcept>
using std::string;
using rocksdb::SliceTransform;
using rocksdb::Slice;
using rocksdb::Logger;
namespace py_rocks {
class SliceTransformWrapper: public SliceTransform {
public:
typedef Slice (*transform_func)(
void*,
Logger*,
string&,
const Slice&);
typedef bool (*in_domain_func)(
void*,
Logger*,
string&,
const Slice&);
typedef bool (*in_range_func)(
void*,
Logger*,
string&,
const Slice&);
SliceTransformWrapper(
string name,
void* ctx,
transform_func transfrom_callback,
in_domain_func in_domain_callback,
in_range_func in_range_callback):
name(name),
ctx(ctx),
transfrom_callback(transfrom_callback),
in_domain_callback(in_domain_callback),
in_range_callback(in_range_callback)
{}
virtual const char* Name() const {
return this->name.c_str();
}
virtual Slice Transform(const Slice& src) const {
string error_msg;
Slice val;
val = this->transfrom_callback(
this->ctx,
this->info_log.get(),
error_msg,
src);
if (error_msg.size()) {
throw std::runtime_error(error_msg.c_str());
}
return val;
}
virtual bool InDomain(const Slice& src) const {
string error_msg;
bool val;
val = this->in_domain_callback(
this->ctx,
this->info_log.get(),
error_msg,
src);
if (error_msg.size()) {
throw std::runtime_error(error_msg.c_str());
}
return val;
}
virtual bool InRange(const Slice& dst) const {
string error_msg;
bool val;
val = this->in_range_callback(
this->ctx,
this->info_log.get(),
error_msg,
dst);
if (error_msg.size()) {
throw std::runtime_error(error_msg.c_str());
}
return val;
}
void set_info_log(std::shared_ptr<Logger> info_log) {
this->info_log = info_log;
}
private:
string name;
void* ctx;
transform_func transfrom_callback;
in_domain_func in_domain_callback;
in_range_func in_range_callback;
std::shared_ptr<Logger> info_log;
};
}
================================================
FILE: rocksdb/cpp/utils.hpp
================================================
#include <vector>
namespace py_rocks {
template <typename T>
const T* vector_data(std::vector<T>& v) {
return v.data();
}
}
================================================
FILE: rocksdb/cpp/write_batch_iter_helper.hpp
================================================
#pragma once
#include <vector>
#include "rocksdb/write_batch.h"
namespace py_rocks {
class RecordItemsHandler: public rocksdb::WriteBatch::Handler {
public:
enum Optype {PutRecord, MergeRecord, DeleteRecord};
class BatchItem {
public:
BatchItem(
const Optype& op,
const rocksdb::Slice& key,
const rocksdb::Slice& value):
op(op),
key(key),
value(value)
{}
const Optype op;
const rocksdb::Slice key;
const rocksdb::Slice value;
};
typedef std::vector<BatchItem> BatchItems;
public:
/* Items is filled during iteration. */
RecordItemsHandler(BatchItems* items): items(items) {}
void Put(const Slice& key, const Slice& value) {
this->items->emplace_back(PutRecord, key, value);
}
void Merge(const Slice& key, const Slice& value) {
this->items->emplace_back(MergeRecord, key, value);
}
virtual void Delete(const Slice& key) {
this->items->emplace_back(DeleteRecord, key, rocksdb::Slice());
}
private:
BatchItems* items;
};
rocksdb::Status
get_batch_items(const rocksdb::WriteBatch* batch, RecordItemsHandler::BatchItems* items) {
RecordItemsHandler handler(items);
return batch->Iterate(&handler);
}
}
================================================
FILE: rocksdb/db.pxd
================================================
cimport options
from libc.stdint cimport uint64_t
from status cimport Status
from libcpp cimport bool as cpp_bool
from libcpp.string cimport string
from libcpp.vector cimport vector
from slice_ cimport Slice
from snapshot cimport Snapshot
from iterator cimport Iterator
cdef extern from "rocksdb/write_batch.h" namespace "rocksdb":
cdef cppclass WriteBatch:
WriteBatch() nogil except+
WriteBatch(string) nogil except+
void Put(const Slice&, const Slice&) nogil except+
void Merge(const Slice&, const Slice&) nogil except+
void Delete(const Slice&) nogil except+
void PutLogData(const Slice&) nogil except+
void Clear() nogil except+
const string& Data() nogil except+
int Count() nogil except+
cdef extern from "cpp/write_batch_iter_helper.hpp" namespace "py_rocks":
cdef enum BatchItemOp "RecordItemsHandler::Optype":
BatchItemOpPut "py_rocks::RecordItemsHandler::Optype::PutRecord"
BatchItemOpMerge "py_rocks::RecordItemsHandler::Optype::MergeRecord"
BatchItemOpDelte "py_rocks::RecordItemsHandler::Optype::DeleteRecord"
cdef cppclass BatchItem "py_rocks::RecordItemsHandler::BatchItem":
BatchItemOp op
Slice key
Slice value
Status get_batch_items(WriteBatch* batch, vector[BatchItem]* items)
cdef extern from "rocksdb/db.h" namespace "rocksdb":
ctypedef uint64_t SequenceNumber
cdef struct LiveFileMetaData:
string name
int level
uint64_t size
string smallestkey
string largestkey
SequenceNumber smallest_seqno
SequenceNumber largest_seqno
cdef cppclass Range:
Range(const Slice&, const Slice&)
cdef cppclass DB:
Status Put(
const options.WriteOptions&,
const Slice&,
const Slice&) nogil except+
Status Delete(
const options.WriteOptions&,
const Slice&) nogil except+
Status Merge(
const options.WriteOptions&,
const Slice&,
const Slice&) nogil except+
Status Write(
const options.WriteOptions&,
WriteBatch*) nogil except+
Status Get(
const options.ReadOptions&,
const Slice&,
string*) nogil except+
vector[Status] MultiGet(
const options.ReadOptions&,
const vector[Slice]&,
vector[string]*) nogil except+
cpp_bool KeyMayExist(
const options.ReadOptions&,
Slice&,
string*,
cpp_bool*) nogil except+
cpp_bool KeyMayExist(
const options.ReadOptions&,
Slice&,
string*) nogil except+
Iterator* NewIterator(
const options.ReadOptions&) nogil except+
const Snapshot* GetSnapshot() nogil except+
void ReleaseSnapshot(const Snapshot*) nogil except+
cpp_bool GetProperty(
const Slice&,
string*) nogil except+
void GetApproximateSizes(
const Range*
int,
uint64_t*) nogil except+
Status CompactRange(
const options.CompactRangeOptions&,
const Slice*,
const Slice*) nogil except+
int NumberLevels() nogil except+
int MaxMemCompactionLevel() nogil except+
int Level0StopWriteTrigger() nogil except+
const string& GetName() nogil except+
const options.Options& GetOptions() nogil except+
Status Flush(const options.FlushOptions&) nogil except+
Status DisableFileDeletions() nogil except+
Status EnableFileDeletions() nogil except+
# TODO: Status GetSortedWalFiles(VectorLogPtr& files)
# TODO: SequenceNumber GetLatestSequenceNumber()
# TODO: Status GetUpdatesSince(
# SequenceNumber seq_number,
# unique_ptr[TransactionLogIterator]*)
Status DeleteFile(string) nogil except+
void GetLiveFilesMetaData(vector[LiveFileMetaData]*) nogil except+
cdef Status DB_Open "rocksdb::DB::Open"(
const options.Options&,
const string&,
DB**) nogil except+
cdef Status DB_OpenForReadOnly "rocksdb::DB::OpenForReadOnly"(
const options.Options&,
const string&,
DB**,
cpp_bool) nogil except+
cdef Status RepairDB(const string& dbname, const options.Options&)
================================================
FILE: rocksdb/env.pxd
================================================
cdef extern from "rocksdb/env.h" namespace "rocksdb":
cdef cppclass Env:
Env()
cdef Env* Env_Default "rocksdb::Env::Default"()
================================================
FILE: rocksdb/errors.py
================================================
class NotFound(Exception):
pass
class Corruption(Exception):
pass
class NotSupported(Exception):
pass
class InvalidArgument(Exception):
pass
class RocksIOError(Exception):
pass
class MergeInProgress(Exception):
pass
class Incomplete(Exception):
pass
================================================
FILE: rocksdb/filter_policy.pxd
================================================
from libcpp cimport bool as cpp_bool
from libcpp.string cimport string
from libc.string cimport const_char
from slice_ cimport Slice
from std_memory cimport shared_ptr
from logger cimport Logger
cdef extern from "rocksdb/filter_policy.h" namespace "rocksdb":
cdef cppclass FilterPolicy:
void CreateFilter(const Slice*, int, string*) nogil except+
cpp_bool KeyMayMatch(const Slice&, const Slice&) nogil except+
const_char* Name() nogil except+
cdef extern const FilterPolicy* NewBloomFilterPolicy(int) nogil except+
ctypedef void (*create_filter_func)(
void*,
Logger*,
string&,
const Slice*,
int,
string*)
ctypedef cpp_bool (*key_may_match_func)(
void*,
Logger*,
string&,
const Slice&,
const Slice&)
cdef extern from "cpp/filter_policy_wrapper.hpp" namespace "py_rocks":
cdef cppclass FilterPolicyWrapper:
FilterPolicyWrapper(
string,
void*,
create_filter_func,
key_may_match_func) nogil except+
void set_info_log(shared_ptr[Logger]) nogil except+
================================================
FILE: rocksdb/interfaces.py
================================================
from abc import ABCMeta
from abc import abstractmethod
class Comparator:
__metaclass__ = ABCMeta
@abstractmethod
def compare(self, a, b):
pass
@abstractmethod
def name(self):
pass
class AssociativeMergeOperator:
__metaclass__ = ABCMeta
@abstractmethod
def merge(self, key, existing_value, value):
pass
@abstractmethod
def name(self):
pass
class MergeOperator:
__metaclass__ = ABCMeta
@abstractmethod
def full_merge(self, key, existing_value, operand_list):
pass
@abstractmethod
def partial_merge(self, key, left_operand, right_operand):
pass
@abstractmethod
def name(self):
pass
class FilterPolicy:
__metaclass__ = ABCMeta
@abstractmethod
def name(self):
pass
@abstractmethod
def create_filter(self, keys):
pass
@abstractmethod
def key_may_match(self, key, filter_):
pass
class SliceTransform:
__metaclass__ = ABCMeta
@abstractmethod
def name(self):
pass
@abstractmethod
def transform(self, src):
pass
@abstractmethod
def in_domain(self, src):
pass
@abstractmethod
def in_range(self, dst):
pass
================================================
FILE: rocksdb/iterator.pxd
================================================
from libcpp cimport bool as cpp_bool
from slice_ cimport Slice
from status cimport Status
cdef extern from "rocksdb/iterator.h" namespace "rocksdb":
cdef cppclass Iterator:
cpp_bool Valid() nogil except+
void SeekToFirst() nogil except+
void SeekToLast() nogil except+
void Seek(const Slice&) nogil except+
void Next() nogil except+
void Prev() nogil except+
Slice key() nogil except+
Slice value() nogil except+
Status status() nogil except+
================================================
FILE: rocksdb/logger.pxd
================================================
cdef extern from "rocksdb/env.h" namespace "rocksdb":
cdef cppclass Logger:
pass
void Log(Logger*, const char*, ...) nogil except+
================================================
FILE: rocksdb/memtablerep.pxd
================================================
from libc.stdint cimport int32_t
cdef extern from "rocksdb/memtablerep.h" namespace "rocksdb":
cdef cppclass MemTableRepFactory:
MemTableRepFactory()
cdef MemTableRepFactory* NewHashSkipListRepFactory(size_t, int32_t, int32_t)
cdef MemTableRepFactory* NewHashLinkListRepFactory(size_t)
cdef extern from "cpp/memtable_factories.hpp" namespace "py_rocks":
cdef MemTableRepFactory* NewVectorRepFactory(size_t)
cdef MemTableRepFactory* NewSkipListFactory()
================================================
FILE: rocksdb/merge_operator.pxd
================================================
from libcpp.string cimport string
from libcpp cimport bool as cpp_bool
from libcpp.deque cimport deque
from slice_ cimport Slice
from logger cimport Logger
cdef extern from "rocksdb/merge_operator.h" namespace "rocksdb":
cdef cppclass MergeOperator:
pass
ctypedef cpp_bool (*merge_func)(
void*,
const Slice&,
const Slice*,
const Slice&,
string*,
Logger*)
ctypedef cpp_bool (*full_merge_func)(
void* ctx,
const Slice& key,
const Slice* existing_value,
const deque[string]& operand_list,
string* new_value,
Logger* logger)
ctypedef cpp_bool (*partial_merge_func)(
void* ctx,
const Slice& key,
const Slice& left_op,
const Slice& right_op,
string* new_value,
Logger* logger)
cdef extern from "cpp/merge_operator_wrapper.hpp" namespace "py_rocks":
cdef cppclass AssociativeMergeOperatorWrapper:
AssociativeMergeOperatorWrapper(string, void*, merge_func) nogil except+
cdef cppclass MergeOperatorWrapper:
MergeOperatorWrapper(
string,
void*,
void*,
full_merge_func,
partial_merge_func) nogil except+
================================================
FILE: rocksdb/options.pxd
================================================
from libcpp cimport bool as cpp_bool
from libcpp.string cimport string
from libcpp.vector cimport vector
from libc.stdint cimport uint64_t
from libc.stdint cimport uint32_t
from std_memory cimport shared_ptr
from comparator cimport Comparator
from merge_operator cimport MergeOperator
from logger cimport Logger
from slice_ cimport Slice
from snapshot cimport Snapshot
from slice_transform cimport SliceTransform
from table_factory cimport TableFactory
from memtablerep cimport MemTableRepFactory
from universal_compaction cimport CompactionOptionsUniversal
from cache cimport Cache
cdef extern from "rocksdb/options.h" namespace "rocksdb":
ctypedef enum CompactionStyle:
kCompactionStyleLevel
kCompactionStyleUniversal
ctypedef enum CompressionType:
kNoCompression
kSnappyCompression
kZlibCompression
kBZip2Compression
kLZ4Compression
kLZ4HCCompression
ctypedef enum ReadTier:
kReadAllTier
kBlockCacheTier
cdef cppclass Options:
const Comparator* comparator
shared_ptr[MergeOperator] merge_operator
# TODO: compaction_filter
# TODO: compaction_filter_factory
cpp_bool create_if_missing
cpp_bool error_if_exists
cpp_bool paranoid_checks
# TODO: env
shared_ptr[Logger] info_log
size_t write_buffer_size
int max_write_buffer_number
int min_write_buffer_number_to_merge
int max_open_files
CompressionType compression
# TODO: compression_per_level
# TODO: compression_opts
shared_ptr[SliceTransform] prefix_extractor
int num_levels
int level0_file_num_compaction_trigger
int level0_slowdown_writes_trigger
int level0_stop_writes_trigger
int max_mem_compaction_level
uint64_t target_file_size_base
int target_file_size_multiplier
uint64_t max_bytes_for_level_base
int max_bytes_for_level_multiplier
vector[int] max_bytes_for_level_multiplier_additional
int expanded_compaction_factor
int source_compaction_factor
int max_grandparent_overlap_factor
# TODO: statistics
cpp_bool disableDataSync
cpp_bool use_fsync
string db_log_dir
string wal_dir
uint64_t delete_obsolete_files_period_micros
int max_background_compactions
int max_background_flushes
size_t max_log_file_size
size_t log_file_time_to_roll
size_t keep_log_file_num
double soft_rate_limit
double hard_rate_limit
unsigned int rate_limit_delay_max_milliseconds
uint64_t max_manifest_file_size
int table_cache_numshardbits
size_t arena_block_size
# TODO: PrepareForBulkLoad()
cpp_bool disable_auto_compactions
uint64_t WAL_ttl_seconds
uint64_t WAL_size_limit_MB
size_t manifest_preallocation_size
cpp_bool purge_redundant_kvs_while_flush
cpp_bool allow_os_buffer
cpp_bool allow_mmap_reads
cpp_bool allow_mmap_writes
cpp_bool is_fd_close_on_exec
cpp_bool skip_log_error_on_recovery
unsigned int stats_dump_period_sec
cpp_bool advise_random_on_open
# TODO: enum { NONE, NORMAL, SEQUENTIAL, WILLNEED } access_hint_on_compaction_start
cpp_bool use_adaptive_mutex
uint64_t bytes_per_sync
cpp_bool verify_checksums_in_compaction
CompactionStyle compaction_style
CompactionOptionsUniversal compaction_options_universal
cpp_bool filter_deletes
uint64_t max_sequential_skip_in_iterations
shared_ptr[MemTableRepFactory] memtable_factory
shared_ptr[TableFactory] table_factory
# TODO: table_properties_collectors
cpp_bool inplace_update_support
size_t inplace_update_num_locks
shared_ptr[Cache] row_cache
cdef cppclass WriteOptions:
cpp_bool sync
cpp_bool disableWAL
cdef cppclass ReadOptions:
cpp_bool verify_checksums
cpp_bool fill_cache
const Snapshot* snapshot
ReadTier read_tier
cdef cppclass FlushOptions:
cpp_bool wait
ctypedef enum BottommostLevelCompaction:
blc_skip "rocksdb::BottommostLevelCompaction::kSkip"
blc_is_filter "rocksdb::BottommostLevelCompaction::kIfHaveCompactionFilter"
blc_force "rocksdb::BottommostLevelCompaction::kForce"
cdef cppclass CompactRangeOptions:
cpp_bool change_level
int target_level
uint32_t target_path_id
BottommostLevelCompaction bottommost_level_compaction
================================================
FILE: rocksdb/slice_.pxd
================================================
from libcpp.string cimport string
from libcpp cimport bool as cpp_bool
cdef extern from "rocksdb/slice.h" namespace "rocksdb":
cdef cppclass Slice:
Slice() nogil
Slice(const char*, size_t) nogil
Slice(const string&) nogil
Slice(const char*) nogil
const char* data() nogil
size_t size() nogil
cpp_bool empty() nogil
char operator[](int) nogil
void clear() nogil
void remove_prefix(size_t) nogil
string ToString() nogil
string ToString(cpp_bool) nogil
int compare(const Slice&) nogil
cpp_bool starts_with(const Slice&) nogil
================================================
FILE: rocksdb/slice_transform.pxd
================================================
from slice_ cimport Slice
from libcpp.string cimport string
from libcpp cimport bool as cpp_bool
from logger cimport Logger
from std_memory cimport shared_ptr
cdef extern from "rocksdb/slice_transform.h" namespace "rocksdb":
cdef cppclass SliceTransform:
pass
ctypedef Slice (*transform_func)(
void*,
Logger*,
string&,
const Slice&)
ctypedef cpp_bool (*in_domain_func)(
void*,
Logger*,
string&,
const Slice&)
ctypedef cpp_bool (*in_range_func)(
void*,
Logger*,
string&,
const Slice&)
cdef extern from "cpp/slice_transform_wrapper.hpp" namespace "py_rocks":
cdef cppclass SliceTransformWrapper:
SliceTransformWrapper(
string name,
void*,
transform_func,
in_domain_func,
in_range_func) nogil except+
void set_info_log(shared_ptr[Logger]) nogil except+
================================================
FILE: rocksdb/snapshot.pxd
================================================
cdef extern from "rocksdb/db.h" namespace "rocksdb":
cdef cppclass Snapshot:
pass
================================================
FILE: rocksdb/status.pxd
================================================
from libcpp cimport bool as cpp_bool
from libcpp.string cimport string
cdef extern from "rocksdb/status.h" namespace "rocksdb":
cdef cppclass Status:
Status()
cpp_bool ok() nogil
cpp_bool IsNotFound() nogil const
cpp_bool IsCorruption() nogil const
cpp_bool IsNotSupported() nogil const
cpp_bool IsInvalidArgument() nogil const
cpp_bool IsIOError() nogil const
cpp_bool IsMergeInProgress() nogil const
cpp_bool IsIncomplete() nogil const
string ToString() nogil except+
================================================
FILE: rocksdb/std_memory.pxd
================================================
cdef extern from "<memory>" namespace "std":
cdef cppclass shared_ptr[T]:
shared_ptr() nogil except+
shared_ptr(T*) nogil except+
void reset() nogil except+
void reset(T*) nogil except+
T* get() nogil except+
================================================
FILE: rocksdb/table_factory.pxd
================================================
from libc.stdint cimport uint32_t
from libcpp cimport bool as cpp_bool
from std_memory cimport shared_ptr
from cache cimport Cache
from filter_policy cimport FilterPolicy
cdef extern from "rocksdb/table.h" namespace "rocksdb":
cdef cppclass TableFactory:
TableFactory()
ctypedef enum BlockBasedTableIndexType:
kBinarySearch "rocksdb::BlockBasedTableOptions::IndexType::kBinarySearch"
kHashSearch "rocksdb::BlockBasedTableOptions::IndexType::kHashSearch"
ctypedef enum ChecksumType:
kCRC32c
kxxHash
cdef cppclass BlockBasedTableOptions:
BlockBasedTableOptions()
BlockBasedTableIndexType index_type
cpp_bool hash_index_allow_collision
ChecksumType checksum
cpp_bool no_block_cache
size_t block_size
int block_size_deviation
int block_restart_interval
cpp_bool whole_key_filtering
shared_ptr[Cache] block_cache
shared_ptr[Cache] block_cache_compressed
shared_ptr[FilterPolicy] filter_policy
cdef TableFactory* NewBlockBasedTableFactory(const BlockBasedTableOptions&)
ctypedef enum EncodingType:
kPlain
kPrefix
cdef cppclass PlainTableOptions:
uint32_t user_key_len
int bloom_bits_per_key
double hash_table_ratio
size_t index_sparseness
size_t huge_page_tlb_size
EncodingType encoding_type
cpp_bool full_scan_mode
cpp_bool store_index_in_file
cdef TableFactory* NewPlainTableFactory(const PlainTableOptions&)
================================================
FILE: rocksdb/tests/__init__.py
================================================
================================================
FILE: rocksdb/tests/test_db.py
================================================
import os
import shutil
import gc
import unittest
import rocksdb
from itertools import takewhile
def int_to_bytes(ob):
return str(ob).encode('ascii')
class TestHelper(object):
def _clean(self):
if os.path.exists('/tmp/test'):
shutil.rmtree("/tmp/test")
def _close_db(self):
del self.db
gc.collect()
class TestDB(unittest.TestCase, TestHelper):
def setUp(self):
opts = rocksdb.Options(create_if_missing=True)
self._clean()
self.db = rocksdb.DB("/tmp/test", opts)
def tearDown(self):
self._close_db()
def test_options_used_twice(self):
expected = "Options object is already used by another DB"
with self.assertRaisesRegexp(Exception, expected):
rocksdb.DB("/tmp/test2", self.db.options)
def test_unicode_path(self):
name = b'/tmp/M\xc3\xbcnchen'.decode('utf8')
rocksdb.DB(name, rocksdb.Options(create_if_missing=True))
self.addCleanup(shutil.rmtree, name)
self.assertTrue(os.path.isdir(name))
def test_get_none(self):
self.assertIsNone(self.db.get(b'xxx'))
def test_put_get(self):
se
gitextract_d2ye60kk/ ├── .gitignore ├── LICENSE.md ├── MANIFEST.in ├── README.rst ├── docs/ │ ├── Makefile │ ├── api/ │ │ ├── backup.rst │ │ ├── database.rst │ │ ├── index.rst │ │ ├── interfaces.rst │ │ └── options.rst │ ├── changelog.rst │ ├── conf.py │ ├── index.rst │ ├── installation.rst │ └── tutorial/ │ └── index.rst ├── rocksdb/ │ ├── __init__.py │ ├── _rocksdb.pyx │ ├── backup.pxd │ ├── cache.pxd │ ├── comparator.pxd │ ├── cpp/ │ │ ├── comparator_wrapper.hpp │ │ ├── filter_policy_wrapper.hpp │ │ ├── memtable_factories.hpp │ │ ├── merge_operator_wrapper.hpp │ │ ├── slice_transform_wrapper.hpp │ │ ├── utils.hpp │ │ └── write_batch_iter_helper.hpp │ ├── db.pxd │ ├── env.pxd │ ├── errors.py │ ├── filter_policy.pxd │ ├── interfaces.py │ ├── iterator.pxd │ ├── logger.pxd │ ├── memtablerep.pxd │ ├── merge_operator.pxd │ ├── options.pxd │ ├── slice_.pxd │ ├── slice_transform.pxd │ ├── snapshot.pxd │ ├── status.pxd │ ├── std_memory.pxd │ ├── table_factory.pxd │ ├── tests/ │ │ ├── __init__.py │ │ ├── test_db.py │ │ └── test_options.py │ └── universal_compaction.pxd └── setup.py
SYMBOL INDEX (142 symbols across 12 files)
FILE: rocksdb/cpp/comparator_wrapper.hpp
type py_rocks (line 10) | namespace py_rocks {
class ComparatorWrapper (line 11) | class ComparatorWrapper: public Comparator {
method ComparatorWrapper (line 20) | ComparatorWrapper(
method Compare (line 29) | virtual int Compare(const Slice& a, const Slice& b) const {
method FindShortestSeparator (line 50) | virtual void FindShortestSeparator(string*, const Slice&) const {}
method FindShortSuccessor (line 51) | virtual void FindShortSuccessor(string*) const {}
method set_info_log (line 53) | void set_info_log(std::shared_ptr<Logger> info_log) {
FILE: rocksdb/cpp/filter_policy_wrapper.hpp
type py_rocks (line 10) | namespace py_rocks {
class FilterPolicyWrapper (line 11) | class FilterPolicyWrapper: public FilterPolicy {
method FilterPolicyWrapper (line 28) | FilterPolicyWrapper(
method CreateFilter (line 39) | virtual void
method KeyMayMatch (line 56) | virtual bool
method set_info_log (line 78) | void set_info_log(std::shared_ptr<Logger> info_log) {
FILE: rocksdb/cpp/memtable_factories.hpp
type py_rocks (line 7) | namespace py_rocks {
function MemTableRepFactory (line 8) | MemTableRepFactory* NewVectorRepFactory(size_t count = 0) {
function MemTableRepFactory (line 12) | MemTableRepFactory* NewSkipListFactory() {
FILE: rocksdb/cpp/merge_operator_wrapper.hpp
type py_rocks (line 10) | namespace py_rocks {
class AssociativeMergeOperatorWrapper (line 11) | class AssociativeMergeOperatorWrapper: public AssociativeMergeOperator {
method AssociativeMergeOperatorWrapper (line 22) | AssociativeMergeOperatorWrapper(
method Merge (line 31) | virtual bool Merge(
class MergeOperatorWrapper (line 57) | class MergeOperatorWrapper: public MergeOperator {
method MergeOperatorWrapper (line 75) | MergeOperatorWrapper(
method FullMerge (line 88) | virtual bool FullMerge(
method PartialMerge (line 104) | virtual bool PartialMerge (
FILE: rocksdb/cpp/slice_transform_wrapper.hpp
type py_rocks (line 11) | namespace py_rocks {
class SliceTransformWrapper (line 12) | class SliceTransformWrapper: public SliceTransform {
method SliceTransformWrapper (line 32) | SliceTransformWrapper(
method Slice (line 49) | virtual Slice Transform(const Slice& src) const {
method InDomain (line 65) | virtual bool InDomain(const Slice& src) const {
method InRange (line 81) | virtual bool InRange(const Slice& dst) const {
method set_info_log (line 97) | void set_info_log(std::shared_ptr<Logger> info_log) {
FILE: rocksdb/cpp/utils.hpp
type py_rocks (line 3) | namespace py_rocks {
function T (line 5) | const T* vector_data(std::vector<T>& v) {
FILE: rocksdb/cpp/write_batch_iter_helper.hpp
type py_rocks (line 6) | namespace py_rocks {
class RecordItemsHandler (line 8) | class RecordItemsHandler: public rocksdb::WriteBatch::Handler {
type Optype (line 10) | enum Optype {PutRecord, MergeRecord, DeleteRecord}
class BatchItem (line 12) | class BatchItem {
method BatchItem (line 14) | BatchItem(
method RecordItemsHandler (line 32) | RecordItemsHandler(BatchItems* items): items(items) {}
method Put (line 34) | void Put(const Slice& key, const Slice& value) {
method Merge (line 38) | void Merge(const Slice& key, const Slice& value) {
method Delete (line 42) | virtual void Delete(const Slice& key) {
function get_batch_items (line 50) | rocksdb::Status
FILE: rocksdb/errors.py
class NotFound (line 1) | class NotFound(Exception):
class Corruption (line 4) | class Corruption(Exception):
class NotSupported (line 7) | class NotSupported(Exception):
class InvalidArgument (line 10) | class InvalidArgument(Exception):
class RocksIOError (line 13) | class RocksIOError(Exception):
class MergeInProgress (line 16) | class MergeInProgress(Exception):
class Incomplete (line 19) | class Incomplete(Exception):
FILE: rocksdb/interfaces.py
class Comparator (line 5) | class Comparator:
method compare (line 9) | def compare(self, a, b):
method name (line 13) | def name(self):
class AssociativeMergeOperator (line 17) | class AssociativeMergeOperator:
method merge (line 21) | def merge(self, key, existing_value, value):
method name (line 25) | def name(self):
class MergeOperator (line 29) | class MergeOperator:
method full_merge (line 33) | def full_merge(self, key, existing_value, operand_list):
method partial_merge (line 37) | def partial_merge(self, key, left_operand, right_operand):
method name (line 41) | def name(self):
class FilterPolicy (line 45) | class FilterPolicy:
method name (line 49) | def name(self):
method create_filter (line 53) | def create_filter(self, keys):
method key_may_match (line 57) | def key_may_match(self, key, filter_):
class SliceTransform (line 60) | class SliceTransform:
method name (line 64) | def name(self):
method transform (line 68) | def transform(self, src):
method in_domain (line 72) | def in_domain(self, src):
method in_range (line 76) | def in_range(self, dst):
FILE: rocksdb/tests/test_db.py
function int_to_bytes (line 8) | def int_to_bytes(ob):
class TestHelper (line 11) | class TestHelper(object):
method _clean (line 12) | def _clean(self):
method _close_db (line 16) | def _close_db(self):
class TestDB (line 21) | class TestDB(unittest.TestCase, TestHelper):
method setUp (line 22) | def setUp(self):
method tearDown (line 27) | def tearDown(self):
method test_options_used_twice (line 30) | def test_options_used_twice(self):
method test_unicode_path (line 35) | def test_unicode_path(self):
method test_get_none (line 41) | def test_get_none(self):
method test_put_get (line 44) | def test_put_get(self):
method test_multi_get (line 48) | def test_multi_get(self):
method test_delete (line 57) | def test_delete(self):
method test_write_batch (line 63) | def test_write_batch(self):
method test_write_batch_iter (line 76) | def test_write_batch_iter(self):
method test_key_may_exists (line 100) | def test_key_may_exists(self):
method test_iter_keys (line 108) | def test_iter_keys(self):
method test_iter_values (line 138) | def test_iter_values(self):
method test_iter_items (line 158) | def test_iter_items(self):
method test_reverse_iter (line 178) | def test_reverse_iter(self):
method test_snapshot (line 190) | def test_snapshot(self):
method test_get_property (line 206) | def test_get_property(self):
method test_compact_range (line 216) | def test_compact_range(self):
class AssocCounter (line 224) | class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator):
method merge (line 225) | def merge(self, key, existing_value, value):
method name (line 230) | def name(self):
class TestAssocMerge (line 234) | class TestAssocMerge(unittest.TestCase, TestHelper):
method setUp (line 235) | def setUp(self):
method tearDown (line 242) | def tearDown(self):
method test_merge (line 245) | def test_merge(self):
class FullCounter (line 251) | class FullCounter(rocksdb.interfaces.MergeOperator):
method name (line 252) | def name(self):
method full_merge (line 255) | def full_merge(self, key, existing_value, operand_list):
method partial_merge (line 262) | def partial_merge(self, key, left, right):
class TestFullMerge (line 266) | class TestFullMerge(unittest.TestCase, TestHelper):
method setUp (line 267) | def setUp(self):
method tearDown (line 274) | def tearDown(self):
method test_merge (line 277) | def test_merge(self):
class SimpleComparator (line 283) | class SimpleComparator(rocksdb.interfaces.Comparator):
method name (line 284) | def name(self):
method compare (line 287) | def compare(self, a, b):
class TestComparator (line 298) | class TestComparator(unittest.TestCase, TestHelper):
method setUp (line 299) | def setUp(self):
method tearDown (line 306) | def tearDown(self):
method test_compare (line 309) | def test_compare(self):
class StaticPrefix (line 315) | class StaticPrefix(rocksdb.interfaces.SliceTransform):
method name (line 316) | def name(self):
method transform (line 319) | def transform(self, src):
method in_domain (line 322) | def in_domain(self, src):
method in_range (line 325) | def in_range(self, dst):
class TestPrefixExtractor (line 328) | class TestPrefixExtractor(unittest.TestCase, TestHelper):
method setUp (line 329) | def setUp(self):
method tearDown (line 335) | def tearDown(self):
method _fill_db (line 338) | def _fill_db(self):
method test_prefix_iterkeys (line 348) | def test_prefix_iterkeys(self):
method test_prefix_iteritems (line 361) | def test_prefix_iteritems(self):
FILE: rocksdb/tests/test_options.py
class TestFilterPolicy (line 4) | class TestFilterPolicy(rocksdb.interfaces.FilterPolicy):
method create_filter (line 5) | def create_filter(self, keys):
method key_may_match (line 8) | def key_may_match(self, key, fil):
method name (line 11) | def name(self):
class TestMergeOperator (line 14) | class TestMergeOperator(rocksdb.interfaces.MergeOperator):
method full_merge (line 15) | def full_merge(self, *args, **kwargs):
method partial_merge (line 18) | def partial_merge(self, *args, **kwargs):
method name (line 21) | def name(self):
class TestOptions (line 24) | class TestOptions(unittest.TestCase):
method test_simple (line 25) | def test_simple(self):
method test_block_options (line 44) | def test_block_options(self):
method test_unicode_path (line 50) | def test_unicode_path(self):
method test_table_factory (line 59) | def test_table_factory(self):
method test_compaction_style (line 66) | def test_compaction_style(self):
method test_compaction_opts_universal (line 79) | def test_compaction_opts_universal(self):
method test_row_cache (line 100) | def test_row_cache(self):
FILE: setup.py
function cythonize (line 8) | def cythonize(extensions): return extensions
Condensed preview — 48 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (219K chars).
[
{
"path": ".gitignore",
"chars": 22,
"preview": "/build/\n/docs/_build/\n"
},
{
"path": "LICENSE.md",
"chars": 1485,
"preview": "Copyright (c) 2014, Stephan Hofmockel\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or w"
},
{
"path": "MANIFEST.in",
"chars": 90,
"preview": "include rocksdb/cpp/*.hpp\nrecursive-include rocksdb *.pxd\nrecursive-include rocksdb *.pyx\n"
},
{
"path": "README.rst",
"chars": 1002,
"preview": "pyrocksdb\n=========\n\nPython bindings for RocksDB.\nSee http://pyrocksdb.readthedocs.org for a more comprehensive install "
},
{
"path": "docs/Makefile",
"chars": 6766,
"preview": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD "
},
{
"path": "docs/api/backup.rst",
"chars": 2061,
"preview": "Backup and Restore\n******************\n\nBackupEngine\n============\n\n.. py:class:: rocksdb.BackupEngine\n\n .. py:method::"
},
{
"path": "docs/api/database.rst",
"chars": 16665,
"preview": "Database interactions\n*********************\n\nDatabase object\n===============\n\n.. py:class:: rocksdb.DB\n\n .. py:method"
},
{
"path": "docs/api/index.rst",
"chars": 186,
"preview": "Python driver for RocksDB\n=========================\n\n .. py:module:: rocksdb\n\n.. toctree::\n\n Options <options>\n Da"
},
{
"path": "docs/api/interfaces.rst",
"chars": 9562,
"preview": "Interfaces\n**********\n\nComparator\n==========\n\n.. py:class:: rocksdb.interfaces.Comparator\n\n A Comparator object provi"
},
{
"path": "docs/api/options.rst",
"chars": 37885,
"preview": "Options creation\n****************\n\nOptions object\n==============\n\n\n.. py:class:: rocksdb.Options\n\n .. IMPORTANT:: \n\n "
},
{
"path": "docs/changelog.rst",
"chars": 4308,
"preview": "Changelog\n*********\n\nVersion 0.5\n-----------\n\n\nVersion 0.4\n-----------\nThis version works with RocksDB v3.12.\n\n* Added :"
},
{
"path": "docs/conf.py",
"chars": 8198,
"preview": "# -*- coding: utf-8 -*-\n#\n# pyrocksdb documentation build configuration file, created by\n# sphinx-quickstart on Tue Dec "
},
{
"path": "docs/index.rst",
"chars": 951,
"preview": "Welcome to pyrocksdb's documentation!\n=====================================\n\nOverview\n--------\nPython bindings to the C+"
},
{
"path": "docs/installation.rst",
"chars": 1408,
"preview": "Installing\n**********\n.. highlight:: bash\n\n\nBuilding rocksdb\n----------------\n\nBriefly describes how to build rocksdb un"
},
{
"path": "docs/tutorial/index.rst",
"chars": 9945,
"preview": "Basic Usage of pyrocksdb\n************************\n\nOpen\n====\n\nThe most basic open call is ::\n\n import rocksdb\n\n db"
},
{
"path": "rocksdb/__init__.py",
"chars": 24,
"preview": "from ._rocksdb import *\n"
},
{
"path": "rocksdb/_rocksdb.pyx",
"chars": 58436,
"preview": "import cython\nfrom libcpp.string cimport string\nfrom libcpp.deque cimport deque\nfrom libcpp.vector cimport vector\nfrom c"
},
{
"path": "rocksdb/backup.pxd",
"chars": 1176,
"preview": "from libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\nfrom libcpp.vector cimport vector\nfrom libc.stdin"
},
{
"path": "rocksdb/cache.pxd",
"chars": 244,
"preview": "from std_memory cimport shared_ptr\n\ncdef extern from \"rocksdb/cache.h\" namespace \"rocksdb\":\n cdef cppclass Cache:\n "
},
{
"path": "rocksdb/comparator.pxd",
"chars": 706,
"preview": "from libcpp.string cimport string\nfrom slice_ cimport Slice\nfrom logger cimport Logger\nfrom std_memory cimport shared_pt"
},
{
"path": "rocksdb/cpp/comparator_wrapper.hpp",
"chars": 1791,
"preview": "#include \"rocksdb/comparator.h\"\n#include \"rocksdb/env.h\"\n#include <stdexcept>\n\nusing std::string;\nusing rocksdb::Compara"
},
{
"path": "rocksdb/cpp/filter_policy_wrapper.hpp",
"chars": 2593,
"preview": "#include \"rocksdb/filter_policy.h\"\n#include \"rocksdb/env.h\"\n#include <stdexcept>\n\nusing std::string;\nusing rocksdb::Filt"
},
{
"path": "rocksdb/cpp/memtable_factories.hpp",
"chars": 364,
"preview": "#include \"rocksdb/memtablerep.h\"\n\nusing rocksdb::MemTableRepFactory;\nusing rocksdb::VectorRepFactory;\nusing rocksdb::Ski"
},
{
"path": "rocksdb/cpp/merge_operator_wrapper.hpp",
"chars": 4161,
"preview": "#include \"rocksdb/merge_operator.h\"\n\nusing std::string;\nusing std::deque;\nusing rocksdb::Slice;\nusing rocksdb::Logger;\nu"
},
{
"path": "rocksdb/cpp/slice_transform_wrapper.hpp",
"chars": 3124,
"preview": "#include <string>\n#include \"rocksdb/slice_transform.h\"\n#include \"rocksdb/env.h\"\n#include <stdexcept>\n\nusing std::string;"
},
{
"path": "rocksdb/cpp/utils.hpp",
"chars": 145,
"preview": "#include <vector>\n\nnamespace py_rocks {\n template <typename T>\n const T* vector_data(std::vector<T>& v) {\n "
},
{
"path": "rocksdb/cpp/write_batch_iter_helper.hpp",
"chars": 1483,
"preview": "#pragma once\n\n#include <vector>\n#include \"rocksdb/write_batch.h\"\n\nnamespace py_rocks {\n\nclass RecordItemsHandler: public"
},
{
"path": "rocksdb/db.pxd",
"chars": 4477,
"preview": "cimport options\nfrom libc.stdint cimport uint64_t\nfrom status cimport Status\nfrom libcpp cimport bool as cpp_bool\nfrom l"
},
{
"path": "rocksdb/env.pxd",
"chars": 144,
"preview": "cdef extern from \"rocksdb/env.h\" namespace \"rocksdb\":\n cdef cppclass Env:\n Env()\n\n cdef Env* Env_Default \"r"
},
{
"path": "rocksdb/errors.py",
"chars": 284,
"preview": "class NotFound(Exception):\n pass\n\nclass Corruption(Exception):\n pass\n\nclass NotSupported(Exception):\n pass\n\ncla"
},
{
"path": "rocksdb/filter_policy.pxd",
"chars": 1097,
"preview": "from libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\nfrom libc.string cimport const_char\nfrom slice_ c"
},
{
"path": "rocksdb/interfaces.py",
"chars": 1262,
"preview": "from abc import ABCMeta\nfrom abc import abstractmethod\n\n\nclass Comparator:\n __metaclass__ = ABCMeta\n\n @abstractmet"
},
{
"path": "rocksdb/iterator.pxd",
"chars": 520,
"preview": "from libcpp cimport bool as cpp_bool\nfrom slice_ cimport Slice\nfrom status cimport Status\n\ncdef extern from \"rocksdb/ite"
},
{
"path": "rocksdb/logger.pxd",
"chars": 148,
"preview": "cdef extern from \"rocksdb/env.h\" namespace \"rocksdb\":\n cdef cppclass Logger:\n pass\n\n void Log(Logger*, cons"
},
{
"path": "rocksdb/memtablerep.pxd",
"chars": 484,
"preview": "from libc.stdint cimport int32_t\n\ncdef extern from \"rocksdb/memtablerep.h\" namespace \"rocksdb\":\n cdef cppclass MemTab"
},
{
"path": "rocksdb/merge_operator.pxd",
"chars": 1168,
"preview": "from libcpp.string cimport string\nfrom libcpp cimport bool as cpp_bool\nfrom libcpp.deque cimport deque\nfrom slice_ cimpo"
},
{
"path": "rocksdb/options.pxd",
"chars": 4673,
"preview": "from libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\nfrom libcpp.vector cimport vector\nfrom libc.stdin"
},
{
"path": "rocksdb/slice_.pxd",
"chars": 641,
"preview": "from libcpp.string cimport string\nfrom libcpp cimport bool as cpp_bool\n\ncdef extern from \"rocksdb/slice.h\" namespace \"ro"
},
{
"path": "rocksdb/slice_transform.pxd",
"chars": 915,
"preview": "from slice_ cimport Slice\nfrom libcpp.string cimport string\nfrom libcpp cimport bool as cpp_bool\nfrom logger cimport Log"
},
{
"path": "rocksdb/snapshot.pxd",
"chars": 94,
"preview": "cdef extern from \"rocksdb/db.h\" namespace \"rocksdb\":\n cdef cppclass Snapshot:\n pass\n"
},
{
"path": "rocksdb/status.pxd",
"chars": 555,
"preview": "from libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\n\ncdef extern from \"rocksdb/status.h\" namespace \"r"
},
{
"path": "rocksdb/std_memory.pxd",
"chars": 253,
"preview": "cdef extern from \"<memory>\" namespace \"std\":\n cdef cppclass shared_ptr[T]:\n shared_ptr() nogil except+\n "
},
{
"path": "rocksdb/table_factory.pxd",
"chars": 1562,
"preview": "from libc.stdint cimport uint32_t\nfrom libcpp cimport bool as cpp_bool\nfrom std_memory cimport shared_ptr\n\nfrom cache ci"
},
{
"path": "rocksdb/tests/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "rocksdb/tests/test_db.py",
"chars": 10490,
"preview": "import os\nimport shutil\nimport gc\nimport unittest\nimport rocksdb\nfrom itertools import takewhile\n\ndef int_to_bytes(ob):\n"
},
{
"path": "rocksdb/tests/test_options.py",
"chars": 3620,
"preview": "import unittest\nimport rocksdb\n\nclass TestFilterPolicy(rocksdb.interfaces.FilterPolicy):\n def create_filter(self, key"
},
{
"path": "rocksdb/universal_compaction.pxd",
"chars": 508,
"preview": "cdef extern from \"rocksdb/universal_compaction.h\" namespace \"rocksdb\":\n\n ctypedef enum CompactionStopStyle:\n k"
},
{
"path": "setup.py",
"chars": 1112,
"preview": "from setuptools import setup\nfrom setuptools import find_packages\nfrom distutils.extension import Extension\n\ntry:\n fr"
}
]
About this extraction
This page contains the full source code of the stephan-hof/pyrocksdb GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 48 files (203.9 KB), approximately 49.3k tokens, and a symbol index with 142 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.