[ { "path": ".gitignore", "content": "/build/\n/docs/_build/\n" }, { "path": "LICENSE.md", "content": "Copyright (c) 2014, Stephan Hofmockel\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice, this\n list of conditions and the following disclaimer in the documentation and/or\n other materials provided with the distribution.\n\n* Neither the name of the author nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR\nANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON\nANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "MANIFEST.in", "content": "include rocksdb/cpp/*.hpp\nrecursive-include rocksdb *.pxd\nrecursive-include rocksdb *.pyx\n" }, { "path": "README.rst", "content": "pyrocksdb\n=========\n\nPython bindings for RocksDB.\nSee http://pyrocksdb.readthedocs.org for a more comprehensive install and usage description.\n\n\nQuick Install\n-------------\n\nQuick install for debian/ubuntu like linux distributions.\n\n.. code-block:: bash\n\n $ apt-get install build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev\n $ git clone https://github.com/facebook/rocksdb.git\n $ cd rocksdb\n $ make shared_lib\n $ export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:`pwd`/include\n $ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:`pwd`\n $ export LIBRARY_PATH=${LIBRARY_PATH}:`pwd`\n\n $ cd ../\n $ apt-get install python-virtualenv python-dev\n $ virtualenv pyrocks_test\n $ cd pyrocks_test\n $ . bin/active\n $ pip install pyrocksdb\n\n\nQuick Usage Guide\n-----------------\n\n.. code-block:: pycon\n\n >>> import rocksdb\n >>> db = rocksdb.DB(\"test.db\", rocksdb.Options(create_if_missing=True))\n >>> db.put(b'a', b'data')\n >>> print db.get(b'a')\n b'data'\n" }, { "path": "docs/Makefile", "content": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nPAPER =\nBUILDDIR = _build\n\n# User-friendly check for sphinx-build\nifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)\n$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)\nendif\n\n# Internal variables.\nPAPEROPT_a4 = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n# the i18n builder cannot share the environment and doctrees with the others\nI18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n\n.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext\n\nhelp:\n\t@echo \"Please use \\`make ' where is one of\"\n\t@echo \" html to make standalone HTML files\"\n\t@echo \" dirhtml to make HTML files named index.html in directories\"\n\t@echo \" singlehtml to make a single large HTML file\"\n\t@echo \" pickle to make pickle files\"\n\t@echo \" json to make JSON files\"\n\t@echo \" htmlhelp to make HTML files and a HTML help project\"\n\t@echo \" qthelp to make HTML files and a qthelp project\"\n\t@echo \" devhelp to make HTML files and a Devhelp project\"\n\t@echo \" epub to make an epub\"\n\t@echo \" latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \" latexpdf to make LaTeX files and run them through pdflatex\"\n\t@echo \" latexpdfja to make LaTeX files and run them through platex/dvipdfmx\"\n\t@echo \" text to make text files\"\n\t@echo \" man to make manual pages\"\n\t@echo \" texinfo to make Texinfo files\"\n\t@echo \" info to make Texinfo files and run them through makeinfo\"\n\t@echo \" gettext to make PO message catalogs\"\n\t@echo \" changes to make an overview of all changed/added/deprecated items\"\n\t@echo \" xml to make Docutils-native XML files\"\n\t@echo \" pseudoxml to make pseudoxml-XML files for display purposes\"\n\t@echo \" linkcheck to check all external links for integrity\"\n\t@echo \" doctest to run all doctests embedded in the documentation (if enabled)\"\n\nclean:\n\trm -rf $(BUILDDIR)/*\n\nhtml:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\nsinglehtml:\n\t$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml\n\t@echo\n\t@echo \"Build finished. The HTML page is in $(BUILDDIR)/singlehtml.\"\n\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/rocksdb.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/rocksdb.qhc\"\n\ndevhelp:\n\t$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp\n\t@echo\n\t@echo \"Build finished.\"\n\t@echo \"To view the help file:\"\n\t@echo \"# mkdir -p $$HOME/.local/share/devhelp/rocksdb\"\n\t@echo \"# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/rocksdb\"\n\t@echo \"# devhelp\"\n\nepub:\n\t$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub\n\t@echo\n\t@echo \"Build finished. The epub file is in $(BUILDDIR)/epub.\"\n\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make' in that directory to run these through (pdf)latex\" \\\n\t \"(use \\`make latexpdf' here to do that automatically).\"\n\nlatexpdf:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through pdflatex...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\nlatexpdfja:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through platex and dvipdfmx...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\ntext:\n\t$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text\n\t@echo\n\t@echo \"Build finished. The text files are in $(BUILDDIR)/text.\"\n\nman:\n\t$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(BUILDDIR)/man.\"\n\ntexinfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo\n\t@echo \"Build finished. The Texinfo files are in $(BUILDDIR)/texinfo.\"\n\t@echo \"Run \\`make' in that directory to run these through makeinfo\" \\\n\t \"(use \\`make info' here to do that automatically).\"\n\ninfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo \"Running Texinfo files through makeinfo...\"\n\tmake -C $(BUILDDIR)/texinfo info\n\t@echo \"makeinfo finished; the Info files are in $(BUILDDIR)/texinfo.\"\n\ngettext:\n\t$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale\n\t@echo\n\t@echo \"Build finished. The message catalogs are in $(BUILDDIR)/locale.\"\n\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/doctest/output.txt.\"\n\nxml:\n\t$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml\n\t@echo\n\t@echo \"Build finished. The XML files are in $(BUILDDIR)/xml.\"\n\npseudoxml:\n\t$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml\n\t@echo\n\t@echo \"Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml.\"\n" }, { "path": "docs/api/backup.rst", "content": "Backup and Restore\n******************\n\nBackupEngine\n============\n\n.. py:class:: rocksdb.BackupEngine\n\n .. py:method:: __init__(backup_dir)\n\n Creates a object to manage backup of a single database.\n\n :param unicode backup_dir: Where to keep the backup files.\n Has to be different than db.db_name.\n For example db.db_name + '/backups'.\n\n .. py:method:: create_backup(db, flush_before_backup=False)\n\n Triggers the creation of a backup.\n\n :param db: Database object to backup.\n :type db: :py:class:`rocksdb.DB`\n\n :param bool flush_before_backup: If ``True`` the current memtable is flushed.\n\n .. py:method:: restore_backup(backup_id, db_dir, wal_dir)\n\n Restores the backup from the given id.\n\n :param int backup_id: id of the backup to restore.\n :param unicode db_dir: Target directory to restore backup.\n :param unicode wal_dir: Target directory to restore backuped WAL files.\n\n .. py:method:: restore_latest_backup(db_dir, wal_dir)\n\n Restores the latest backup.\n\n :param unicode db_dir: see :py:meth:`restore_backup`\n :param unicode wal_dir: see :py:meth:`restore_backup`\n\n .. py:method:: stop_backup()\n\n Can be called from another thread to stop the current backup process.\n\n .. py:method:: purge_old_backups(num_backups_to_keep)\n\n Deletes all backups (oldest first) until \"num_backups_to_keep\" are left.\n\n :param int num_backups_to_keep: Number of backupfiles to keep.\n\n .. py:method:: delete_backup(backup_id)\n\n :param int backup_id: Delete the backup with the given id.\n\n .. py:method:: get_backup_info()\n\n Returns information about all backups.\n\n It returns a list of dict's where each dict as the following keys.\n\n ``backup_id``\n (int): id of this backup.\n\n ``timestamp``\n (int): Seconds since epoch, when the backup was created.\n\n ``size``\n (int): Size in bytes of the backup.\n" }, { "path": "docs/api/database.rst", "content": "Database interactions\n*********************\n\nDatabase object\n===============\n\n.. py:class:: rocksdb.DB\n\n .. py:method:: __init__(db_name, Options opts, read_only=False)\n\n :param unicode db_name: Name of the database to open\n :param opts: Options for this specific database\n :type opts: :py:class:`rocksdb.Options`\n :param bool read_only: If ``True`` the database is opened read-only.\n All DB calls which modify data will raise an\n Exception.\n\n\n .. py:method:: put(key, value, sync=False, disable_wal=False)\n\n Set the database entry for \"key\" to \"value\".\n\n :param bytes key: Name for this entry\n :param bytes value: Data for this entry\n :param bool sync: \n If ``True``, the write will be flushed from the operating system\n buffer cache (by calling WritableFile::Sync()) before the write\n is considered complete. If this flag is true, writes will be\n slower.\n\n If this flag is ``False``, and the machine crashes, some recent\n writes may be lost. Note that if it is just the process that\n crashes (i.e., the machine does not reboot), no writes will be\n lost even if ``sync == False``.\n\n In other words, a DB write with ``sync == False`` has similar\n crash semantics as the \"write()\" system call. A DB write\n with ``sync == True`` has similar crash semantics to a \"write()\"\n system call followed by \"fdatasync()\".\n\n :param bool disable_wal:\n If ``True``, writes will not first go to the write ahead log,\n and the write may got lost after a crash.\n\n .. py:method:: delete(key, sync=False, disable_wal=False)\n\n Remove the database entry for \"key\".\n\n :param bytes key: Name to delete\n :param sync: See :py:meth:`rocksdb.DB.put`\n :param disable_wal: See :py:meth:`rocksdb.DB.put`\n :raises rocksdb.errors.NotFound: If the key did not exists\n\n .. py:method:: merge(key, value, sync=False, disable_wal=False)\n\n Merge the database entry for \"key\" with \"value\".\n The semantics of this operation is determined by the user provided\n merge_operator when opening DB.\n\n See :py:meth:`rocksdb.DB.put` for the parameters\n\n :raises:\n :py:exc:`rocksdb.errors.NotSupported` if this is called and\n no :py:attr:`rocksdb.Options.merge_operator` was set at creation\n\n\n .. py:method:: write(batch, sync=False, disable_wal=False)\n\n Apply the specified updates to the database.\n\n :param rocksdb.WriteBatch batch: Batch to apply\n :param sync: See :py:meth:`rocksdb.DB.put`\n :param disable_wal: See :py:meth:`rocksdb.DB.put`\n\n .. py:method:: get(key, verify_checksums=False, fill_cache=True, snapshot=None, read_tier=\"all\")\n\n :param bytes key: Name to get\n\n :param bool verify_checksums: \n If ``True``, all data read from underlying storage will be\n verified against corresponding checksums.\n\n :param bool fill_cache:\n Should the \"data block\", \"index block\" or \"filter block\"\n read for this iteration be cached in memory?\n Callers may wish to set this field to ``False`` for bulk scans.\n \n :param snapshot:\n If not ``None``, read as of the supplied snapshot\n (which must belong to the DB that is being read and which must\n not have been released). Is it ``None`` a implicit snapshot of the\n state at the beginning of this read operation is used\n :type snapshot: :py:class:`rocksdb.Snapshot`\n\n :param string read_tier:\n Specify if this read request should process data that ALREADY\n resides on a particular cache. If the required data is not\n found at the specified cache,\n then :py:exc:`rocksdb.errors.Incomplete` is raised.\n\n | Use ``all`` if a fetch from disk is allowed.\n | Use ``cache`` if only data from cache is allowed.\n \n :returns: ``None`` if not found, else the value for this key\n\n .. py:method:: multi_get(keys, verify_checksums=False, fill_cache=True, snapshot=None, read_tier=\"all\")\n\n :param keys: Keys to fetch\n :type keys: list of bytes\n\n For the other params see :py:meth:`rocksdb.DB.get`\n\n :returns:\n A ``dict`` where the value is either ``bytes`` or ``None`` if not found\n\n :raises: If the fetch for a single key fails\n \n .. note::\n keys will not be \"de-duplicated\".\n Duplicate keys will return duplicate values in order.\n\n .. py:method:: key_may_exist(key, fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier=\"all\")\n\n If the key definitely does not exist in the database, then this method\n returns ``False``, else ``True``. If the caller wants to obtain value\n when the key is found in memory, fetch should be set to ``True``.\n This check is potentially lighter-weight than invoking DB::get().\n One way to make this lighter weight is to avoid doing any IOs.\n\n :param bytes key: Key to check\n :param bool fetch: Obtain also the value if found\n\n For the other params see :py:meth:`rocksdb.DB.get`\n\n :returns: \n * ``(True, None)`` if key is found but value not in memory\n * ``(True, None)`` if key is found and ``fetch=False``\n * ``(True, )`` if key is found and value in memory and ``fetch=True``\n * ``(False, None)`` if key is not found\n\n .. py:method:: iterkeys(fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier=\"all\")\n\n Iterate over the keys\n\n For other params see :py:meth:`rocksdb.DB.get`\n\n :returns:\n A iterator object which is not valid yet.\n Call first one of the seek methods of the iterator to position it\n\n :rtype: :py:class:`rocksdb.BaseIterator`\n\n .. py:method:: itervalues(fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier=\"all\")\n\n Iterate over the values\n\n For other params see :py:meth:`rocksdb.DB.get`\n\n :returns:\n A iterator object which is not valid yet.\n Call first one of the seek methods of the iterator to position it\n\n :rtype: :py:class:`rocksdb.BaseIterator`\n\n .. py:method:: iteritems(fetch=False, verify_checksums=False, fill_cache=True, snapshot=None, read_tier=\"all\")\n\n Iterate over the items\n\n For other params see :py:meth:`rocksdb.DB.get`\n\n :returns:\n A iterator object which is not valid yet.\n Call first one of the seek methods of the iterator to position it\n\n :rtype: :py:class:`rocksdb.BaseIterator`\n\n .. py:method:: snapshot()\n \n Return a handle to the current DB state.\n Iterators created with this handle will all observe a stable snapshot\n of the current DB state.\n \n :rtype: :py:class:`rocksdb.Snapshot`\n\n\n .. py:method:: get_property(prop)\n\n DB implementations can export properties about their state\n via this method. If \"property\" is a valid property understood by this\n DB implementation, a byte string with its value is returned.\n Otherwise ``None``\n \n Valid property names include:\n \n * ``b\"rocksdb.num-files-at-level\"``: return the number of files at level ,\n where is an ASCII representation of a level number (e.g. \"0\").\n\n * ``b\"rocksdb.stats\"``: returns a multi-line byte string that describes statistics\n about the internal operation of the DB.\n\n * ``b\"rocksdb.sstables\"``: returns a multi-line byte string that describes all\n of the sstables that make up the db contents.\n\n * ``b\"rocksdb.num-immutable-mem-table\"``: Number of immutable mem tables.\n\n * ``b\"rocksdb.mem-table-flush-pending\"``: Returns ``1`` if mem table flush is pending, otherwise ``0``.\n\n * ``b\"rocksdb.compaction-pending\"``: Returns ``1`` if a compaction is pending, otherweise ``0``.\n\n * ``b\"rocksdb.background-errors\"``: Returns accumulated background errors encountered.\n\n * ``b\"rocksdb.cur-size-active-mem-table\"``: Returns current size of the active memtable.\n\n .. py:method:: get_live_files_metadata()\n\n Returns a list of all table files.\n\n It returns a list of dict's were each dict has the following keys.\n\n ``name``\n Name of the file\n\n ``level``\n Level at which this file resides\n\n ``size``\n File size in bytes\n\n ``smallestkey``\n Smallest user defined key in the file\n\n ``largestkey``\n Largest user defined key in the file\n\n ``smallest_seqno``\n smallest seqno in file\n\n ``largest_seqno``\n largest seqno in file\n\n .. py:method:: compact_range(begin=None, end=None, ** options)\n\n Compact the underlying storage for the key range [begin,end].\n The actual compaction interval might be superset of [begin, end].\n In particular, deleted and overwritten versions are discarded,\n and the data is rearranged to reduce the cost of operations\n needed to access the data.\n\n This operation should typically only be invoked by users who understand\n the underlying implementation.\n\n ``begin == None`` is treated as a key before all keys in the database.\n ``end == None`` is treated as a key after all keys in the database.\n Therefore the following call will compact the entire database: ``db.compact_range()``.\n\n Note that after the entire database is compacted, all data are pushed\n down to the last level containing any data. If the total data size\n after compaction is reduced, that level might not be appropriate for\n hosting all the files. In this case, client could set change_level\n to ``True``, to move the files back to the minimum level capable of holding\n the data set or a given level (specified by non-negative target_level).\n\n :param bytes begin: Key where to start compaction.\n If ``None`` start at the beginning of the database.\n :param bytes end: Key where to end compaction.\n If ``None`` end at the last key of the database.\n :param bool change_level: If ``True``, compacted files will be moved to\n the minimum level capable of holding the data\n or given level (specified by non-negative target_level).\n If ``False`` you may end with a bigger level\n than configured. Default is ``False``.\n :param int target_level: If change_level is true and target_level have non-negative\n value, compacted files will be moved to target_level.\n Default is ``-1``.\n :param string bottommost_level_compaction:\n For level based compaction, we can configure if we want to\n skip/force bottommost level compaction. By default level based\n compaction will only compact the bottommost level if there is a\n compaction filter. It can be set to the following values.\n\n ``skip``\n Skip bottommost level compaction\n\n ``if_compaction_filter``\n Only compact bottommost level if there is a compaction filter.\n This is the default.\n\n ``force``\n Always compact bottommost level\n \n .. py:attribute:: options\n\n Returns the associated :py:class:`rocksdb.Options` instance.\n\n .. note::\n\n Changes to this object have no effect anymore.\n Consider this as read-only\n\nIterator\n========\n\n.. py:class:: rocksdb.BaseIterator\n\n Base class for all iterators in this module. After creation a iterator is\n invalid. Call one of the seek methods first before starting iteration\n\n .. py:method:: seek_to_first()\n\n Position at the first key in the source\n\n .. py:method:: seek_to_last()\n \n Position at the last key in the source\n\n .. py:method:: seek(key)\n \n :param bytes key: Position at the first key in the source that at or past\n \n Methods to support the python iterator protocol\n\n .. py:method:: __iter__()\n .. py:method:: __next__()\n .. py:method:: __reversed__()\n\nSnapshot\n========\n\n.. py:class:: rocksdb.Snapshot\n\n Opaque handler for a single Snapshot.\n Snapshot is released if nobody holds a reference on it.\n Retrieved via :py:meth:`rocksdb.DB.snapshot`\n\nWriteBatch\n==========\n\n.. py:class:: rocksdb.WriteBatch\n\n WriteBatch holds a collection of updates to apply atomically to a DB.\n\n The updates are applied in the order in which they are added\n to the WriteBatch. For example, the value of \"key\" will be \"v3\"\n after the following batch is written::\n \n batch = rocksdb.WriteBatch()\n batch.put(b\"key\", b\"v1\")\n batch.delete(b\"key\")\n batch.put(b\"key\", b\"v2\")\n batch.put(b\"key\", b\"v3\")\n\n .. py:method:: __init__(data=None)\n\n Creates a WriteBatch.\n\n :param bytes data:\n A serialized version of a previous WriteBatch. As retrieved\n from a previous .data() call. If ``None`` a empty WriteBatch is\n generated\n\n .. py:method:: put(key, value)\n \n Store the mapping \"key->value\" in the database.\n\n :param bytes key: Name of the entry to store\n :param bytes value: Data of this entry\n\n .. py:method:: merge(key, value)\n \n Merge \"value\" with the existing value of \"key\" in the database.\n\n :param bytes key: Name of the entry to merge\n :param bytes value: Data to merge\n\n .. py:method:: delete(key)\n \n If the database contains a mapping for \"key\", erase it. Else do nothing.\n\n :param bytes key: Key to erase\n\n .. py:method:: clear()\n\n Clear all updates buffered in this batch.\n\n .. note::\n Don't call this method if there is an outstanding iterator.\n Calling :py:meth:`rocksdb.WriteBatch.clear()` with outstanding\n iterator, leads to SEGFAULT.\n\n .. py:method:: data()\n\n Retrieve the serialized version of this batch.\n\n :rtype: ``bytes``\n\n .. py:method:: count()\n \n Returns the number of updates in the batch\n\n :rtype: int\n\n .. py:method:: __iter__()\n\n Returns an iterator over the current contents of the write batch.\n\n If you add new items to the batch, they are not visible for this\n iterator. Create a new one if you need to see them.\n\n .. note::\n Calling :py:meth:`rocksdb.WriteBatch.clear()` on the write batch\n invalidates the iterator. Using a iterator where its corresponding\n write batch has been cleared, leads to SEGFAULT.\n\n :rtype: :py:class:`rocksdb.WriteBatchIterator`\n\nWriteBatchIterator\n==================\n\n.. py:class:: rocksdb.WriteBatchIterator\n\n .. py:method:: __iter__()\n\n Returns self.\n\n .. py:method:: __next__()\n\n Returns the next item inside the corresponding write batch.\n The return value is a tuple of always size three.\n\n First item (Name of the operation):\n\n * ``\"Put\"``\n * ``\"Merge\"``\n * ``\"Delete\"``\n\n Second item (key):\n Key for this operation.\n\n Third item (value):\n The value for this operation. Empty for ``\"Delete\"``.\n\nRepair DB\n=========\n\n.. py:function:: repair_db(db_name, opts)\n\n :param unicode db_name: Name of the database to open\n :param opts: Options for this specific database\n :type opts: :py:class:`rocksdb.Options`\n\n If a DB cannot be opened, you may attempt to call this method to\n resurrect as much of the contents of the database as possible.\n Some data may be lost, so be careful when calling this function\n on a database that contains important information.\n\n\nErrors\n======\n\n.. py:exception:: rocksdb.errors.NotFound\n.. py:exception:: rocksdb.errors.Corruption\n.. py:exception:: rocksdb.errors.NotSupported\n.. py:exception:: rocksdb.errors.InvalidArgument\n.. py:exception:: rocksdb.errors.RocksIOError\n.. py:exception:: rocksdb.errors.MergeInProgress\n.. py:exception:: rocksdb.errors.Incomplete\n\n\n" }, { "path": "docs/api/index.rst", "content": "Python driver for RocksDB\n=========================\n\n .. py:module:: rocksdb\n\n.. toctree::\n\n Options \n Database \n Interfaces \n Backup \n" }, { "path": "docs/api/interfaces.rst", "content": "Interfaces\n**********\n\nComparator\n==========\n\n.. py:class:: rocksdb.interfaces.Comparator\n\n A Comparator object provides a total order across slices that are\n used as keys in an sstable or a database. A Comparator implementation\n must be thread-safe since rocksdb may invoke its methods concurrently\n from multiple threads.\n\n .. py:method:: compare(a, b)\n\n Three-way comparison.\n\n :param bytes a: First field to compare\n :param bytes b: Second field to compare\n :returns: * -1 if a < b\n * 0 if a == b\n * 1 if a > b\n :rtype: ``int``\n\n .. py:method:: name()\n\n The name of the comparator. Used to check for comparator\n mismatches (i.e., a DB created with one comparator is\n accessed using a different comparator).\n\n The client of this package should switch to a new name whenever\n the comparator implementation changes in a way that will cause\n the relative ordering of any two keys to change.\n\n Names starting with \"rocksdb.\" are reserved and should not be used\n by any clients of this package.\n\n :rtype: ``bytes``\n\nMerge Operator\n==============\n\n Essentially, a MergeOperator specifies the SEMANTICS of a merge, which only\n client knows. It could be numeric addition, list append, string\n concatenation, edit data structure, whatever.\n The library, on the other hand, is concerned with the exercise of this\n interface, at the right time (during get, iteration, compaction...)\n\n To use merge, the client needs to provide an object implementing one of\n the following interfaces:\n\n * AssociativeMergeOperator - for most simple semantics (always take\n two values, and merge them into one value, which is then put back\n into rocksdb).\n numeric addition and string concatenation are examples.\n\n * MergeOperator - the generic class for all the more complex operations.\n One method (FullMerge) to merge a Put/Delete value with a merge operand.\n Another method (PartialMerge) that merges two operands together.\n This is especially useful if your key values have a complex structure but\n you would still like to support client-specific incremental updates.\n\n AssociativeMergeOperator is simpler to implement.\n MergeOperator is simply more powerful.\n\n See this page for more details\n https://github.com/facebook/rocksdb/wiki/Merge-Operator\n\nAssociativeMergeOperator\n------------------------\n\n.. py:class:: rocksdb.interfaces.AssociativeMergeOperator\n\n .. py:method:: merge(key, existing_value, value)\n\n Gives the client a way to express the read -> modify -> write semantics\n\n :param bytes key: The key that's associated with this merge operation\n :param bytes existing_value: The current value in the db.\n ``None`` indicates the key does not exist\n before this op\n :param bytes value: The value to update/merge the existing_value with\n\n :returns: ``True`` and the new value on success.\n All values passed in will be client-specific values.\n So if this method returns false, it is because client\n specified bad data or there was internal corruption.\n The client should assume that this will be treated as an\n error by the library.\n\n :rtype: ``(bool, bytes)``\n\n .. py:method:: name()\n\n The name of the MergeOperator. Used to check for MergeOperator mismatches.\n For example a DB created with one MergeOperator is accessed using a\n different MergeOperator.\n\n :rtype: ``bytes``\n\nMergeOperator\n-------------\n\n.. py:class:: rocksdb.interfaces.MergeOperator\n\n .. py:method:: full_merge(key, existing_value, operand_list)\n\n Gives the client a way to express the read -> modify -> write semantics\n\n :param bytes key: The key that's associated with this merge operation.\n Client could multiplex the merge operator based on it\n if the key space is partitioned and different subspaces\n refer to different types of data which have different\n merge operation semantics\n\n :param bytes existing_value: The current value in the db.\n ``None`` indicates the key does not exist\n before this op\n\n :param operand_list: The sequence of merge operations to apply.\n :type operand_list: list of bytes \n\n :returns: ``True`` and the new value on success.\n All values passed in will be client-specific values.\n So if this method returns false, it is because client\n specified bad data or there was internal corruption.\n The client should assume that this will be treated as an\n error by the library.\n\n :rtype: ``(bool, bytes)``\n\n .. py:method:: partial_merge(key, left_operand, right_operand)\n\n This function performs merge(left_op, right_op)\n when both the operands are themselves merge operation types\n that you would have passed to a DB::Merge() call in the same order.\n For example DB::Merge(key,left_op), followed by DB::Merge(key,right_op)).\n\n PartialMerge should combine them into a single merge operation that is\n returned together with ``True``\n This new value should be constructed such that a call to\n DB::Merge(key, new_value) would yield the same result as a call\n to DB::Merge(key, left_op) followed by DB::Merge(key, right_op).\n\n If it is impossible or infeasible to combine the two operations,\n return ``(False, None)`` The library will internally keep track of the\n operations, and apply them in the correct order once a base-value\n (a Put/Delete/End-of-Database) is seen.\n\n :param bytes key: the key that is associated with this merge operation.\n :param bytes left_operand: First operand to merge\n :param bytes right_operand: Second operand to merge\n :rtype: ``(bool, bytes)``\n\n .. note::\n\n Presently there is no way to differentiate between error/corruption\n and simply \"return false\". For now, the client should simply return\n false in any case it cannot perform partial-merge, regardless of reason.\n If there is corruption in the data, handle it in the FullMerge() function,\n and return false there.\n\n .. py:method:: name()\n\n The name of the MergeOperator. Used to check for MergeOperator mismatches.\n For example a DB created with one MergeOperator is accessed using a\n different MergeOperator.\n\n :rtype: ``bytes``\n\nFilterPolicy\n============\n\n.. py:class:: rocksdb.interfaces.FilterPolicy\n\n .. py:method:: create_filter(keys)\n\n Create a bytestring which can act as a filter for keys.\n\n :param keys: list of keys (potentially with duplicates)\n that are ordered according to the user supplied\n comparator. \n :type keys: list of bytes\n\n :returns: A filter that summarizes keys\n :rtype: ``bytes``\n\n .. py:method:: key_may_match(key, filter)\n\n Check if the key is maybe in the filter. \n\n :param bytes key: Key for a single entry inside the database\n :param bytes filter: Contains the data returned by a preceding call\n to create_filter on this class\n :returns: This method must return ``True`` if the key was in the list\n of keys passed to create_filter().\n This method may return ``True`` or ``False`` if the key was\n not on the list, but it should aim to return ``False`` with\n a high probability.\n :rtype: ``bool``\n\n \n .. py:method:: name()\n\n Return the name of this policy. Note that if the filter encoding\n changes in an incompatible way, the name returned by this method\n must be changed. Otherwise, old incompatible filters may be\n passed to methods of this type.\n\n :rtype: ``bytes``\n\n\nSliceTransform\n==============\n\n.. py:class:: rocksdb.interfaces.SliceTransform\n\n SliceTransform is currently used to implement the 'prefix-API' of rocksdb.\n https://github.com/facebook/rocksdb/wiki/Proposal-for-prefix-API\n\n .. py:method:: transform(src)\n\n :param bytes src: Full key to extract the prefix from.\n\n :returns: A tuple of two interges ``(offset, size)``.\n Where the first integer is the offset within the ``src``\n and the second the size of the prefix after the offset.\n Which means the prefix is generted by ``src[offset:offset+size]``\n\n :rtype: ``(int, int)``\n\n\n .. py:method:: in_domain(src)\n\n Decide if a prefix can be extraced from ``src``.\n Only if this method returns ``True`` :py:meth:`transform` will be\n called.\n\n :param bytes src: Full key to check.\n :rtype: ``bool``\n\n .. py:method:: in_range(prefix)\n\n Checks if prefix is a valid prefix\n\n :param bytes prefix: Prefix to check.\n :returns: ``True`` if ``prefix`` is a valid prefix.\n :rtype: ``bool``\n\n .. py:method:: name()\n\n Return the name of this transformation.\n\n :rtype: ``bytes``\n" }, { "path": "docs/api/options.rst", "content": "Options creation\n****************\n\nOptions object\n==============\n\n\n.. py:class:: rocksdb.Options\n\n .. IMPORTANT:: \n\n The default values mentioned here, describe the values of the\n C++ library only. This wrapper does not set any default value\n itself. So as soon as the rocksdb developers change a default value\n this document could be outdated. So if you really depend on a default\n value, double check it with the according version of the C++ library.\n\n | Most recent default values should be here\n | https://github.com/facebook/rocksdb/blob/master/include/rocksdb/options.h\n | https://github.com/facebook/rocksdb/blob/master/util/options.cc\n \n .. py:method:: __init__(**kwargs)\n\n All options mentioned below can also be passed as keyword-arguments in\n the constructor. For example::\n\n import rocksdb\n\n opts = rocksdb.Options(create_if_missing=True)\n # is the same as\n opts = rocksdb.Options()\n opts.create_if_missing = True\n\n\n .. py:attribute:: create_if_missing\n\n If ``True``, the database will be created if it is missing.\n\n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: error_if_exists\n\n If ``True``, an error is raised if the database already exists.\n\n | *Type:* ``bool``\n | *Default:* ``False``\n\n\n .. py:attribute:: paranoid_checks\n\n If ``True``, the implementation will do aggressive checking of the\n data it is processing and will stop early if it detects any\n errors. This may have unforeseen ramifications: for example, a\n corruption of one DB entry may cause a large number of entries to\n become unreadable or for the entire DB to become unopenable.\n If any of the writes to the database fails (Put, Delete, Merge, Write),\n the database will switch to read-only mode and fail all other\n Write operations.\n\n | *Type:* ``bool``\n | *Default:* ``True``\n\n .. py:attribute:: write_buffer_size\n\n Amount of data to build up in memory (backed by an unsorted log\n on disk) before converting to a sorted on-disk file.\n\n Larger values increase performance, especially during bulk loads.\n Up to max_write_buffer_number write buffers may be held in memory\n at the same time, so you may wish to adjust this parameter to control\n memory usage. Also, a larger write buffer will result in a longer recovery\n time the next time the database is opened.\n\n | *Type:* ``int``\n | *Default:* ``4194304``\n\n .. py:attribute:: max_write_buffer_number\n\n The maximum number of write buffers that are built up in memory.\n The default is 2, so that when 1 write buffer is being flushed to\n storage, new writes can continue to the other write buffer.\n\n | *Type:* ``int``\n | *Default:* ``2``\n\n .. py:attribute:: min_write_buffer_number_to_merge\n\n The minimum number of write buffers that will be merged together\n before writing to storage. If set to 1, then\n all write buffers are fushed to L0 as individual files and this increases\n read amplification because a get request has to check in all of these\n files. Also, an in-memory merge may result in writing lesser\n data to storage if there are duplicate records in each of these\n individual write buffers.\n\n | *Type:* ``int``\n | *Default:* ``1``\n\n .. py:attribute:: max_open_files\n\n Number of open files that can be used by the DB. You may need to\n increase this if your database has a large working set. Value -1 means\n files opened are always kept open. You can estimate number of\n files based on target_file_size_base and target_file_size_multiplier\n for level-based compaction.\n For universal-style compaction, you can usually set it to -1.\n\n | *Type:* ``int``\n | *Default:* ``5000``\n\n .. py:attribute:: compression\n\n Compress blocks using the specified compression algorithm.\n This parameter can be changed dynamically.\n\n | *Type:* Member of :py:class:`rocksdb.CompressionType`\n | *Default:* :py:attr:`rocksdb.CompressionType.snappy_compression`\n\n .. py:attribute:: num_levels\n\n Number of levels for this database\n\n | *Type:* ``int``\n | *Default:* ``7``\n\n\n .. py:attribute:: level0_file_num_compaction_trigger\n\n Number of files to trigger level-0 compaction. A value <0 means that\n level-0 compaction will not be triggered by number of files at all.\n\n | *Type:* ``int``\n | *Default:* ``4``\n\n .. py:attribute:: level0_slowdown_writes_trigger\n\n Soft limit on number of level-0 files. We start slowing down writes at this\n point. A value <0 means that no writing slow down will be triggered by\n number of files in level-0.\n\n | *Type:* ``int``\n | *Default:* ``20``\n\n .. py:attribute:: level0_stop_writes_trigger\n\n Maximum number of level-0 files. We stop writes at this point.\n\n | *Type:* ``int``\n | *Default:* ``24``\n\n .. py:attribute:: max_mem_compaction_level\n\n Maximum level to which a new compacted memtable is pushed if it\n does not create overlap. We try to push to level 2 to avoid the\n relatively expensive level 0=>1 compactions and to avoid some\n expensive manifest file operations. We do not push all the way to\n the largest level since that can generate a lot of wasted disk\n space if the same key space is being repeatedly overwritten.\n\n | *Type:* ``int``\n | *Default:* ``2``\n\n\n .. py:attribute:: target_file_size_base\n\n | Target file size for compaction.\n | target_file_size_base is per-file size for level-1.\n | Target file size for level L can be calculated by\n | target_file_size_base * (target_file_size_multiplier ^ (L-1)).\n\n For example, if target_file_size_base is 2MB and\n target_file_size_multiplier is 10, then each file on level-1 will\n be 2MB, and each file on level 2 will be 20MB,\n and each file on level-3 will be 200MB.\n\n | *Type:* ``int``\n | *Default:* ``2097152``\n\n .. py:attribute:: target_file_size_multiplier\n\n | by default target_file_size_multiplier is 1, which means\n | by default files in different levels will have similar size.\n\n | *Type:* ``int``\n | *Default:* ``1``\n\n .. py:attribute:: max_bytes_for_level_base\n\n Control maximum total data size for a level.\n *max_bytes_for_level_base* is the max total for level-1.\n Maximum number of bytes for level L can be calculated as\n (*max_bytes_for_level_base*) * (*max_bytes_for_level_multiplier* ^ (L-1))\n For example, if *max_bytes_for_level_base* is 20MB, and if\n *max_bytes_for_level_multiplier* is 10, total data size for level-1\n will be 20MB, total file size for level-2 will be 200MB,\n and total file size for level-3 will be 2GB.\n\n | *Type:* ``int``\n | *Default:* ``10485760``\n\n .. py:attribute:: max_bytes_for_level_multiplier\n\n See :py:attr:`max_bytes_for_level_base`\n\n | *Type:* ``int``\n | *Default:* ``10``\n\n .. py:attribute:: max_bytes_for_level_multiplier_additional\n\n Different max-size multipliers for different levels.\n These are multiplied by max_bytes_for_level_multiplier to arrive\n at the max-size of each level.\n\n | *Type:* ``[int]``\n | *Default:* ``[1, 1, 1, 1, 1, 1, 1]``\n\n .. py:attribute:: expanded_compaction_factor\n\n Maximum number of bytes in all compacted files. We avoid expanding\n the lower level file set of a compaction if it would make the\n total compaction cover more than\n (expanded_compaction_factor * targetFileSizeLevel()) many bytes.\n \n | *Type:* ``int``\n | *Default:* ``25``\n\n .. py:attribute:: source_compaction_factor\n\n Maximum number of bytes in all source files to be compacted in a\n single compaction run. We avoid picking too many files in the\n source level so that we do not exceed the total source bytes\n for compaction to exceed\n (source_compaction_factor * targetFileSizeLevel()) many bytes.\n If 1 pick maxfilesize amount of data as the source of\n a compaction.\n\n | *Type:* ``int``\n | *Default:* ``1``\n\n .. py:attribute:: max_grandparent_overlap_factor\n\n Control maximum bytes of overlaps in grandparent (i.e., level+2) before we\n stop building a single file in a level->level+1 compaction.\n\n | *Type:* ``int``\n | *Default:* ``10``\n\n .. py:attribute:: disable_data_sync\n\n If true, then the contents of data files are not synced\n to stable storage. Their contents remain in the OS buffers till the\n OS decides to flush them. This option is good for bulk-loading\n of data. Once the bulk-loading is complete, please issue a\n sync to the OS to flush all dirty buffesrs to stable storage.\n\n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: use_fsync\n\n If true, then every store to stable storage will issue a fsync.\n If false, then every store to stable storage will issue a fdatasync.\n This parameter should be set to true while storing data to\n filesystem like ext3 that can lose files after a reboot.\n\n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: db_log_dir\n\n This specifies the info LOG dir.\n If it is empty, the log files will be in the same dir as data.\n If it is non empty, the log files will be in the specified dir,\n and the db data dir's absolute path will be used as the log file\n name's prefix.\n\n | *Type:* ``unicode``\n | *Default:* ``\"\"``\n\n .. py:attribute:: wal_dir\n\n This specifies the absolute dir path for write-ahead logs (WAL).\n If it is empty, the log files will be in the same dir as data,\n dbname is used as the data dir by default.\n If it is non empty, the log files will be in kept the specified dir.\n When destroying the db, all log files in wal_dir and the dir itself is deleted\n\n | *Type:* ``unicode``\n | *Default:* ``\"\"``\n\n .. py:attribute:: delete_obsolete_files_period_micros\n\n The periodicity when obsolete files get deleted. The default\n value is 6 hours. The files that get out of scope by compaction\n process will still get automatically delete on every compaction,\n regardless of this setting\n\n | *Type:* ``int``\n | *Default:* ``21600000000``\n\n .. py:attribute:: max_background_compactions\n\n Maximum number of concurrent background jobs, submitted to\n the default LOW priority thread pool\n\n | *Type:* ``int``\n | *Default:* ``1``\n\n .. py:attribute:: max_background_flushes\n\n Maximum number of concurrent background memtable flush jobs, submitted to\n the HIGH priority thread pool.\n By default, all background jobs (major compaction and memtable flush) go\n to the LOW priority pool. If this option is set to a positive number,\n memtable flush jobs will be submitted to the HIGH priority pool.\n It is important when the same Env is shared by multiple db instances.\n Without a separate pool, long running major compaction jobs could\n potentially block memtable flush jobs of other db instances, leading to\n unnecessary Put stalls.\n\n | *Type:* ``int``\n | *Default:* ``1``\n\n .. py:attribute:: max_log_file_size\n\n Specify the maximal size of the info log file. If the log file\n is larger than `max_log_file_size`, a new info log file will\n be created.\n If max_log_file_size == 0, all logs will be written to one\n log file.\n\n | *Type:* ``int``\n | *Default:* ``0``\n\n .. py:attribute:: log_file_time_to_roll\n\n Time for the info log file to roll (in seconds).\n If specified with non-zero value, log file will be rolled\n if it has been active longer than `log_file_time_to_roll`.\n A value of ``0`` means disabled.\n\n | *Type:* ``int``\n | *Default:* ``0``\n\n .. py:attribute:: keep_log_file_num\n\n Maximal info log files to be kept.\n\n | *Type:* ``int``\n | *Default:* ``1000``\n\n .. py:attribute:: soft_rate_limit\n\n Puts are delayed 0-1 ms when any level has a compaction score that exceeds\n soft_rate_limit. This is ignored when == 0.0.\n CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not\n hold, RocksDB will set soft_rate_limit = hard_rate_limit.\n A value of ``0`` means disabled.\n\n | *Type:* ``float``\n | *Default:* ``0``\n\n .. py:attribute:: hard_rate_limit\n\n Puts are delayed 1ms at a time when any level has a compaction score that\n exceeds hard_rate_limit. This is ignored when <= 1.0.\n A value fo ``0`` means disabled.\n\n | *Type:* ``float``\n | *Default:* ``0``\n\n .. py:attribute:: rate_limit_delay_max_milliseconds\n\n Max time a put will be stalled when hard_rate_limit is enforced. If 0, then\n there is no limit.\n\n | *Type:* ``int``\n | *Default:* ``1000``\n\n .. py:attribute:: max_manifest_file_size\n\n manifest file is rolled over on reaching this limit.\n The older manifest file be deleted.\n The default value is MAX_INT so that roll-over does not take place.\n\n | *Type:* ``int``\n | *Default:* ``(2**64) - 1``\n\n .. py:attribute:: table_cache_numshardbits\n\n Number of shards used for table cache.\n\n | *Type:* ``int``\n | *Default:* ``4``\n\n .. py:attribute:: arena_block_size\n\n size of one block in arena memory allocation.\n If <= 0, a proper value is automatically calculated (usually 1/10 of\n writer_buffer_size).\n \n | *Type:* ``int``\n | *Default:* ``0``\n\n .. py:attribute:: disable_auto_compactions\n\n Disable automatic compactions. Manual compactions can still\n be issued on this database.\n \n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: wal_ttl_seconds, wal_size_limit_mb\n\n The following two fields affect how archived logs will be deleted.\n\n 1. If both set to 0, logs will be deleted asap and will not get into\n the archive.\n 2. If wal_ttl_seconds is 0 and wal_size_limit_mb is not 0,\n WAL files will be checked every 10 min and if total size is greater\n then wal_size_limit_mb, they will be deleted starting with the\n earliest until size_limit is met. All empty files will be deleted.\n 3. If wal_ttl_seconds is not 0 and wal_size_limit_mb is 0, then\n WAL files will be checked every wal_ttl_secondsi / 2 and those that\n are older than wal_ttl_seconds will be deleted.\n 4. If both are not 0, WAL files will be checked every 10 min and both\n checks will be performed with ttl being first.\n\n | *Type:* ``int``\n | *Default:* ``0``\n\n .. py:attribute:: manifest_preallocation_size\n\n Number of bytes to preallocate (via fallocate) the manifest\n files. Default is 4mb, which is reasonable to reduce random IO\n as well as prevent overallocation for mounts that preallocate\n large amounts of data (such as xfs's allocsize option).\n\n | *Type:* ``int``\n | *Default:* ``4194304``\n\n .. py:attribute:: purge_redundant_kvs_while_flush\n\n Purge duplicate/deleted keys when a memtable is flushed to storage.\n\n | *Type:* ``bool``\n | *Default:* ``True``\n\n .. py:attribute:: allow_os_buffer\n\n Data being read from file storage may be buffered in the OS\n\n | *Type:* ``bool``\n | *Default:* ``True``\n\n .. py:attribute:: allow_mmap_reads\n\n Allow the OS to mmap file for reading sst tables\n\n | *Type:* ``bool``\n | *Default:* ``True``\n\n .. py:attribute:: allow_mmap_writes\n\n Allow the OS to mmap file for writing\n\n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: is_fd_close_on_exec\n\n Disable child process inherit open files\n\n | *Type:* ``bool``\n | *Default:* ``True``\n\n .. py:attribute:: skip_log_error_on_recovery\n\n Skip log corruption error on recovery\n (If client is ok with losing most recent changes)\n \n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: stats_dump_period_sec\n\n If not zero, dump rocksdb.stats to LOG every stats_dump_period_sec\n\n | *Type:* ``int``\n | *Default:* ``3600``\n\n .. py:attribute:: advise_random_on_open\n\n If set true, will hint the underlying file system that the file\n access pattern is random, when a sst file is opened.\n\n | *Type:* ``bool``\n | *Default:* ``True``\n\n .. py:attribute:: use_adaptive_mutex\n\n Use adaptive mutex, which spins in the user space before resorting\n to kernel. This could reduce context switch when the mutex is not\n heavily contended. However, if the mutex is hot, we could end up\n wasting spin time.\n \n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: bytes_per_sync\n\n Allows OS to incrementally sync files to disk while they are being\n written, asynchronously, in the background.\n Issue one request for every bytes_per_sync written. 0 turns it off.\n \n | *Type:* ``int``\n | *Default:* ``0``\n\n .. py:attribute:: verify_checksums_in_compaction\n\n If ``True``, compaction will verify checksum on every read that\n happens as part of compaction.\n\n | *Type:* ``bool``\n | *Default:* ``True``\n\n .. py:attribute:: compaction_style\n\n The compaction style. Could be set to ``\"level\"`` to use level-style\n compaction. For universal-style compaction use ``\"universal\"``.\n\n | *Type:* ``string``\n | *Default:* ``level``\n\n .. py:attribute:: compaction_options_universal\n\n Options to use for universal-style compaction. They make only sense if\n :py:attr:`rocksdb.Options.compaction_style` is set to ``\"universal\"``.\n\n It is a dict with the following keys.\n\n * ``size_ratio``:\n Percentage flexibilty while comparing file size.\n If the candidate file(s) size is 1% smaller than the next file's size,\n then include next file into this candidate set.\n Default: ``1``\n\n * ``min_merge_width``:\n The minimum number of files in a single compaction run.\n Default: ``2``\n\n * ``max_merge_width``:\n The maximum number of files in a single compaction run.\n Default: ``UINT_MAX``\n\n * ``max_size_amplification_percent``:\n The size amplification is defined as the amount (in percentage) of\n additional storage needed to store a single byte of data in the database.\n For example, a size amplification of 2% means that a database that\n contains 100 bytes of user-data may occupy upto 102 bytes of\n physical storage. By this definition, a fully compacted database has\n a size amplification of 0%. Rocksdb uses the following heuristic\n to calculate size amplification: it assumes that all files excluding\n the earliest file contribute to the size amplification.\n Default: ``200``, which means that a 100 byte database could require upto\n 300 bytes of storage.\n\n * ``compression_size_percent``:\n If this option is set to be -1 (the default value), all the output\n files will follow compression type specified.\n\n If this option is not negative, we will try to make sure compressed\n size is just above this value. In normal cases, at least this\n percentage of data will be compressed.\n\n When we are compacting to a new file, here is the criteria whether\n it needs to be compressed: assuming here are the list of files sorted\n by generation time: ``A1...An B1...Bm C1...Ct``\n where ``A1`` is the newest and ``Ct`` is the oldest, and we are going\n to compact ``B1...Bm``, we calculate the total size of all the files\n as total_size, as well as the total size of ``C1...Ct`` as\n ``total_C``, the compaction output file will be compressed if\n ``total_C / total_size < this percentage``.\n Default: -1\n\n * ``stop_style``:\n The algorithm used to stop picking files into a single compaction.\n Can be either ``\"similar_size\"`` or ``\"total_size\"``.\n\n * ``similar_size``: Pick files of similar size.\n * ``total_size``: Total size of picked files is greater than next file.\n\n Default: ``\"total_size\"``\n\n For setting options, just assign a dict with the fields to set.\n It is allowed to omit keys in this dict. Missing keys are just not set\n to the underlying options object.\n\n This example just changes the stop_style and leaves the other options\n untouched. ::\n\n opts = rocksdb.Options()\n opts.compaction_options_universal = {'stop_style': 'similar_size'}\n\n .. py:attribute:: filter_deletes\n\n Use KeyMayExist API to filter deletes when this is true.\n If KeyMayExist returns false, i.e. the key definitely does not exist, then\n the delete is a noop. KeyMayExist only incurs in-memory look up.\n This optimization avoids writing the delete to storage when appropriate.\n \n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: max_sequential_skip_in_iterations\n\n An iteration->Next() sequentially skips over keys with the same\n user-key unless this option is set. This number specifies the number\n of keys (with the same userkey) that will be sequentially\n skipped before a reseek is issued.\n \n | *Type:* ``int``\n | *Default:* ``8``\n\n .. py:attribute:: memtable_factory\n\n This is a factory that provides MemTableRep objects.\n Right now you can assing instances of the following classes.\n\n * :py:class:`rocksdb.VectorMemtableFactory`\n * :py:class:`rocksdb.SkipListMemtableFactory`\n * :py:class:`rocksdb.HashSkipListMemtableFactory`\n * :py:class:`rocksdb.HashLinkListMemtableFactory`\n\n *Default:* :py:class:`rocksdb.SkipListMemtableFactory`\n\n .. py:attribute:: table_factory\n\n Factory for the files forming the persisten data storage.\n Sometimes they are also named SST-Files. Right now you can assign\n instances of the following classes.\n\n * :py:class:`rocksdb.BlockBasedTableFactory`\n * :py:class:`rocksdb.PlainTableFactory`\n * :py:class:`rocksdb.TotalOrderPlainTableFactory`\n\n *Default:* :py:class:`rocksdb.BlockBasedTableFactory`\n\n .. py:attribute:: inplace_update_support\n\n Allows thread-safe inplace updates. Requires Updates if\n\n * key exists in current memtable\n * new sizeof(new_value) <= sizeof(old_value)\n * old_value for that key is a put i.e. kTypeValue\n \n | *Type:* ``bool``\n | *Default:* ``False``\n\n .. py:attribute:: inplace_update_num_locks\n\n | Number of locks used for inplace update.\n | Default: 10000, if inplace_update_support = true, else 0.\n\n | *Type:* ``int``\n | *Default:* ``10000``\n\n .. py:attribute:: comparator\n\n Comparator used to define the order of keys in the table.\n A python comparator must implement the :py:class:`rocksdb.interfaces.Comparator`\n interface.\n\n *Requires*: The client must ensure that the comparator supplied\n here has the same name and orders keys *exactly* the same as the\n comparator provided to previous open calls on the same DB.\n\n *Default:* :py:class:`rocksdb.BytewiseComparator`\n\n .. py:attribute:: merge_operator\n\n The client must provide a merge operator if Merge operation\n needs to be accessed. Calling Merge on a DB without a merge operator\n would result in :py:exc:`rocksdb.errors.NotSupported`. The client must\n ensure that the merge operator supplied here has the same name and\n *exactly* the same semantics as the merge operator provided to\n previous open calls on the same DB. The only exception is reserved\n for upgrade, where a DB previously without a merge operator is\n introduced to Merge operation for the first time. It's necessary to\n specify a merge operator when openning the DB in this case.\n\n A python merge operator must implement the\n :py:class:`rocksdb.interfaces.MergeOperator` or\n :py:class:`rocksdb.interfaces.AssociativeMergeOperator`\n interface.\n \n *Default:* ``None``\n\n .. py:attribute:: prefix_extractor\n\n If not ``None``, use the specified function to determine the\n prefixes for keys. These prefixes will be placed in the filter.\n Depending on the workload, this can reduce the number of read-IOP\n cost for scans when a prefix is passed to the calls generating an\n iterator (:py:meth:`rocksdb.DB.iterkeys` ...).\n\n A python prefix_extractor must implement the\n :py:class:`rocksdb.interfaces.SliceTransform` interface\n\n For prefix filtering to work properly, \"prefix_extractor\" and \"comparator\"\n must be such that the following properties hold:\n\n 1. ``key.starts_with(prefix(key))``\n 2. ``compare(prefix(key), key) <= 0``\n 3. ``If compare(k1, k2) <= 0, then compare(prefix(k1), prefix(k2)) <= 0``\n 4. ``prefix(prefix(key)) == prefix(key)``\n\n *Default:* ``None``\n\n .. py:attribute:: row_cache\n\n A global cache for table-level rows. If ``None`` this cache is not used.\n Otherwise it must be an instance of :py:class:`rocksdb.LRUCache`\n\n *Default:* ``None``\n\n\nCompressionTypes\n================\n\n.. py:class:: rocksdb.CompressionType\n\n Defines the support compression types\n\n .. py:attribute:: no_compression\n .. py:attribute:: snappy_compression\n .. py:attribute:: zlib_compression\n .. py:attribute:: bzip2_compression\n .. py:attribute:: lz4_compression\n .. py:attribute:: lz4hc_compression\n\nBytewiseComparator\n==================\n\n.. py:class:: rocksdb.BytewiseComparator\n\n Wraps the rocksdb Bytewise Comparator, it uses lexicographic byte-wise\n ordering\n\nBloomFilterPolicy\n=================\n\n.. py:class:: rocksdb.BloomFilterPolicy\n\n Wraps the rocksdb BloomFilter Policy\n\n .. py:method:: __init__(bits_per_key)\n\n :param int bits_per_key:\n Specifies the approximately number of bits per key.\n A good value for bits_per_key is 10, which yields a filter with\n ~ 1% false positive rate.\n\n\nLRUCache\n========\n\n.. py:class:: rocksdb.LRUCache\n\n Wraps the rocksdb LRUCache\n\n .. py:method:: __init__(capacity, shard_bits=None)\n\n Create a new cache with a fixed size capacity (in bytes).\n The cache is sharded to 2^numShardBits shards, by hash of the key.\n The total capacity is divided and evenly assigned to each shard.\n\n.. _table_factories_label:\n\nTableFactories\n==============\n\nCurrently RocksDB supports two types of tables: plain table and block-based table.\nInstances of this classes can assigned to :py:attr:`rocksdb.Options.table_factory`\n\n* *Block-based table:* This is the default table type that RocksDB inherited from\n LevelDB. It was designed for storing data in hard disk or flash device.\n\n* *Plain table:* It is one of RocksDB's SST file format optimized\n for low query latency on pure-memory or really low-latency media.\n\nTutorial of rocksdb table formats is available here:\nhttps://github.com/facebook/rocksdb/wiki/A-Tutorial-of-RocksDB-SST-formats\n\n.. py:class:: rocksdb.BlockBasedTableFactory\n\n Wraps BlockBasedTableFactory of RocksDB.\n\n .. py:method:: __init__(index_type='binary_search', hash_index_allow_collision=True, checksum='crc32', block_cache, block_cache_compressed, filter_policy=None, no_block_cache=False, block_size=None, block_size_deviation=None, block_restart_interval=None, whole_key_filtering=None):\n\n\n :param string index_type:\n * ``binary_search`` a space efficient index block that is optimized\n for binary-search-based index.\n * ``hash_search`` the hash index. If enabled, will do hash lookup\n when `Options.prefix_extractor` is provided.\n\n :param bool hash_index_allow_collision:\n Influence the behavior when ``hash_search`` is used.\n If ``False``, stores a precise prefix to block range mapping.\n If ``True``, does not store prefix and allows prefix hash collision\n (less memory consumption)\n\n :param string checksum:\n Use the specified checksum type. Newly created table files will be\n protected with this checksum type. Old table files will still be readable,\n even though they have different checksum type.\n Can be either ``crc32`` or ``xxhash``.\n\n :param block_cache:\n Control over blocks (user data is stored in a set of blocks, and\n a block is the unit of reading from disk).\n\n If ``None``, rocksdb will automatically create and use an 8MB internal cache.\n If not ``None`` use the specified cache for blocks. In that case it must\n be an instance of :py:class:`rocksdb.LRUCache`\n\n :param block_cache_compressed:\n If ``None``, rocksdb will not use a compressed block cache.\n If not ``None`` use the specified cache for compressed blocks. In that\n case it must be an instance of :py:class:`rocksdb.LRUCache`\n\n :param filter_policy:\n If not ``None`` use the specified filter policy to reduce disk reads.\n A python filter policy must implement the\n :py:class:`rocksdb.interfaces.FilterPolicy` interface.\n Recommended is a instance of :py:class:`rocksdb.BloomFilterPolicy`\n\n :param bool no_block_cache:\n Disable block cache. If this is set to true,\n then no block cache should be used, and the block_cache should\n point to ``None``\n\n :param int block_size:\n If set to ``None`` the rocksdb default of ``4096`` is used.\n Approximate size of user data packed per block. Note that the\n block size specified here corresponds to uncompressed data. The\n actual size of the unit read from disk may be smaller if\n compression is enabled. This parameter can be changed dynamically.\n\n :param int block_size_deviation:\n If set to ``None`` the rocksdb default of ``10`` is used.\n This is used to close a block before it reaches the configured\n 'block_size'. If the percentage of free space in the current block is less\n than this specified number and adding a new record to the block will\n exceed the configured block size, then this block will be closed and the\n new record will be written to the next block.\n\n :param int block_restart_interval:\n If set to ``None`` the rocksdb default of ``16`` is used.\n Number of keys between restart points for delta encoding of keys.\n This parameter can be changed dynamically. Most clients should\n leave this parameter alone.\n\n :param bool whole_key_filtering:\n If set to ``None`` the rocksdb default of ``True`` is used.\n If ``True``, place whole keys in the filter (not just prefixes).\n This must generally be true for gets to be efficient.\n\n.. py:class:: rocksdb.PlainTableFactory\n\n Plain Table with prefix-only seek. It wraps rocksdb PlainTableFactory.\n\n For this factory, you need to set :py:attr:`rocksdb.Options.prefix_extractor`\n properly to make it work. Look-up will start with prefix hash lookup for\n key prefix. Inside the hash bucket found, a binary search is executed for\n hash conflicts. Finally, a linear search is used.\n\n .. py:method:: __init__(user_key_len=0, bloom_bits_per_key=10, hash_table_ratio=0.75, index_sparseness=10, huge_page_tlb_size=0, encoding_type='plain', full_scan_mode=False, store_index_in_file=False)\n\n :param int user_key_len:\n Plain table has optimization for fix-sized keys, which can be\n specified via user_key_len.\n Alternatively, you can pass `0` if your keys have variable lengths.\n\n :param int bloom_bits_per_key:\n The number of bits used for bloom filer per prefix.\n You may disable it by passing `0`.\n\n :param float hash_table_ratio:\n The desired utilization of the hash table used for prefix hashing.\n hash_table_ratio = number of prefixes / #buckets in the hash table.\n\n :param int index_sparseness:\n Inside each prefix, need to build one index record for how\n many keys for binary search inside each hash bucket.\n For encoding type ``prefix``, the value will be used when\n writing to determine an interval to rewrite the full key.\n It will also be used as a suggestion and satisfied when possible.\n\n :param int huge_page_tlb_size:\n If <=0, allocate hash indexes and blooms from malloc.\n Otherwise from huge page TLB.\n The user needs to reserve huge pages for it to be allocated, like:\n ``sysctl -w vm.nr_hugepages=20``\n See linux doc Documentation/vm/hugetlbpage.txt\n\n :param string encoding_type:\n How to encode the keys. The value will determine how to encode keys\n when writing to a new SST file. This value will be stored\n inside the SST file which will be used when reading from the\n file, which makes it possible for users to choose different\n encoding type when reopening a DB. Files with different\n encoding types can co-exist in the same DB and can be read.\n\n * ``plain``: Always write full keys without any special encoding.\n * ``prefix``: Find opportunity to write the same prefix once for multiple rows.\n In some cases, when a key follows a previous key with the same prefix,\n instead of writing out the full key, it just writes out the size of the\n shared prefix, as well as other bytes, to save some bytes.\n\n When using this option, the user is required to use the same prefix\n extractor to make sure the same prefix will be extracted from the same key.\n The Name() value of the prefix extractor will be stored in the file.\n When reopening the file, the name of the options.prefix_extractor given\n will be bitwise compared to the prefix extractors stored in the file.\n An error will be returned if the two don't match.\n\n :param bool full_scan_mode:\n Mode for reading the whole file one record by one without using the index.\n\n :param bool store_index_in_file:\n Compute plain table index and bloom filter during file building\n and store it in file. When reading file, index will be mmaped\n instead of recomputation.\n\n.. _memtable_factories_label:\n\nMemtableFactories\n=================\n\nRocksDB has different classes to represent the in-memory buffer for the current\noperations. You have to assing instances of the following classes to\n:py:attr:`rocksdb.Options.memtable_factory`.\nThis page has a comparison the most popular ones.\nhttps://github.com/facebook/rocksdb/wiki/Hash-based-memtable-implementations\n\n.. py:class:: rocksdb.VectorMemtableFactory\n\n This creates MemTableReps that are backed by an std::vector.\n On iteration, the vector is sorted. This is useful for workloads where\n iteration is very rare and writes are generally not issued after reads begin.\n\n .. py:method:: __init__(count=0)\n\n :param int count:\n Passed to the constructor of the underlying std::vector of each\n VectorRep. On initialization, the underlying array will be at\n least count bytes reserved for usage.\n\n.. py:class:: rocksdb.SkipListMemtableFactory\n\n This uses a skip list to store keys.\n\n .. py:method:: __init__()\n\n.. py:class:: rocksdb.HashSkipListMemtableFactory\n\n This class contains a fixed array of buckets, each pointing\n to a skiplist (null if the bucket is empty).\n\n .. note::\n\n :py:attr:`rocksdb.Options.prefix_extractor` must be set, otherwise\n rocksdb fails back to skip-list.\n\n .. py:method:: __init__(bucket_count = 1000000, skiplist_height = 4, skiplist_branching_factor = 4)\n\n :param int bucket_count: number of fixed array buckets\n :param int skiplist_height: the max height of the skiplist\n :param int skiplist_branching_factor:\n probabilistic size ratio between adjacent link lists in the skiplist\n\n.. py:class:: rocksdb.HashLinkListMemtableFactory\n\n The factory is to create memtables with a hashed linked list.\n It contains a fixed array of buckets, each pointing to a sorted single\n linked list (null if the bucket is empty).\n\n .. note::\n\n :py:attr:`rocksdb.Options.prefix_extractor` must be set, otherwise\n rocksdb fails back to skip-list.\n\n\n .. py:method:: __init__(bucket_count=50000)\n\n :param int bucket: number of fixed array buckets\n" }, { "path": "docs/changelog.rst", "content": "Changelog\n*********\n\nVersion 0.5\n-----------\n\n\nVersion 0.4\n-----------\nThis version works with RocksDB v3.12.\n\n* Added :py:func:`repair_db`.\n* Added :py:meth:`rocksdb.Options.row_cache`\n* Publish to pypi.\n\nBackward Incompatible Changes:\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n* Changed API of :py:meth:`rocksdb.DB.compact_range`.\n\n * Only allow keyword arguments.\n * Changed ``reduce_level`` to ``change_level``.\n * Add new argument called ``bottommost_level_compaction``.\n\n\nVersion 0.3\n-----------\nThis version works with RocksDB version v3.11.\n\nBackward Incompatible Changes:\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n**Prefix Seeks:**\n\nAccording to this page https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes,\nall the prefix related parameters on ``ReadOptions`` are removed.\nRocksdb realizes now if ``Options.prefix_extractor`` is set and uses then\nprefix-seeks automatically. This means the following changes on pyrocksdb.\n\n* DB.iterkeys, DB.itervalues, DB.iteritems have *no* ``prefix`` parameter anymore.\n* DB.get, DB.multi_get, DB.key_may_exist, DB.iterkeys, DB.itervalues, DB.iteritems\n have *no* ``prefix_seek`` parameter anymore.\n\nWhich means all the iterators walk now always to the *end* of the database.\nSo if you need to stay within a prefix, write your own code to ensure that.\nFor DB.iterkeys and DB.iteritems ``itertools.takewhile`` is a possible solution. ::\n\n from itertools import takewhile\n\n it = self.db.iterkeys()\n it.seek(b'00002')\n print list(takewhile(lambda key: key.startswith(b'00002'), it))\n\n it = self.db.iteritems()\n it.seek(b'00002')\n print dict(takewhile(lambda item: item[0].startswith(b'00002'), it))\n\n**SST Table Builders:**\n\n* Removed ``NewTotalOrderPlainTableFactory``, because rocksdb drops it too.\n\n**Changed Options:**\n\nIn newer versions of rocksdb a bunch of options were moved or removed.\n\n* Rename ``bloom_bits_per_prefix`` of :py:class:`rocksdb.PlainTableFactory` to ``bloom_bits_per_key``\n* Removed ``Options.db_stats_log_interval``.\n* Removed ``Options.disable_seek_compaction``\n* Moved ``Options.no_block_cache`` to ``BlockBasedTableFactory``\n* Moved ``Options.block_size`` to ``BlockBasedTableFactory``\n* Moved ``Options.block_size_deviation`` to ``BlockBasedTableFactory``\n* Moved ``Options.block_restart_interval`` to ``BlockBasedTableFactory``\n* Moved ``Options.whole_key_filtering`` to ``BlockBasedTableFactory``\n* Removed ``Options.table_cache_remove_scan_count_limit``\n* Removed rm_scan_count_limit from ``LRUCache``\n\n\nNew:\n^^^^\n* Make CompactRange available: :py:meth:`rocksdb.DB.compact_range`\n* Add init options to :py:class:`rocksdb.BlockBasedTableFactory`\n* Add more option to :py:class:`rocksdb.PlainTableFactory`\n* Add :py:class:`rocksdb.WriteBatchIterator`\n* add :py:attr:`rocksdb.CompressionType.lz4_compression`\n* add :py:attr:`rocksdb.CompressionType.lz4hc_compression`\n\n\nVersion 0.2\n-----------\n\nThis version works with RocksDB version 2.8.fb. Now you have access to the more\nadvanced options of rocksdb. Like changing the memtable or SST representation.\nIt is also possible now to enable *Universal Style Compaction*.\n\n* Fixed `issue 3 `_.\n Which fixed the change of prefix_extractor from raw-pointer to smart-pointer.\n\n* Support the new :py:attr:`rocksdb.Options.verify_checksums_in_compaction` option.\n\n* Add :py:attr:`rocksdb.Options.table_factory` option. So you could use the new\n 'PlainTableFactories' which are optimized for in-memory-databases.\n\n * https://github.com/facebook/rocksdb/wiki/PlainTable-Format\n * https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database%3F\n\n* Add :py:attr:`rocksdb.Options.memtable_factory` option.\n\n* Add options :py:attr:`rocksdb.Options.compaction_style` and\n :py:attr:`rocksdb.Options.compaction_options_universal` to change the\n compaction style.\n\n* Update documentation to the new default values\n\n * allow_mmap_reads=true\n * allow_mmap_writes=false\n * max_background_flushes=1\n * max_open_files=5000\n * paranoid_checks=true\n * disable_seek_compaction=true\n * level0_stop_writes_trigger=24\n * level0_slowdown_writes_trigger=20\n\n* Document new property names for :py:meth:`rocksdb.DB.get_property`.\n\nVersion 0.1\n-----------\n\nInitial version. Works with rocksdb version 2.7.fb.\n" }, { "path": "docs/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# pyrocksdb documentation build configuration file, created by\n# sphinx-quickstart on Tue Dec 31 12:50:54 2013.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\nimport sys\nimport os\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#sys.path.insert(0, os.path.abspath('.'))\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'sphinx.ext.autodoc',\n 'sphinx.ext.todo',\n 'sphinx.ext.viewcode',\n]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix of source filenames.\nsource_suffix = '.rst'\n\n# The encoding of source files.\n#source_encoding = 'utf-8-sig'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = u'pyrocksdb'\ncopyright = u'2014, sh'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = '0.4'\n# The full version, including alpha/beta/rc tags.\nrelease = '0.4'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#language = None\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n#today = ''\n# Else, today_fmt is used as the format for a strftime call.\n#today_fmt = '%B %d, %Y'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\nexclude_patterns = ['_build']\n\n# The reST default role (used for this markup: `text`) to use for all\n# documents.\n#default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n#add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n#add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n#show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# A list of ignored prefixes for module index sorting.\n#modindex_common_prefix = []\n\n# If true, keep warnings as \"system message\" paragraphs in the built documents.\n#keep_warnings = False\n\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n# html_theme = 'default'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\n#html_theme_path = []\n\n# The name for this set of Sphinx documents. If None, it defaults to\n# \" v documentation\".\n#html_title = None\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n#html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\n#html_logo = None\n\n# The name of an image file (within the static path) to use as favicon of the\n# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\n#html_favicon = None\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n# Add any extra paths that contain custom files (such as robots.txt or\n# .htaccess) here, relative to this directory. These files are copied\n# directly to the root of the documentation.\n#html_extra_path = []\n\n# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,\n# using the given strftime format.\n#html_last_updated_fmt = '%b %d, %Y'\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n#html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\n#html_sidebars = {}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n#html_additional_pages = {}\n\n# If false, no module index is generated.\n#html_domain_indices = True\n\n# If false, no index is generated.\n#html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n#html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n#html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\n#html_show_sphinx = True\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\n#html_show_copyright = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n#html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n#html_file_suffix = None\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'pyrocksdbdoc'\n\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n# The paper size ('letterpaper' or 'a4paper').\n#'papersize': 'letterpaper',\n\n# The font size ('10pt', '11pt' or '12pt').\n#'pointsize': '10pt',\n\n# Additional stuff for the LaTeX preamble.\n#'preamble': '',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n ('index', 'pyrocksdb.tex', u'pyrocksdb Documentation',\n u'sh', 'manual'),\n]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n#latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n#latex_use_parts = False\n\n# If true, show page references after internal links.\n#latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\n#latex_show_urls = False\n\n# Documents to append as an appendix to all manuals.\n#latex_appendices = []\n\n# If false, no module index is generated.\n#latex_domain_indices = True\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n ('index', 'pyrocksdb', u'pyrocksdb Documentation',\n [u'sh'], 1)\n]\n\n# If true, show URL addresses after external links.\n#man_show_urls = False\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n ('index', 'pyrocksdb', u'pyrocksdb Documentation',\n u'sh', 'pyrocksdb', 'One line description of project.',\n 'Miscellaneous'),\n]\n\n# Documents to append as an appendix to all manuals.\n#texinfo_appendices = []\n\n# If false, no module index is generated.\n#texinfo_domain_indices = True\n\n# How to display URL addresses: 'footnote', 'no', or 'inline'.\n#texinfo_show_urls = 'footnote'\n\n# If true, do not generate a @detailmenu in the \"Top\" node's menu.\n#texinfo_no_detailmenu = False\n" }, { "path": "docs/index.rst", "content": "Welcome to pyrocksdb's documentation!\n=====================================\n\nOverview\n--------\nPython bindings to the C++ interface of http://rocksdb.org/ using cython::\n\n import rocksdb\n db = rocksdb.DB(\"test.db\", rocksdb.Options(create_if_missing=True))\n db.put(b\"a\", b\"b\")\n print db.get(b\"a\")\n\n\nTested with python2.7 and python3.4 and RocksDB version 3.12\n\n.. toctree::\n :maxdepth: 2\n\n Instructions how to install \n Tutorial \n API \n Changelog \n\n\nContributing\n------------\n\nSource can be found on `github `_.\nFeel free to fork and send pull-requests or create issues on the\n`github issue tracker `_\n\nRoadMap/TODO\n------------\n\nNo plans so far. Please submit wishes to the github issues.\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "docs/installation.rst", "content": "Installing\n**********\n.. highlight:: bash\n\n\nBuilding rocksdb\n----------------\n\nBriefly describes how to build rocksdb under an ordinary debian/ubuntu.\nFor more details consider https://github.com/facebook/rocksdb/blob/master/INSTALL.md\n\n.. code-block:: bash\n\n apt-get install build-essential\n apt-get install libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev\n git clone https://github.com/facebook/rocksdb.git\n cd rocksdb\n make shared_lib\n\n\nSystemwide rocksdb\n^^^^^^^^^^^^^^^^^^\nThe following command installs the shared library in ``/usr/lib/`` and the\nheader files in ``/usr/include/rocksdb/``::\n\n make install-shared INSTALL_PATH=/usr\n\nTo uninstall use::\n\n make uninstall INSTALL_PATH=/usr\n\nLocal rocksdb\n^^^^^^^^^^^^^\nIf you don't like the system wide installation, or you don't have the\npermissions, it is possible to set the following environment variables.\nThese varialbes are picked up by the compiler, linker and loader\n\n.. code-block:: bash\n\n export CPLUS_INCLUDE_PATH=${CPLUS_INCLUDE_PATH}:`pwd`/include\n export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:`pwd`\n export LIBRARY_PATH=${LIBRARY_PATH}:`pwd`\n\n\nBuilding pyrocksdb\n------------------\n\n.. code-block:: bash\n\n apt-get install python-virtualenv python-dev\n virtualenv pyrocks_test\n cd pyrocks_test\n . bin/active\n pip install \"Cython>=0.20\"\n pip install git+git://github.com/stephan-hof/pyrocksdb.git\n" }, { "path": "docs/tutorial/index.rst", "content": "Basic Usage of pyrocksdb\n************************\n\nOpen\n====\n\nThe most basic open call is ::\n\n import rocksdb\n\n db = rocksdb.DB(\"test.db\", rocksdb.Options(create_if_missing=True))\n\nA more production ready open can look like this ::\n\n import rocksdb\n\n opts = rocksdb.Options()\n opts.create_if_missing = True\n opts.max_open_files = 300000\n opts.write_buffer_size = 67108864\n opts.max_write_buffer_number = 3\n opts.target_file_size_base = 67108864\n\n opts.table_factory = rocksdb.BlockBasedTableFactory(\n filter_policy=rocksdb.BloomFilterPolicy(10),\n block_cache=rocksdb.LRUCache(2 * (1024 ** 3)),\n block_cache_compressed=rocksdb.LRUCache(500 * (1024 ** 2)))\n\n db = rocksdb.DB(\"test.db\", opts)\n\nIt assings a cache of 2.5G, uses a bloom filter for faster lookups and keeps\nmore data (64 MB) in memory before writting a .sst file.\n\nAbout Bytes And Unicode\n========================\n\nRocksDB stores all data as uninterpreted *byte strings*.\npyrocksdb behaves the same and uses nearly everywhere byte strings too.\nIn python2 this is the ``str`` type. In python3 the ``bytes`` type. \nSince the default string type for string literals differs between python 2 and 3,\nit is strongly recommended to use an explicit ``b`` prefix for all byte string\nliterals in both python2 and python3 code.\nFor example ``b'this is a byte string'``. This avoids ambiguity and ensures\nthat your code keeps working as intended if you switch between python2 and python3.\n\nThe only place where you can pass unicode objects are filesytem paths like\n\n* Directory name of the database itself :py:meth:`rocksdb.DB.__init__`\n\n* :py:attr:`rocksdb.Options.wal_dir`\n\n* :py:attr:`rocksdb.Options.db_log_dir`\n\nTo encode this path name, `sys.getfilesystemencoding()` encoding is used.\n\nAccess\n======\n\nStore, Get, Delete is straight forward ::\n\n # Store\n db.put(b\"key\", b\"value\")\n\n # Get\n db.get(b\"key\")\n\n # Delete\n db.delete(b\"key\")\n\nIt is also possible to gather modifications and\napply them in a single operation ::\n\n batch = rocksdb.WriteBatch()\n batch.put(b\"key\", b\"v1\")\n batch.delete(b\"key\")\n batch.put(b\"key\", b\"v2\")\n batch.put(b\"key\", b\"v3\")\n\n db.write(batch)\n\nFetch of multiple values at once ::\n\n db.put(b\"key1\", b\"v1\")\n db.put(b\"key2\", b\"v2\")\n\n ret = db.multi_get([b\"key1\", b\"key2\", b\"key3\"])\n\n # prints b\"v1\"\n print ret[b\"key1\"]\n\n # prints None\n print ret[b\"key3\"]\n\nIteration\n=========\n\nIterators behave slightly different than expected. Per default they are not\nvalid. So you have to call one of its seek methods first ::\n\n db.put(b\"key1\", b\"v1\")\n db.put(b\"key2\", b\"v2\")\n db.put(b\"key3\", b\"v3\")\n\n it = db.iterkeys()\n it.seek_to_first()\n\n # prints [b'key1', b'key2', b'key3']\n print list(it)\n\n it.seek_to_last()\n # prints [b'key3']\n print list(it)\n\n it.seek(b'key2')\n # prints [b'key2', b'key3']\n print list(it)\n\nThere are also methods to iterate over values/items ::\n\n it = db.itervalues()\n it.seek_to_first()\n\n # prints [b'v1', b'v2', b'v3']\n print list(it)\n\n it = db.iteritems()\n it.seek_to_first()\n\n # prints [(b'key1', b'v1'), (b'key2, b'v2'), (b'key3', b'v3')]\n print list(it)\n\nReversed iteration ::\n\n it = db.iteritems()\n it.seek_to_last()\n\n # prints [(b'key3', b'v3'), (b'key2', b'v2'), (b'key1', b'v1')]\n print list(reversed(it))\n\n\nSnapshots\n=========\n\nSnapshots are nice to get a consistent view on the database ::\n\n self.db.put(b\"a\", b\"1\")\n self.db.put(b\"b\", b\"2\")\n\n snapshot = self.db.snapshot()\n self.db.put(b\"a\", b\"2\")\n self.db.delete(b\"b\")\n\n it = self.db.iteritems()\n it.seek_to_first()\n\n # prints {b'a': b'2'}\n print dict(it)\n\n it = self.db.iteritems(snapshot=snapshot)\n it.seek_to_first()\n\n # prints {b'a': b'1', b'b': b'2'}\n print dict(it)\n\n\nMergeOperator\n=============\n\nMerge operators are useful for efficient read-modify-write operations.\nFor more details see `Merge Operator `_\n\nA python merge operator must either implement the\n:py:class:`rocksdb.interfaces.AssociativeMergeOperator` or\n:py:class:`rocksdb.interfaces.MergeOperator` interface.\n\nThe following example python merge operator implements a counter ::\n\n class AssocCounter(rocksdb.interfaces.AssociativeMergeOperator):\n def merge(self, key, existing_value, value):\n if existing_value:\n s = int(existing_value) + int(value)\n return (True, str(s).encode('ascii'))\n return (True, value)\n\n def name(self):\n return b'AssocCounter'\n\n\n opts = rocksdb.Options()\n opts.create_if_missing = True\n opts.merge_operator = AssocCounter()\n db = rocksdb.DB('test.db', opts)\n\n db.merge(b\"a\", b\"1\")\n db.merge(b\"a\", b\"1\")\n\n # prints b'2'\n print db.get(b\"a\")\n\nPrefixExtractor\n===============\n\nAccording to `Prefix API `_\na prefix_extractor can reduce IO for scans within a prefix range.\nA python prefix extractor must implement the :py:class:`rocksdb.interfaces.SliceTransform` interface.\n\nThe following example presents a prefix extractor of a static size.\nSo always the first 5 bytes are used as the prefix ::\n\n class StaticPrefix(rocksdb.interfaces.SliceTransform):\n def name(self):\n return b'static'\n\n def transform(self, src):\n return (0, 5)\n\n def in_domain(self, src):\n return len(src) >= 5\n\n def in_range(self, dst):\n return len(dst) == 5\n\n opts = rocksdb.Options()\n opts.create_if_missing=True\n opts.prefix_extractor = StaticPrefix()\n\n db = rocksdb.DB('test.db', opts)\n\n db.put(b'00001.x', b'x')\n db.put(b'00001.y', b'y')\n db.put(b'00001.z', b'z')\n\n db.put(b'00002.x', b'x')\n db.put(b'00002.y', b'y')\n db.put(b'00002.z', b'z')\n\n db.put(b'00003.x', b'x')\n db.put(b'00003.y', b'y')\n db.put(b'00003.z', b'z')\n\n prefix = b'00002'\n\n it = db.iteritems()\n it.seek(prefix)\n\n # prints {b'00002.z': b'z', b'00002.y': b'y', b'00002.x': b'x'}\n print dict(itertools.takewhile(lambda item: item[0].startswith(prefix), it))\n\n\nBackup And Restore\n==================\n\nBackup and Restore is done with a separate :py:class:`rocksdb.BackupEngine` object.\n\nA backup can only be created on a living database object. ::\n\n import rocksdb\n\n db = rocksdb.DB(\"test.db\", rocksdb.Options(create_if_missing=True))\n db.put(b'a', b'v1')\n db.put(b'b', b'v2')\n db.put(b'c', b'v3')\n\nBackup is created like this.\nYou can choose any path for the backup destination except the db path itself.\nIf ``flush_before_backup`` is ``True`` the current memtable is flushed to disk\nbefore backup. ::\n\n backup = rocksdb.BackupEngine(\"test.db/backups\")\n backup.create_backup(db, flush_before_backup=True)\n\nRestore is done like this.\nThe two arguments are the db_dir and wal_dir, which are mostly the same. ::\n\n backup = rocksdb.BackupEngine(\"test.db/backups\")\n backup.restore_latest_backup(\"test.db\", \"test.db\")\n\n\nChange Memtable Or SST Implementations\n======================================\n\nAs noted here :ref:`memtable_factories_label`, RocksDB offers different implementations for the memtable\nrepresentation. Per default :py:class:`rocksdb.SkipListMemtableFactory` is used,\nbut changing it to a different one is veary easy.\n\nHere is an example for HashSkipList-MemtableFactory.\nKeep in mind: To use the hashed based MemtableFactories you must set\n:py:attr:`rocksdb.Options.prefix_extractor`.\nIn this example all keys have a static prefix of len 5. ::\n\n class StaticPrefix(rocksdb.interfaces.SliceTransform):\n def name(self):\n return b'static'\n\n def transform(self, src):\n return (0, 5)\n\n def in_domain(self, src):\n return len(src) >= 5\n\n def in_range(self, dst):\n return len(dst) == 5\n\n\n opts = rocksdb.Options()\n opts.prefix_extractor = StaticPrefix()\n opts.memtable_factory = rocksdb.HashSkipListMemtableFactory()\n opts.create_if_missing = True\n\n db = rocksdb.DB(\"test.db\", opts)\n db.put(b'00001.x', b'x')\n db.put(b'00001.y', b'y')\n db.put(b'00002.x', b'x')\n\nFor initial bulk loads the Vector-MemtableFactory makes sense. ::\n\n opts = rocksdb.Options()\n opts.memtable_factory = rocksdb.VectorMemtableFactory()\n opts.create_if_missing = True\n\n db = rocksdb.DB(\"test.db\", opts)\n\nAs noted here :ref:`table_factories_label`, it is also possible to change the\nrepresentation of the final data files.\nHere is an example how to use a 'PlainTable'. ::\n\n opts = rocksdb.Options()\n opts.table_factory = rocksdb.PlainTableFactory()\n opts.create_if_missing = True\n\n db = rocksdb.DB(\"test.db\", opts)\n\nChange Compaction Style\n=======================\n\nRocksDB has a compaction algorithm called *universal*. This style typically\nresults in lower write amplification but higher space amplification than\nLevel Style Compaction. See here for more details,\nhttps://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide#multi-threaded-compactions\n\nHere is an example to switch to *universal style compaction*. ::\n\n opts = rocksdb.Options()\n opts.compaction_style = \"universal\"\n opts.compaction_options_universal = {\"min_merge_width\": 3}\n\nSee here for more options on *universal style compaction*,\n:py:attr:`rocksdb.Options.compaction_options_universal`\n\nIterate Over WriteBatch\n=======================\n\nIn same cases you need to know, what operations happened on a WriteBatch.\nThe pyrocksdb WriteBatch supports the iterator protocol, see this example. ::\n\n batch = rocksdb.WriteBatch()\n batch.put(b\"key1\", b\"v1\")\n batch.delete(b'a')\n batch.merge(b'xxx', b'value')\n\n for op, key, value in batch:\n print op, key, value\n\n # prints the following three lines\n # Put key1 v1\n # Delete a\n # Merge xxx value\n" }, { "path": "rocksdb/__init__.py", "content": "from ._rocksdb import *\n" }, { "path": "rocksdb/_rocksdb.pyx", "content": "import cython\nfrom libcpp.string cimport string\nfrom libcpp.deque cimport deque\nfrom libcpp.vector cimport vector\nfrom cpython cimport bool as py_bool\nfrom libcpp cimport bool as cpp_bool\nfrom libc.stdint cimport uint32_t\nfrom cython.operator cimport dereference as deref\nfrom cpython.bytes cimport PyBytes_AsString\nfrom cpython.bytes cimport PyBytes_Size\nfrom cpython.bytes cimport PyBytes_FromString\nfrom cpython.bytes cimport PyBytes_FromStringAndSize\nfrom cpython.unicode cimport PyUnicode_Decode\n\nfrom std_memory cimport shared_ptr\ncimport options\ncimport merge_operator\ncimport filter_policy\ncimport comparator\ncimport slice_transform\ncimport cache\ncimport logger\ncimport snapshot\ncimport db\ncimport iterator\ncimport backup\ncimport env\ncimport table_factory\ncimport memtablerep\ncimport universal_compaction\n\n# Enums are the only exception for direct imports\n# Their name als already unique enough\nfrom universal_compaction cimport kCompactionStopStyleSimilarSize\nfrom universal_compaction cimport kCompactionStopStyleTotalSize\n\nfrom options cimport kCompactionStyleLevel\nfrom options cimport kCompactionStyleUniversal\n\nfrom slice_ cimport Slice\nfrom status cimport Status\n\nimport sys\nfrom interfaces import MergeOperator as IMergeOperator\nfrom interfaces import AssociativeMergeOperator as IAssociativeMergeOperator\nfrom interfaces import FilterPolicy as IFilterPolicy\nfrom interfaces import Comparator as IComparator\nfrom interfaces import SliceTransform as ISliceTransform\nimport traceback\nimport errors\n\nctypedef const filter_policy.FilterPolicy ConstFilterPolicy\n\ncdef extern from \"cpp/utils.hpp\" namespace \"py_rocks\":\n cdef const Slice* vector_data(vector[Slice]&)\n\n# Prepare python for threaded usage.\n# Python callbacks (merge, comparator)\n# could be executed in a rocksdb background thread (eg. compaction).\ncdef extern from \"Python.h\":\n void PyEval_InitThreads()\nPyEval_InitThreads()\n\n## Here comes the stuff to wrap the status to exception\ncdef check_status(const Status& st):\n if st.ok():\n return\n\n if st.IsNotFound():\n raise errors.NotFound(st.ToString())\n\n if st.IsCorruption():\n raise errors.Corruption(st.ToString())\n\n if st.IsNotSupported():\n raise errors.NotSupported(st.ToString())\n\n if st.IsInvalidArgument():\n raise errors.InvalidArgument(st.ToString())\n\n if st.IsIOError():\n raise errors.RocksIOError(st.ToString())\n\n if st.IsMergeInProgress():\n raise errors.MergeInProgress(st.ToString())\n\n if st.IsIncomplete():\n raise errors.Incomplete(st.ToString())\n\n raise Exception(\"Unknown error: %s\" % st.ToString())\n######################################################\n\n\ncdef string bytes_to_string(path) except *:\n return string(PyBytes_AsString(path), PyBytes_Size(path))\n\ncdef string_to_bytes(string ob):\n return PyBytes_FromStringAndSize(ob.c_str(), ob.size())\n\ncdef Slice bytes_to_slice(ob) except *:\n return Slice(PyBytes_AsString(ob), PyBytes_Size(ob))\n\ncdef slice_to_bytes(Slice sl):\n return PyBytes_FromStringAndSize(sl.data(), sl.size())\n\n## only for filsystem paths\ncdef string path_to_string(object path) except *:\n if isinstance(path, bytes):\n return bytes_to_string(path)\n if isinstance(path, unicode):\n path = path.encode(sys.getfilesystemencoding())\n return bytes_to_string(path)\n else:\n raise TypeError(\"Wrong type for path: %s\" % path)\n\ncdef object string_to_path(string path):\n fs_encoding = sys.getfilesystemencoding().encode('ascii')\n return PyUnicode_Decode(path.c_str(), path.size(), fs_encoding, \"replace\")\n\n## Here comes the stuff for the comparator\n@cython.internal\ncdef class PyComparator(object):\n cdef object get_ob(self):\n return None\n\n cdef const comparator.Comparator* get_comparator(self):\n return NULL\n\n cdef set_info_log(self, shared_ptr[logger.Logger] info_log):\n pass\n\n@cython.internal\ncdef class PyGenericComparator(PyComparator):\n cdef comparator.ComparatorWrapper* comparator_ptr\n cdef object ob\n\n def __cinit__(self, object ob):\n self.comparator_ptr = NULL\n if not isinstance(ob, IComparator):\n raise TypeError(\"%s is not of type %s\" % (ob, IComparator))\n\n self.ob = ob\n self.comparator_ptr = new comparator.ComparatorWrapper(\n bytes_to_string(ob.name()),\n ob,\n compare_callback)\n\n def __dealloc__(self):\n if not self.comparator_ptr == NULL:\n del self.comparator_ptr\n\n cdef object get_ob(self):\n return self.ob\n\n cdef const comparator.Comparator* get_comparator(self):\n return self.comparator_ptr\n\n cdef set_info_log(self, shared_ptr[logger.Logger] info_log):\n self.comparator_ptr.set_info_log(info_log)\n\n@cython.internal\ncdef class PyBytewiseComparator(PyComparator):\n cdef const comparator.Comparator* comparator_ptr\n\n def __cinit__(self):\n self.comparator_ptr = comparator.BytewiseComparator()\n\n def name(self):\n return PyBytes_FromString(self.comparator_ptr.Name())\n\n def compare(self, a, b):\n return self.comparator_ptr.Compare(\n bytes_to_slice(a),\n bytes_to_slice(b))\n\n cdef object get_ob(self):\n return self\n\n cdef const comparator.Comparator* get_comparator(self):\n return self.comparator_ptr\n\ncdef int compare_callback(\n void* ctx,\n logger.Logger* log,\n string& error_msg,\n const Slice& a,\n const Slice& b) with gil:\n\n try:\n return (ctx).compare(slice_to_bytes(a), slice_to_bytes(b))\n except BaseException as error:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in compare callback: %s\", tb)\n error_msg.assign(str(error))\n\nBytewiseComparator = PyBytewiseComparator\n#########################################\n\n\n\n## Here comes the stuff for the filter policy\n@cython.internal\ncdef class PyFilterPolicy(object):\n cdef object get_ob(self):\n return None\n\n cdef shared_ptr[ConstFilterPolicy] get_policy(self):\n return shared_ptr[ConstFilterPolicy]()\n\n cdef set_info_log(self, shared_ptr[logger.Logger] info_log):\n pass\n\n@cython.internal\ncdef class PyGenericFilterPolicy(PyFilterPolicy):\n cdef shared_ptr[filter_policy.FilterPolicyWrapper] policy\n cdef object ob\n\n def __cinit__(self, object ob):\n if not isinstance(ob, IFilterPolicy):\n raise TypeError(\"%s is not of type %s\" % (ob, IFilterPolicy))\n\n self.ob = ob\n self.policy.reset(new filter_policy.FilterPolicyWrapper(\n bytes_to_string(ob.name()),\n ob,\n create_filter_callback,\n key_may_match_callback))\n\n cdef object get_ob(self):\n return self.ob\n\n cdef shared_ptr[ConstFilterPolicy] get_policy(self):\n return (self.policy)\n\n cdef set_info_log(self, shared_ptr[logger.Logger] info_log):\n self.policy.get().set_info_log(info_log)\n\n\ncdef void create_filter_callback(\n void* ctx,\n logger.Logger* log,\n string& error_msg,\n const Slice* keys,\n int n,\n string* dst) with gil:\n\n try:\n ret = (ctx).create_filter(\n [slice_to_bytes(keys[i]) for i in range(n)])\n dst.append(bytes_to_string(ret))\n except BaseException as error:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in create filter callback: %s\", tb)\n error_msg.assign(str(error))\n\ncdef cpp_bool key_may_match_callback(\n void* ctx,\n logger.Logger* log,\n string& error_msg,\n const Slice& key,\n const Slice& filt) with gil:\n\n try:\n return (ctx).key_may_match(\n slice_to_bytes(key),\n slice_to_bytes(filt))\n except BaseException as error:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in key_mach_match callback: %s\", tb)\n error_msg.assign(str(error))\n\n@cython.internal\ncdef class PyBloomFilterPolicy(PyFilterPolicy):\n cdef shared_ptr[ConstFilterPolicy] policy\n\n def __cinit__(self, int bits_per_key):\n self.policy.reset(filter_policy.NewBloomFilterPolicy(bits_per_key))\n\n def name(self):\n return PyBytes_FromString(self.policy.get().Name())\n\n def create_filter(self, keys):\n cdef string dst\n cdef vector[Slice] c_keys\n\n for key in keys:\n c_keys.push_back(bytes_to_slice(key))\n\n self.policy.get().CreateFilter(\n vector_data(c_keys),\n c_keys.size(),\n cython.address(dst))\n\n return string_to_bytes(dst)\n\n def key_may_match(self, key, filter_):\n return self.policy.get().KeyMayMatch(\n bytes_to_slice(key),\n bytes_to_slice(filter_))\n\n cdef object get_ob(self):\n return self\n\n cdef shared_ptr[ConstFilterPolicy] get_policy(self):\n return self.policy\n\nBloomFilterPolicy = PyBloomFilterPolicy\n#############################################\n\n\n\n## Here comes the stuff for the merge operator\n@cython.internal\ncdef class PyMergeOperator(object):\n cdef shared_ptr[merge_operator.MergeOperator] merge_op\n cdef object ob\n\n def __cinit__(self, object ob):\n if isinstance(ob, IAssociativeMergeOperator):\n self.ob = ob\n self.merge_op.reset(\n \n new merge_operator.AssociativeMergeOperatorWrapper(\n bytes_to_string(ob.name()),\n (ob),\n merge_callback))\n\n elif isinstance(ob, IMergeOperator):\n self.ob = ob\n self.merge_op.reset(\n \n new merge_operator.MergeOperatorWrapper(\n bytes_to_string(ob.name()),\n ob,\n ob,\n full_merge_callback,\n partial_merge_callback))\n else:\n msg = \"%s is not of this types %s\"\n msg %= (ob, (IAssociativeMergeOperator, IMergeOperator))\n raise TypeError(msg)\n\n cdef object get_ob(self):\n return self.ob\n\n cdef shared_ptr[merge_operator.MergeOperator] get_operator(self):\n return self.merge_op\n\ncdef cpp_bool merge_callback(\n void* ctx,\n const Slice& key,\n const Slice* existing_value,\n const Slice& value,\n string* new_value,\n logger.Logger* log) with gil:\n\n if existing_value == NULL:\n py_existing_value = None\n else:\n py_existing_value = slice_to_bytes(deref(existing_value))\n\n try:\n ret = (ctx).merge(\n slice_to_bytes(key),\n py_existing_value,\n slice_to_bytes(value))\n\n if ret[0]:\n new_value.assign(bytes_to_string(ret[1]))\n return True\n return False\n\n except:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in merge_callback: %s\", tb)\n return False\n\ncdef cpp_bool full_merge_callback(\n void* ctx,\n const Slice& key,\n const Slice* existing_value,\n const deque[string]& op_list,\n string* new_value,\n logger.Logger* log) with gil:\n\n if existing_value == NULL:\n py_existing_value = None\n else:\n py_existing_value = slice_to_bytes(deref(existing_value))\n\n try:\n ret = (ctx).full_merge(\n slice_to_bytes(key),\n py_existing_value,\n [string_to_bytes(op_list[i]) for i in range(op_list.size())])\n\n if ret[0]:\n new_value.assign(bytes_to_string(ret[1]))\n return True\n return False\n\n except:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in full_merge_callback: %s\", tb)\n return False\n\ncdef cpp_bool partial_merge_callback(\n void* ctx,\n const Slice& key,\n const Slice& left_op,\n const Slice& right_op,\n string* new_value,\n logger.Logger* log) with gil:\n\n try:\n ret = (ctx).partial_merge(\n slice_to_bytes(key),\n slice_to_bytes(left_op),\n slice_to_bytes(right_op))\n\n if ret[0]:\n new_value.assign(bytes_to_string(ret[1]))\n return True\n return False\n\n except:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in partial_merge_callback: %s\", tb)\n return False\n##############################################\n\n#### Here comes the Cache stuff\n@cython.internal\ncdef class PyCache(object):\n cdef shared_ptr[cache.Cache] get_cache(self):\n return shared_ptr[cache.Cache]()\n\n@cython.internal\ncdef class PyLRUCache(PyCache):\n cdef shared_ptr[cache.Cache] cache_ob\n\n def __cinit__(self, capacity, shard_bits=None):\n if shard_bits is not None:\n self.cache_ob = cache.NewLRUCache(capacity, shard_bits)\n else:\n self.cache_ob = cache.NewLRUCache(capacity)\n\n cdef shared_ptr[cache.Cache] get_cache(self):\n return self.cache_ob\n\nLRUCache = PyLRUCache\n###############################\n\n### Here comes the stuff for SliceTransform\n@cython.internal\ncdef class PySliceTransform(object):\n cdef shared_ptr[slice_transform.SliceTransform] transfomer\n cdef object ob\n\n def __cinit__(self, object ob):\n if not isinstance(ob, ISliceTransform):\n raise TypeError(\"%s is not of type %s\" % (ob, ISliceTransform))\n\n self.ob = ob\n self.transfomer.reset(\n \n new slice_transform.SliceTransformWrapper(\n bytes_to_string(ob.name()),\n ob,\n slice_transform_callback,\n slice_in_domain_callback,\n slice_in_range_callback))\n\n cdef object get_ob(self):\n return self.ob\n\n cdef shared_ptr[slice_transform.SliceTransform] get_transformer(self):\n return self.transfomer\n\n cdef set_info_log(self, shared_ptr[logger.Logger] info_log):\n cdef slice_transform.SliceTransformWrapper* ptr\n ptr = self.transfomer.get()\n ptr.set_info_log(info_log)\n\n\ncdef Slice slice_transform_callback(\n void* ctx,\n logger.Logger* log,\n string& error_msg,\n const Slice& src) with gil:\n\n cdef size_t offset\n cdef size_t size\n\n try:\n ret = (ctx).transform(slice_to_bytes(src))\n offset = ret[0]\n size = ret[1]\n if (offset + size) > src.size():\n msg = \"offset(%i) + size(%i) is bigger than slice(%i)\"\n raise Exception(msg % (offset, size, src.size()))\n\n return Slice(src.data() + offset, size)\n except BaseException as error:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in slice transfrom callback: %s\", tb)\n error_msg.assign(str(error))\n\ncdef cpp_bool slice_in_domain_callback(\n void* ctx,\n logger.Logger* log,\n string& error_msg,\n const Slice& src) with gil:\n\n try:\n return (ctx).in_domain(slice_to_bytes(src))\n except BaseException as error:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in slice transfrom callback: %s\", tb)\n error_msg.assign(str(error))\n\ncdef cpp_bool slice_in_range_callback(\n void* ctx,\n logger.Logger* log,\n string& error_msg,\n const Slice& src) with gil:\n\n try:\n return (ctx).in_range(slice_to_bytes(src))\n except BaseException as error:\n tb = traceback.format_exc()\n logger.Log(log, \"Error in slice transfrom callback: %s\", tb)\n error_msg.assign(str(error))\n###########################################\n\n## Here are the TableFactories\n@cython.internal\ncdef class PyTableFactory(object):\n cdef shared_ptr[table_factory.TableFactory] factory\n\n cdef shared_ptr[table_factory.TableFactory] get_table_factory(self):\n return self.factory\n\n cdef set_info_log(self, shared_ptr[logger.Logger] info_log):\n pass\n\ncdef class BlockBasedTableFactory(PyTableFactory):\n cdef PyFilterPolicy py_filter_policy\n\n def __init__(self,\n index_type='binary_search',\n py_bool hash_index_allow_collision=True,\n checksum='crc32',\n PyCache block_cache=None,\n PyCache block_cache_compressed=None,\n filter_policy=None,\n no_block_cache=False,\n block_size=None,\n block_size_deviation=None,\n block_restart_interval=None,\n whole_key_filtering=None):\n\n cdef table_factory.BlockBasedTableOptions table_options\n\n if index_type == 'binary_search':\n table_options.index_type = table_factory.kBinarySearch\n elif index_type == 'hash_search':\n table_options.index_type = table_factory.kHashSearch\n else:\n raise ValueError(\"Unknown index_type: %s\" % index_type)\n\n if hash_index_allow_collision:\n table_options.hash_index_allow_collision = True\n else:\n table_options.hash_index_allow_collision = False\n\n if checksum == 'crc32':\n table_options.checksum = table_factory.kCRC32c\n elif checksum == 'xxhash':\n table_options.checksum = table_factory.kxxHash\n else:\n raise ValueError(\"Unknown checksum: %s\" % checksum)\n\n if no_block_cache:\n table_options.no_block_cache = True\n else:\n table_options.no_block_cache = False\n\n # If the following options are None use the rocksdb default.\n if block_size is not None:\n table_options.block_size = block_size\n\n if block_size_deviation is not None:\n table_options.block_size_deviation = block_size_deviation\n\n if block_restart_interval is not None:\n table_options.block_restart_interval = block_restart_interval\n\n if whole_key_filtering is not None:\n if whole_key_filtering:\n table_options.whole_key_filtering = True\n else:\n table_options.whole_key_filtering = False\n\n if block_cache is not None:\n table_options.block_cache = block_cache.get_cache()\n\n if block_cache_compressed is not None:\n table_options.block_cache_compressed = block_cache_compressed.get_cache()\n\n # Set the filter_policy\n self.py_filter_policy = None\n if filter_policy is not None:\n if isinstance(filter_policy, PyFilterPolicy):\n if (filter_policy).get_policy().get() == NULL:\n raise Exception(\"Cannot set filter policy: %s\" % filter_policy)\n self.py_filter_policy = filter_policy\n else:\n self.py_filter_policy = PyGenericFilterPolicy(filter_policy)\n\n table_options.filter_policy = self.py_filter_policy.get_policy()\n\n self.factory.reset(table_factory.NewBlockBasedTableFactory(table_options))\n\n cdef set_info_log(self, shared_ptr[logger.Logger] info_log):\n if self.py_filter_policy is not None:\n self.py_filter_policy.set_info_log(info_log)\n\ncdef class PlainTableFactory(PyTableFactory):\n def __init__(\n self,\n user_key_len=0,\n bloom_bits_per_key=10,\n hash_table_ratio=0.75,\n index_sparseness=10,\n huge_page_tlb_size=0,\n encoding_type='plain',\n py_bool full_scan_mode=False,\n py_bool store_index_in_file=False):\n\n cdef table_factory.PlainTableOptions table_options\n\n table_options.user_key_len = user_key_len\n table_options.bloom_bits_per_key = bloom_bits_per_key\n table_options.hash_table_ratio = hash_table_ratio\n table_options.index_sparseness = index_sparseness\n table_options.huge_page_tlb_size = huge_page_tlb_size\n\n if encoding_type == 'plain':\n table_options.encoding_type = table_factory.kPlain\n elif encoding_type == 'prefix':\n table_options.encoding_type = table_factory.kPrefix\n else:\n raise ValueError(\"Unknown encoding_type: %s\" % encoding_type)\n\n table_options.full_scan_mode = full_scan_mode\n table_options.store_index_in_file = store_index_in_file\n\n self.factory.reset( table_factory.NewPlainTableFactory(table_options))\n#############################################\n\n### Here are the MemtableFactories\n@cython.internal\ncdef class PyMemtableFactory(object):\n cdef shared_ptr[memtablerep.MemTableRepFactory] factory\n\n cdef shared_ptr[memtablerep.MemTableRepFactory] get_memtable_factory(self):\n return self.factory\n\ncdef class SkipListMemtableFactory(PyMemtableFactory):\n def __init__(self):\n self.factory.reset(memtablerep.NewSkipListFactory())\n\ncdef class VectorMemtableFactory(PyMemtableFactory):\n def __init__(self, count=0):\n self.factory.reset(memtablerep.NewVectorRepFactory(count))\n\ncdef class HashSkipListMemtableFactory(PyMemtableFactory):\n def __init__(\n self,\n bucket_count=1000000,\n skiplist_height=4,\n skiplist_branching_factor=4):\n\n self.factory.reset(\n memtablerep.NewHashSkipListRepFactory(\n bucket_count,\n skiplist_height,\n skiplist_branching_factor))\n\ncdef class HashLinkListMemtableFactory(PyMemtableFactory):\n def __init__(self, bucket_count=50000):\n self.factory.reset(memtablerep.NewHashLinkListRepFactory(bucket_count))\n##################################\n\ncdef class CompressionType(object):\n no_compression = u'no_compression'\n snappy_compression = u'snappy_compression'\n zlib_compression = u'zlib_compression'\n bzip2_compression = u'bzip2_compression'\n lz4_compression = u'lz4_compression'\n lz4hc_compression = u'lz4hc_compression'\n\ncdef class Options(object):\n cdef options.Options* opts\n cdef PyComparator py_comparator\n cdef PyMergeOperator py_merge_operator\n cdef PySliceTransform py_prefix_extractor\n cdef PyTableFactory py_table_factory\n cdef PyMemtableFactory py_memtable_factory\n cdef PyCache py_row_cache\n\n # Used to protect sharing of Options with many DB-objects\n cdef cpp_bool in_use\n\n def __cinit__(self):\n self.opts = NULL\n self.opts = new options.Options()\n self.in_use = False\n\n def __dealloc__(self):\n if not self.opts == NULL:\n del self.opts\n\n def __init__(self, **kwargs):\n self.py_comparator = BytewiseComparator()\n self.py_merge_operator = None\n self.py_prefix_extractor = None\n self.py_table_factory = None\n self.py_memtable_factory = None\n self.py_row_cache = None\n\n for key, value in kwargs.items():\n setattr(self, key, value)\n\n property create_if_missing:\n def __get__(self):\n return self.opts.create_if_missing\n def __set__(self, value):\n self.opts.create_if_missing = value\n\n property error_if_exists:\n def __get__(self):\n return self.opts.error_if_exists\n def __set__(self, value):\n self.opts.error_if_exists = value\n\n property paranoid_checks:\n def __get__(self):\n return self.opts.paranoid_checks\n def __set__(self, value):\n self.opts.paranoid_checks = value\n\n property write_buffer_size:\n def __get__(self):\n return self.opts.write_buffer_size\n def __set__(self, value):\n self.opts.write_buffer_size = value\n\n property max_write_buffer_number:\n def __get__(self):\n return self.opts.max_write_buffer_number\n def __set__(self, value):\n self.opts.max_write_buffer_number = value\n\n property min_write_buffer_number_to_merge:\n def __get__(self):\n return self.opts.min_write_buffer_number_to_merge\n def __set__(self, value):\n self.opts.min_write_buffer_number_to_merge = value\n\n property max_open_files:\n def __get__(self):\n return self.opts.max_open_files\n def __set__(self, value):\n self.opts.max_open_files = value\n\n property compression:\n def __get__(self):\n if self.opts.compression == options.kNoCompression:\n return CompressionType.no_compression\n elif self.opts.compression == options.kSnappyCompression:\n return CompressionType.snappy_compression\n elif self.opts.compression == options.kZlibCompression:\n return CompressionType.zlib_compression\n elif self.opts.compression == options.kBZip2Compression:\n return CompressionType.bzip2_compression\n elif self.opts.compression == options.kLZ4Compression:\n return CompressionType.lz4_compression\n elif self.opts.compression == options.kLZ4HCCompression:\n return CompressionType.lz4hc_compression\n else:\n raise Exception(\"Unknonw type: %s\" % self.opts.compression)\n\n def __set__(self, value):\n if value == CompressionType.no_compression:\n self.opts.compression = options.kNoCompression\n elif value == CompressionType.snappy_compression:\n self.opts.compression = options.kSnappyCompression\n elif value == CompressionType.zlib_compression:\n self.opts.compression = options.kZlibCompression\n elif value == CompressionType.bzip2_compression:\n self.opts.compression = options.kBZip2Compression\n elif value == CompressionType.lz4_compression:\n self.opts.compression = options.kLZ4Compression\n elif value == CompressionType.lz4hc_compression:\n self.opts.compression = options.kLZ4HCCompression\n else:\n raise TypeError(\"Unknown compression: %s\" % value)\n\n property num_levels:\n def __get__(self):\n return self.opts.num_levels\n def __set__(self, value):\n self.opts.num_levels = value\n\n property level0_file_num_compaction_trigger:\n def __get__(self):\n return self.opts.level0_file_num_compaction_trigger\n def __set__(self, value):\n self.opts.level0_file_num_compaction_trigger = value\n\n property level0_slowdown_writes_trigger:\n def __get__(self):\n return self.opts.level0_slowdown_writes_trigger\n def __set__(self, value):\n self.opts.level0_slowdown_writes_trigger = value\n\n property level0_stop_writes_trigger:\n def __get__(self):\n return self.opts.level0_stop_writes_trigger\n def __set__(self, value):\n self.opts.level0_stop_writes_trigger = value\n\n property max_mem_compaction_level:\n def __get__(self):\n return self.opts.max_mem_compaction_level\n def __set__(self, value):\n self.opts.max_mem_compaction_level = value\n\n property target_file_size_base:\n def __get__(self):\n return self.opts.target_file_size_base\n def __set__(self, value):\n self.opts.target_file_size_base = value\n\n property target_file_size_multiplier:\n def __get__(self):\n return self.opts.target_file_size_multiplier\n def __set__(self, value):\n self.opts.target_file_size_multiplier = value\n\n property max_bytes_for_level_base:\n def __get__(self):\n return self.opts.max_bytes_for_level_base\n def __set__(self, value):\n self.opts.max_bytes_for_level_base = value\n\n property max_bytes_for_level_multiplier:\n def __get__(self):\n return self.opts.max_bytes_for_level_multiplier\n def __set__(self, value):\n self.opts.max_bytes_for_level_multiplier = value\n\n property max_bytes_for_level_multiplier_additional:\n def __get__(self):\n return self.opts.max_bytes_for_level_multiplier_additional\n def __set__(self, value):\n self.opts.max_bytes_for_level_multiplier_additional = value\n\n property expanded_compaction_factor:\n def __get__(self):\n return self.opts.expanded_compaction_factor\n def __set__(self, value):\n self.opts.expanded_compaction_factor = value\n\n property source_compaction_factor:\n def __get__(self):\n return self.opts.source_compaction_factor\n def __set__(self, value):\n self.opts.source_compaction_factor = value\n\n property max_grandparent_overlap_factor:\n def __get__(self):\n return self.opts.max_grandparent_overlap_factor\n def __set__(self, value):\n self.opts.max_grandparent_overlap_factor = value\n\n property disable_data_sync:\n def __get__(self):\n return self.opts.disableDataSync\n def __set__(self, value):\n self.opts.disableDataSync = value\n\n property use_fsync:\n def __get__(self):\n return self.opts.use_fsync\n def __set__(self, value):\n self.opts.use_fsync = value\n\n property db_log_dir:\n def __get__(self):\n return string_to_path(self.opts.db_log_dir)\n def __set__(self, value):\n self.opts.db_log_dir = path_to_string(value)\n\n property wal_dir:\n def __get__(self):\n return string_to_path(self.opts.wal_dir)\n def __set__(self, value):\n self.opts.wal_dir = path_to_string(value)\n\n property delete_obsolete_files_period_micros:\n def __get__(self):\n return self.opts.delete_obsolete_files_period_micros\n def __set__(self, value):\n self.opts.delete_obsolete_files_period_micros = value\n\n property max_background_compactions:\n def __get__(self):\n return self.opts.max_background_compactions\n def __set__(self, value):\n self.opts.max_background_compactions = value\n\n property max_background_flushes:\n def __get__(self):\n return self.opts.max_background_flushes\n def __set__(self, value):\n self.opts.max_background_flushes = value\n\n property max_log_file_size:\n def __get__(self):\n return self.opts.max_log_file_size\n def __set__(self, value):\n self.opts.max_log_file_size = value\n\n property log_file_time_to_roll:\n def __get__(self):\n return self.opts.log_file_time_to_roll\n def __set__(self, value):\n self.opts.log_file_time_to_roll = value\n\n property keep_log_file_num:\n def __get__(self):\n return self.opts.keep_log_file_num\n def __set__(self, value):\n self.opts.keep_log_file_num = value\n\n property soft_rate_limit:\n def __get__(self):\n return self.opts.soft_rate_limit\n def __set__(self, value):\n self.opts.soft_rate_limit = value\n\n property hard_rate_limit:\n def __get__(self):\n return self.opts.hard_rate_limit\n def __set__(self, value):\n self.opts.hard_rate_limit = value\n\n property rate_limit_delay_max_milliseconds:\n def __get__(self):\n return self.opts.rate_limit_delay_max_milliseconds\n def __set__(self, value):\n self.opts.rate_limit_delay_max_milliseconds = value\n\n property max_manifest_file_size:\n def __get__(self):\n return self.opts.max_manifest_file_size\n def __set__(self, value):\n self.opts.max_manifest_file_size = value\n\n property table_cache_numshardbits:\n def __get__(self):\n return self.opts.table_cache_numshardbits\n def __set__(self, value):\n self.opts.table_cache_numshardbits = value\n\n property arena_block_size:\n def __get__(self):\n return self.opts.arena_block_size\n def __set__(self, value):\n self.opts.arena_block_size = value\n\n property disable_auto_compactions:\n def __get__(self):\n return self.opts.disable_auto_compactions\n def __set__(self, value):\n self.opts.disable_auto_compactions = value\n\n property wal_ttl_seconds:\n def __get__(self):\n return self.opts.WAL_ttl_seconds\n def __set__(self, value):\n self.opts.WAL_ttl_seconds = value\n\n property wal_size_limit_mb:\n def __get__(self):\n return self.opts.WAL_size_limit_MB\n def __set__(self, value):\n self.opts.WAL_size_limit_MB = value\n\n property manifest_preallocation_size:\n def __get__(self):\n return self.opts.manifest_preallocation_size\n def __set__(self, value):\n self.opts.manifest_preallocation_size = value\n\n property purge_redundant_kvs_while_flush:\n def __get__(self):\n return self.opts.purge_redundant_kvs_while_flush\n def __set__(self, value):\n self.opts.purge_redundant_kvs_while_flush = value\n\n property allow_os_buffer:\n def __get__(self):\n return self.opts.allow_os_buffer\n def __set__(self, value):\n self.opts.allow_os_buffer = value\n\n property allow_mmap_reads:\n def __get__(self):\n return self.opts.allow_mmap_reads\n def __set__(self, value):\n self.opts.allow_mmap_reads = value\n\n property allow_mmap_writes:\n def __get__(self):\n return self.opts.allow_mmap_writes\n def __set__(self, value):\n self.opts.allow_mmap_writes = value\n\n property is_fd_close_on_exec:\n def __get__(self):\n return self.opts.is_fd_close_on_exec\n def __set__(self, value):\n self.opts.is_fd_close_on_exec = value\n\n property skip_log_error_on_recovery:\n def __get__(self):\n return self.opts.skip_log_error_on_recovery\n def __set__(self, value):\n self.opts.skip_log_error_on_recovery = value\n\n property stats_dump_period_sec:\n def __get__(self):\n return self.opts.stats_dump_period_sec\n def __set__(self, value):\n self.opts.stats_dump_period_sec = value\n\n property advise_random_on_open:\n def __get__(self):\n return self.opts.advise_random_on_open\n def __set__(self, value):\n self.opts.advise_random_on_open = value\n\n property use_adaptive_mutex:\n def __get__(self):\n return self.opts.use_adaptive_mutex\n def __set__(self, value):\n self.opts.use_adaptive_mutex = value\n\n property bytes_per_sync:\n def __get__(self):\n return self.opts.bytes_per_sync\n def __set__(self, value):\n self.opts.bytes_per_sync = value\n\n property verify_checksums_in_compaction:\n def __get__(self):\n return self.opts.verify_checksums_in_compaction\n def __set__(self, value):\n self.opts.verify_checksums_in_compaction = value\n\n property compaction_style:\n def __get__(self):\n if self.opts.compaction_style == kCompactionStyleLevel:\n return 'level'\n if self.opts.compaction_style == kCompactionStyleUniversal:\n return 'universal'\n raise Exception(\"Unknown compaction_style\")\n\n def __set__(self, str value):\n if value == 'level':\n self.opts.compaction_style = kCompactionStyleLevel\n elif value == 'universal':\n self.opts.compaction_style = kCompactionStyleUniversal\n else:\n raise Exception(\"Unknown compaction style\")\n\n property compaction_options_universal:\n def __get__(self):\n cdef universal_compaction.CompactionOptionsUniversal uopts\n cdef dict ret_ob = {}\n\n uopts = self.opts.compaction_options_universal\n\n ret_ob['size_ratio'] = uopts.size_ratio\n ret_ob['min_merge_width'] = uopts.min_merge_width\n ret_ob['max_merge_width'] = uopts.max_merge_width\n ret_ob['max_size_amplification_percent'] = uopts.max_size_amplification_percent\n ret_ob['compression_size_percent'] = uopts.compression_size_percent\n\n if uopts.stop_style == kCompactionStopStyleSimilarSize:\n ret_ob['stop_style'] = 'similar_size'\n elif uopts.stop_style == kCompactionStopStyleTotalSize:\n ret_ob['stop_style'] = 'total_size'\n else:\n raise Exception(\"Unknown compaction style\")\n\n return ret_ob\n\n def __set__(self, dict value):\n cdef universal_compaction.CompactionOptionsUniversal* uopts\n uopts = cython.address(self.opts.compaction_options_universal)\n\n if 'size_ratio' in value:\n uopts.size_ratio = value['size_ratio']\n\n if 'min_merge_width' in value:\n uopts.min_merge_width = value['min_merge_width']\n\n if 'max_merge_width' in value:\n uopts.max_merge_width = value['max_merge_width']\n\n if 'max_size_amplification_percent' in value:\n uopts.max_size_amplification_percent = value['max_size_amplification_percent']\n\n if 'compression_size_percent' in value:\n uopts.compression_size_percent = value['compression_size_percent']\n\n if 'stop_style' in value:\n if value['stop_style'] == 'similar_size':\n uopts.stop_style = kCompactionStopStyleSimilarSize\n elif value['stop_style'] == 'total_size':\n uopts.stop_style = kCompactionStopStyleTotalSize\n else:\n raise Exception(\"Unknown compaction style\")\n\n property filter_deletes:\n def __get__(self):\n return self.opts.filter_deletes\n def __set__(self, value):\n self.opts.filter_deletes = value\n\n property max_sequential_skip_in_iterations:\n def __get__(self):\n return self.opts.max_sequential_skip_in_iterations\n def __set__(self, value):\n self.opts.max_sequential_skip_in_iterations = value\n\n property inplace_update_support:\n def __get__(self):\n return self.opts.inplace_update_support\n def __set__(self, value):\n self.opts.inplace_update_support = value\n\n property table_factory:\n def __get__(self):\n return self.py_table_factory\n\n def __set__(self, PyTableFactory value):\n self.py_table_factory = value\n self.opts.table_factory = value.get_table_factory()\n\n property memtable_factory:\n def __get__(self):\n return self.py_memtable_factory\n\n def __set__(self, PyMemtableFactory value):\n self.py_memtable_factory = value\n self.opts.memtable_factory = value.get_memtable_factory()\n\n property inplace_update_num_locks:\n def __get__(self):\n return self.opts.inplace_update_num_locks\n def __set__(self, value):\n self.opts.inplace_update_num_locks = value\n\n property comparator:\n def __get__(self):\n return self.py_comparator.get_ob()\n\n def __set__(self, value):\n if isinstance(value, PyComparator):\n if (value).get_comparator() == NULL:\n raise Exception(\"Cannot set %s as comparator\" % value)\n else:\n self.py_comparator = value\n else:\n self.py_comparator = PyGenericComparator(value)\n\n self.opts.comparator = self.py_comparator.get_comparator()\n\n property merge_operator:\n def __get__(self):\n if self.py_merge_operator is None:\n return None\n return self.py_merge_operator.get_ob()\n\n def __set__(self, value):\n self.py_merge_operator = PyMergeOperator(value)\n self.opts.merge_operator = self.py_merge_operator.get_operator()\n\n property prefix_extractor:\n def __get__(self):\n if self.py_prefix_extractor is None:\n return None\n return self.py_prefix_extractor.get_ob()\n\n def __set__(self, value):\n self.py_prefix_extractor = PySliceTransform(value)\n self.opts.prefix_extractor = self.py_prefix_extractor.get_transformer()\n\n property row_cache:\n def __get__(self):\n return self.py_row_cache\n\n def __set__(self, value):\n if value is None:\n self.py_row_cache = None\n self.opts.row_cache.reset()\n elif not isinstance(value, PyCache):\n raise Exception(\"row_cache must be a Cache object\")\n else:\n self.py_row_cache = value\n self.opts.row_cache = self.py_row_cache.get_cache()\n\n\n# Forward declaration\ncdef class Snapshot\n\ncdef class KeysIterator\ncdef class ValuesIterator\ncdef class ItemsIterator\ncdef class ReversedIterator\n\n# Forward declaration\ncdef class WriteBatchIterator\n\ncdef class WriteBatch(object):\n cdef db.WriteBatch* batch\n\n def __cinit__(self, data=None):\n self.batch = NULL\n if data is not None:\n self.batch = new db.WriteBatch(bytes_to_string(data))\n else:\n self.batch = new db.WriteBatch()\n\n def __dealloc__(self):\n if not self.batch == NULL:\n del self.batch\n\n def put(self, key, value):\n self.batch.Put(bytes_to_slice(key), bytes_to_slice(value))\n\n def merge(self, key, value):\n self.batch.Merge(bytes_to_slice(key), bytes_to_slice(value))\n\n def delete(self, key):\n self.batch.Delete(bytes_to_slice(key))\n\n def clear(self):\n self.batch.Clear()\n\n def data(self):\n return string_to_bytes(self.batch.Data())\n\n def count(self):\n return self.batch.Count()\n\n def __iter__(self):\n return WriteBatchIterator(self)\n\n\n@cython.internal\ncdef class WriteBatchIterator(object):\n # Need a reference to the WriteBatch.\n # The BatchItems are only pointers to the memory in WriteBatch.\n cdef WriteBatch batch\n cdef vector[db.BatchItem] items\n cdef size_t pos\n\n def __init__(self, WriteBatch batch):\n cdef Status st\n\n self.batch = batch\n self.pos = 0\n\n st = db.get_batch_items(batch.batch, cython.address(self.items))\n check_status(st)\n\n def __iter__(self):\n return self\n\n def __next__(self):\n if self.pos == self.items.size():\n raise StopIteration()\n\n cdef str op\n\n if self.items[self.pos].op == db.BatchItemOpPut:\n op = \"Put\"\n elif self.items[self.pos].op == db.BatchItemOpMerge:\n op = \"Merge\"\n elif self.items[self.pos].op == db.BatchItemOpDelte:\n op = \"Delete\"\n\n ret = (\n op,\n slice_to_bytes(self.items[self.pos].key),\n slice_to_bytes(self.items[self.pos].value))\n\n self.pos += 1\n return ret\n\n@cython.no_gc_clear\ncdef class DB(object):\n cdef Options opts\n cdef db.DB* db\n\n def __cinit__(self, db_name, Options opts, read_only=False):\n cdef Status st\n cdef string db_path\n self.db = NULL\n self.opts = None\n\n if opts.in_use:\n raise Exception(\"Options object is already used by another DB\")\n\n db_path = path_to_string(db_name)\n if read_only:\n with nogil:\n st = db.DB_OpenForReadOnly(\n deref(opts.opts),\n db_path,\n cython.address(self.db),\n False)\n else:\n with nogil:\n st = db.DB_Open(\n deref(opts.opts),\n db_path,\n cython.address(self.db))\n check_status(st)\n\n # Inject the loggers into the python callbacks\n cdef shared_ptr[logger.Logger] info_log = self.db.GetOptions().info_log\n if opts.py_comparator is not None:\n opts.py_comparator.set_info_log(info_log)\n\n if opts.py_table_factory is not None:\n opts.py_table_factory.set_info_log(info_log)\n\n if opts.prefix_extractor is not None:\n opts.py_prefix_extractor.set_info_log(info_log)\n\n self.opts = opts\n self.opts.in_use = True\n\n def __dealloc__(self):\n if not self.db == NULL:\n with nogil:\n del self.db\n\n if self.opts is not None:\n self.opts.in_use = False\n\n def put(self, key, value, sync=False, disable_wal=False):\n cdef Status st\n cdef options.WriteOptions opts\n opts.sync = sync\n opts.disableWAL = disable_wal\n\n cdef Slice c_key = bytes_to_slice(key)\n cdef Slice c_value = bytes_to_slice(value)\n\n with nogil:\n st = self.db.Put(opts, c_key, c_value)\n check_status(st)\n\n def delete(self, key, sync=False, disable_wal=False):\n cdef Status st\n cdef options.WriteOptions opts\n opts.sync = sync\n opts.disableWAL = disable_wal\n\n cdef Slice c_key = bytes_to_slice(key)\n with nogil:\n st = self.db.Delete(opts, c_key)\n check_status(st)\n\n def merge(self, key, value, sync=False, disable_wal=False):\n cdef Status st\n cdef options.WriteOptions opts\n opts.sync = sync\n opts.disableWAL = disable_wal\n\n cdef Slice c_key = bytes_to_slice(key)\n cdef Slice c_value = bytes_to_slice(value)\n with nogil:\n st = self.db.Merge(opts, c_key, c_value)\n check_status(st)\n\n def write(self, WriteBatch batch, sync=False, disable_wal=False):\n cdef Status st\n cdef options.WriteOptions opts\n opts.sync = sync\n opts.disableWAL = disable_wal\n\n with nogil:\n st = self.db.Write(opts, batch.batch)\n check_status(st)\n\n def get(self, key, *args, **kwargs):\n cdef string res\n cdef Status st\n cdef options.ReadOptions opts\n\n opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))\n cdef Slice c_key = bytes_to_slice(key)\n\n with nogil:\n st = self.db.Get(opts, c_key, cython.address(res))\n\n if st.ok():\n return string_to_bytes(res)\n elif st.IsNotFound():\n return None\n else:\n check_status(st)\n\n def multi_get(self, keys, *args, **kwargs):\n cdef vector[string] values\n values.resize(len(keys))\n\n cdef vector[Slice] c_keys\n for key in keys:\n c_keys.push_back(bytes_to_slice(key))\n\n cdef options.ReadOptions opts\n opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))\n\n cdef vector[Status] res\n with nogil:\n res = self.db.MultiGet(\n opts,\n c_keys,\n cython.address(values))\n\n cdef dict ret_dict = {}\n for index in range(len(keys)):\n if res[index].ok():\n ret_dict[keys[index]] = string_to_bytes(values[index])\n elif res[index].IsNotFound():\n ret_dict[keys[index]] = None\n else:\n check_status(res[index])\n\n return ret_dict\n\n def key_may_exist(self, key, fetch=False, *args, **kwargs):\n cdef string value\n cdef cpp_bool value_found\n cdef cpp_bool exists\n cdef options.ReadOptions opts\n cdef Slice c_key\n opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))\n\n c_key = bytes_to_slice(key)\n exists = False\n\n if fetch:\n value_found = False\n with nogil:\n exists = self.db.KeyMayExist(\n opts,\n c_key,\n cython.address(value),\n cython.address(value_found))\n\n if exists:\n if value_found:\n return (True, string_to_bytes(value))\n else:\n return (True, None)\n else:\n return (False, None)\n else:\n with nogil:\n exists = self.db.KeyMayExist(\n opts,\n c_key,\n cython.address(value))\n\n return (exists, None)\n\n def iterkeys(self, *args, **kwargs):\n cdef options.ReadOptions opts\n cdef KeysIterator it\n\n opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))\n it = KeysIterator(self)\n\n with nogil:\n it.ptr = self.db.NewIterator(opts)\n return it\n\n def itervalues(self, *args, **kwargs):\n cdef options.ReadOptions opts\n cdef ValuesIterator it\n\n opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))\n\n it = ValuesIterator(self)\n\n with nogil:\n it.ptr = self.db.NewIterator(opts)\n return it\n\n def iteritems(self, *args, **kwargs):\n cdef options.ReadOptions opts\n cdef ItemsIterator it\n\n opts = self.build_read_opts(self.__parse_read_opts(*args, **kwargs))\n\n it = ItemsIterator(self)\n\n with nogil:\n it.ptr = self.db.NewIterator(opts)\n return it\n\n def snapshot(self):\n return Snapshot(self)\n\n def get_property(self, prop):\n cdef string value\n cdef Slice c_prop = bytes_to_slice(prop)\n cdef cpp_bool ret = False\n\n with nogil:\n ret = self.db.GetProperty(c_prop, cython.address(value))\n\n if ret:\n return string_to_bytes(value)\n else:\n return None\n\n def get_live_files_metadata(self):\n cdef vector[db.LiveFileMetaData] metadata\n\n with nogil:\n self.db.GetLiveFilesMetaData(cython.address(metadata))\n\n ret = []\n for ob in metadata:\n t = {}\n t['name'] = string_to_path(ob.name)\n t['level'] = ob.level\n t['size'] = ob.size\n t['smallestkey'] = string_to_bytes(ob.smallestkey)\n t['largestkey'] = string_to_bytes(ob.largestkey)\n t['smallest_seqno'] = ob.smallest_seqno\n t['largest_seqno'] = ob.largest_seqno\n\n ret.append(t)\n\n return ret\n\n def compact_range(self, begin=None, end=None, **py_options):\n cdef options.CompactRangeOptions c_options\n\n c_options.change_level = py_options.get('change_level', False)\n c_options.target_level = py_options.get('target_level', -1)\n\n blc = py_options.get('bottommost_level_compaction', 'if_compaction_filter')\n if blc == 'skip':\n c_options.bottommost_level_compaction = options.blc_skip\n elif blc == 'if_compaction_filter':\n c_options.bottommost_level_compaction = options.blc_is_filter\n elif blc == 'force':\n c_options.bottommost_level_compaction = options.blc_force\n else:\n raise ValueError(\"bottommost_level_compaction is not valid\")\n\n cdef Status st\n cdef Slice begin_val\n cdef Slice end_val\n\n cdef Slice* begin_ptr\n cdef Slice* end_ptr\n\n begin_ptr = NULL\n end_ptr = NULL\n\n if begin is not None:\n begin_val = bytes_to_slice(begin)\n begin_ptr = cython.address(begin_val)\n\n if end is not None:\n end_val = bytes_to_slice(end)\n end_ptr = cython.address(end_val)\n\n st = self.db.CompactRange(c_options, begin_ptr, end_ptr)\n check_status(st)\n\n @staticmethod\n def __parse_read_opts(\n verify_checksums=False,\n fill_cache=True,\n snapshot=None,\n read_tier=\"all\"):\n\n # TODO: Is this really effiencet ?\n return locals()\n\n cdef options.ReadOptions build_read_opts(self, dict py_opts):\n cdef options.ReadOptions opts\n opts.verify_checksums = py_opts['verify_checksums']\n opts.fill_cache = py_opts['fill_cache']\n if py_opts['snapshot'] is not None:\n opts.snapshot = ((py_opts['snapshot'])).ptr\n\n if py_opts['read_tier'] == \"all\":\n opts.read_tier = options.kReadAllTier\n elif py_opts['read_tier'] == 'cache':\n opts.read_tier = options.kBlockCacheTier\n else:\n raise ValueError(\"Invalid read_tier\")\n\n return opts\n\n property options:\n def __get__(self):\n return self.opts\n\n\ndef repair_db(db_name, Options opts):\n cdef Status st\n cdef string db_path\n\n db_path = path_to_string(db_name)\n st = db.RepairDB(db_path, deref(opts.opts))\n check_status(st)\n\n\n@cython.no_gc_clear\n@cython.internal\ncdef class Snapshot(object):\n cdef const snapshot.Snapshot* ptr\n cdef DB db\n\n def __cinit__(self, DB db):\n self.db = db\n self.ptr = NULL\n with nogil:\n self.ptr = db.db.GetSnapshot()\n\n def __dealloc__(self):\n if not self.ptr == NULL:\n with nogil:\n self.db.db.ReleaseSnapshot(self.ptr)\n\n\n@cython.internal\ncdef class BaseIterator(object):\n cdef iterator.Iterator* ptr\n cdef DB db\n\n def __cinit__(self, DB db):\n self.db = db\n self.ptr = NULL\n\n def __dealloc__(self):\n if not self.ptr == NULL:\n del self.ptr\n\n def __iter__(self):\n return self\n\n def __next__(self):\n if not self.ptr.Valid():\n raise StopIteration()\n\n cdef object ret = self.get_ob()\n with nogil:\n self.ptr.Next()\n check_status(self.ptr.status())\n return ret\n\n def __reversed__(self):\n return ReversedIterator(self)\n\n cpdef seek_to_first(self):\n with nogil:\n self.ptr.SeekToFirst()\n check_status(self.ptr.status())\n\n cpdef seek_to_last(self):\n with nogil:\n self.ptr.SeekToLast()\n check_status(self.ptr.status())\n\n cpdef seek(self, key):\n cdef Slice c_key = bytes_to_slice(key)\n with nogil:\n self.ptr.Seek(c_key)\n check_status(self.ptr.status())\n\n cdef object get_ob(self):\n return None\n\n@cython.internal\ncdef class KeysIterator(BaseIterator):\n cdef object get_ob(self):\n cdef Slice c_key\n with nogil:\n c_key = self.ptr.key()\n check_status(self.ptr.status())\n return slice_to_bytes(c_key)\n\n@cython.internal\ncdef class ValuesIterator(BaseIterator):\n cdef object get_ob(self):\n cdef Slice c_value\n with nogil:\n c_value = self.ptr.value()\n check_status(self.ptr.status())\n return slice_to_bytes(c_value)\n\n@cython.internal\ncdef class ItemsIterator(BaseIterator):\n cdef object get_ob(self):\n cdef Slice c_key\n cdef Slice c_value\n with nogil:\n c_key = self.ptr.key()\n c_value = self.ptr.value()\n check_status(self.ptr.status())\n return (slice_to_bytes(c_key), slice_to_bytes(c_value))\n\n@cython.internal\ncdef class ReversedIterator(object):\n cdef BaseIterator it\n\n def __cinit__(self, BaseIterator it):\n self.it = it\n\n def seek_to_first(self):\n self.it.seek_to_first()\n\n def seek_to_last(self):\n self.it.seek_to_last()\n\n def seek(self, key):\n self.it.seek(key)\n\n def __iter__(self):\n return self\n\n def __reversed__(self):\n return self.it\n\n def __next__(self):\n if not self.it.ptr.Valid():\n raise StopIteration()\n\n cdef object ret = self.it.get_ob()\n with nogil:\n self.it.ptr.Prev()\n check_status(self.it.ptr.status())\n return ret\n\ncdef class BackupEngine(object):\n cdef backup.BackupEngine* engine\n\n def __cinit__(self, backup_dir):\n cdef Status st\n cdef string c_backup_dir\n self.engine = NULL\n\n c_backup_dir = path_to_string(backup_dir)\n st = backup.BackupEngine_Open(\n env.Env_Default(),\n backup.BackupableDBOptions(c_backup_dir),\n cython.address(self.engine))\n\n check_status(st)\n\n def __dealloc__(self):\n if not self.engine == NULL:\n with nogil:\n del self.engine\n\n def create_backup(self, DB db, flush_before_backup=False):\n cdef Status st\n cdef cpp_bool c_flush_before_backup\n\n c_flush_before_backup = flush_before_backup\n\n with nogil:\n st = self.engine.CreateNewBackup(db.db, c_flush_before_backup)\n check_status(st)\n\n def restore_backup(self, backup_id, db_dir, wal_dir):\n cdef Status st\n cdef backup.BackupID c_backup_id\n cdef string c_db_dir\n cdef string c_wal_dir\n\n c_backup_id = backup_id\n c_db_dir = path_to_string(db_dir)\n c_wal_dir = path_to_string(wal_dir)\n\n with nogil:\n st = self.engine.RestoreDBFromBackup(\n c_backup_id,\n c_db_dir,\n c_wal_dir)\n\n check_status(st)\n\n def restore_latest_backup(self, db_dir, wal_dir):\n cdef Status st\n cdef string c_db_dir\n cdef string c_wal_dir\n\n c_db_dir = path_to_string(db_dir)\n c_wal_dir = path_to_string(wal_dir)\n\n with nogil:\n st = self.engine.RestoreDBFromLatestBackup(c_db_dir, c_wal_dir)\n\n check_status(st)\n\n def stop_backup(self):\n with nogil:\n self.engine.StopBackup()\n\n def purge_old_backups(self, num_backups_to_keep):\n cdef Status st\n cdef uint32_t c_num_backups_to_keep\n\n c_num_backups_to_keep = num_backups_to_keep\n\n with nogil:\n st = self.engine.PurgeOldBackups(c_num_backups_to_keep)\n check_status(st)\n\n def delete_backup(self, backup_id):\n cdef Status st\n cdef backup.BackupID c_backup_id\n\n c_backup_id = backup_id\n\n with nogil:\n st = self.engine.DeleteBackup(c_backup_id)\n\n check_status(st)\n\n def get_backup_info(self):\n cdef vector[backup.BackupInfo] backup_info\n\n with nogil:\n self.engine.GetBackupInfo(cython.address(backup_info))\n\n ret = []\n for ob in backup_info:\n t = {}\n t['backup_id'] = ob.backup_id\n t['timestamp'] = ob.timestamp\n t['size'] = ob.size\n ret.append(t)\n\n return ret\n" }, { "path": "rocksdb/backup.pxd", "content": "from libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\nfrom libcpp.vector cimport vector\nfrom libc.stdint cimport uint32_t\nfrom libc.stdint cimport int64_t\nfrom libc.stdint cimport uint64_t\n\nfrom status cimport Status\nfrom db cimport DB\nfrom env cimport Env\n\ncdef extern from \"rocksdb/utilities/backupable_db.h\" namespace \"rocksdb\":\n ctypedef uint32_t BackupID\n\n cdef cppclass BackupableDBOptions:\n BackupableDBOptions(const string& backup_dir)\n\n cdef struct BackupInfo:\n BackupID backup_id\n int64_t timestamp\n uint64_t size\n\n cdef cppclass BackupEngine:\n Status CreateNewBackup(DB*, cpp_bool) nogil except+\n Status PurgeOldBackups(uint32_t) nogil except+\n Status DeleteBackup(BackupID) nogil except+\n void StopBackup() nogil except+\n void GetBackupInfo(vector[BackupInfo]*) nogil except+\n Status RestoreDBFromBackup(BackupID, string&, string&) nogil except+\n Status RestoreDBFromLatestBackup(string&, string&) nogil except+\n\n cdef Status BackupEngine_Open \"rocksdb::BackupEngine::Open\"(\n Env*,\n BackupableDBOptions&,\n BackupEngine**)\n" }, { "path": "rocksdb/cache.pxd", "content": "from std_memory cimport shared_ptr\n\ncdef extern from \"rocksdb/cache.h\" namespace \"rocksdb\":\n cdef cppclass Cache:\n pass\n\n cdef extern shared_ptr[Cache] NewLRUCache(size_t)\n cdef extern shared_ptr[Cache] NewLRUCache(size_t, int)\n" }, { "path": "rocksdb/comparator.pxd", "content": "from libcpp.string cimport string\nfrom slice_ cimport Slice\nfrom logger cimport Logger\nfrom std_memory cimport shared_ptr\n\ncdef extern from \"rocksdb/comparator.h\" namespace \"rocksdb\":\n cdef cppclass Comparator:\n const char* Name()\n int Compare(const Slice&, const Slice&) const\n\n cdef extern const Comparator* BytewiseComparator() nogil except +\n\nctypedef int (*compare_func)(\n void*,\n Logger*,\n string&,\n const Slice&,\n const Slice&)\n\ncdef extern from \"cpp/comparator_wrapper.hpp\" namespace \"py_rocks\":\n cdef cppclass ComparatorWrapper:\n ComparatorWrapper(string, void*, compare_func) nogil except +\n void set_info_log(shared_ptr[Logger]) nogil except+\n" }, { "path": "rocksdb/cpp/comparator_wrapper.hpp", "content": "#include \"rocksdb/comparator.h\"\n#include \"rocksdb/env.h\"\n#include \n\nusing std::string;\nusing rocksdb::Comparator;\nusing rocksdb::Slice;\nusing rocksdb::Logger;\n\nnamespace py_rocks {\n class ComparatorWrapper: public Comparator {\n public:\n typedef int (*compare_func)(\n void*,\n Logger*,\n string&,\n const Slice&,\n const Slice&);\n\n ComparatorWrapper(\n string name,\n void* compare_context,\n compare_func compare_callback):\n name(name),\n compare_context(compare_context),\n compare_callback(compare_callback)\n {}\n\n virtual int Compare(const Slice& a, const Slice& b) const {\n string error_msg;\n int val;\n\n val = this->compare_callback(\n this->compare_context,\n this->info_log.get(),\n error_msg,\n a,\n b);\n\n if (error_msg.size()) {\n throw std::runtime_error(error_msg.c_str());\n }\n return val;\n }\n\n virtual const char* Name() const {\n return this->name.c_str();\n }\n\n virtual void FindShortestSeparator(string*, const Slice&) const {}\n virtual void FindShortSuccessor(string*) const {}\n\n void set_info_log(std::shared_ptr info_log) {\n this->info_log = info_log;\n }\n\n private:\n string name;\n void* compare_context;\n compare_func compare_callback;\n std::shared_ptr info_log;\n };\n}\n" }, { "path": "rocksdb/cpp/filter_policy_wrapper.hpp", "content": "#include \"rocksdb/filter_policy.h\"\n#include \"rocksdb/env.h\"\n#include \n\nusing std::string;\nusing rocksdb::FilterPolicy;\nusing rocksdb::Slice;\nusing rocksdb::Logger;\n\nnamespace py_rocks {\n class FilterPolicyWrapper: public FilterPolicy {\n public:\n typedef void (*create_filter_func)(\n void* ctx,\n Logger*,\n string&,\n const Slice* keys,\n int n,\n string* dst);\n\n typedef bool (*key_may_match_func)(\n void* ctx,\n Logger*,\n string&,\n const Slice& key,\n const Slice& filter);\n\n FilterPolicyWrapper(\n string name,\n void* ctx,\n create_filter_func create_filter_callback,\n key_may_match_func key_may_match_callback):\n name(name),\n ctx(ctx),\n create_filter_callback(create_filter_callback),\n key_may_match_callback(key_may_match_callback)\n {}\n\n virtual void\n CreateFilter(const Slice* keys, int n, std::string* dst) const {\n string error_msg;\n\n this->create_filter_callback(\n this->ctx,\n this->info_log.get(),\n error_msg,\n keys,\n n,\n dst);\n\n if (error_msg.size()) {\n throw std::runtime_error(error_msg.c_str());\n }\n }\n\n virtual bool\n KeyMayMatch(const Slice& key, const Slice& filter) const {\n string error_msg;\n bool val;\n\n val = this->key_may_match_callback(\n this->ctx,\n this->info_log.get(),\n error_msg,\n key,\n filter);\n\n if (error_msg.size()) {\n throw std::runtime_error(error_msg.c_str());\n }\n return val;\n }\n\n virtual const char* Name() const {\n return this->name.c_str();\n }\n\n void set_info_log(std::shared_ptr info_log) {\n this->info_log = info_log;\n }\n\n private:\n string name;\n void* ctx;\n create_filter_func create_filter_callback;\n key_may_match_func key_may_match_callback;\n std::shared_ptr info_log;\n };\n}\n" }, { "path": "rocksdb/cpp/memtable_factories.hpp", "content": "#include \"rocksdb/memtablerep.h\"\n\nusing rocksdb::MemTableRepFactory;\nusing rocksdb::VectorRepFactory;\nusing rocksdb::SkipListFactory;\n\nnamespace py_rocks {\n MemTableRepFactory* NewVectorRepFactory(size_t count = 0) {\n return new VectorRepFactory(count);\n }\n\n MemTableRepFactory* NewSkipListFactory() {\n return new SkipListFactory();\n }\n}\n" }, { "path": "rocksdb/cpp/merge_operator_wrapper.hpp", "content": "#include \"rocksdb/merge_operator.h\"\n\nusing std::string;\nusing std::deque;\nusing rocksdb::Slice;\nusing rocksdb::Logger;\nusing rocksdb::MergeOperator;\nusing rocksdb::AssociativeMergeOperator;\n\nnamespace py_rocks {\n class AssociativeMergeOperatorWrapper: public AssociativeMergeOperator {\n public:\n typedef bool (*merge_func)(\n void*, \n const Slice& key,\n const Slice* existing_value,\n const Slice& value,\n std::string* new_value,\n Logger* logger);\n\n\n AssociativeMergeOperatorWrapper(\n string name,\n void* merge_context,\n merge_func merge_callback):\n name(name),\n merge_context(merge_context),\n merge_callback(merge_callback)\n {}\n\n virtual bool Merge(\n const Slice& key,\n const Slice* existing_value,\n const Slice& value,\n std::string* new_value,\n Logger* logger) const \n {\n return this->merge_callback(\n this->merge_context,\n key,\n existing_value,\n value,\n new_value,\n logger);\n }\n\n virtual const char* Name() const {\n return this->name.c_str();\n }\n\n private:\n string name;\n void* merge_context;\n merge_func merge_callback;\n };\n\n class MergeOperatorWrapper: public MergeOperator {\n public:\n typedef bool (*full_merge_func)(\n void* ctx,\n const Slice& key,\n const Slice* existing_value,\n const deque& operand_list,\n string* new_value,\n Logger* logger);\n\n typedef bool (*partial_merge_func)(\n void* ctx,\n const Slice& key,\n const Slice& left_op,\n const Slice& right_op,\n string* new_value,\n Logger* logger);\n\n MergeOperatorWrapper(\n string name,\n void* full_merge_context,\n void* partial_merge_context,\n full_merge_func full_merge_callback,\n partial_merge_func partial_merge_callback):\n name(name),\n full_merge_context(full_merge_context),\n partial_merge_context(partial_merge_context),\n full_merge_callback(full_merge_callback),\n partial_merge_callback(partial_merge_callback)\n {}\n\n virtual bool FullMerge(\n const Slice& key,\n const Slice* existing_value,\n const deque& operand_list,\n string* new_value,\n Logger* logger) const \n {\n return this->full_merge_callback(\n this->full_merge_context,\n key,\n existing_value,\n operand_list,\n new_value,\n logger);\n }\n\n virtual bool PartialMerge (\n const Slice& key,\n const Slice& left_operand,\n const Slice& right_operand,\n string* new_value,\n Logger* logger) const\n {\n return this->partial_merge_callback(\n this->partial_merge_context,\n key,\n left_operand,\n right_operand,\n new_value,\n logger);\n }\n \n virtual const char* Name() const {\n return this->name.c_str();\n }\n\n private:\n string name;\n void* full_merge_context;\n void* partial_merge_context;\n full_merge_func full_merge_callback;\n partial_merge_func partial_merge_callback;\n\n };\n}\n" }, { "path": "rocksdb/cpp/slice_transform_wrapper.hpp", "content": "#include \n#include \"rocksdb/slice_transform.h\"\n#include \"rocksdb/env.h\"\n#include \n\nusing std::string;\nusing rocksdb::SliceTransform;\nusing rocksdb::Slice;\nusing rocksdb::Logger;\n\nnamespace py_rocks {\n class SliceTransformWrapper: public SliceTransform {\n public:\n typedef Slice (*transform_func)(\n void*,\n Logger*,\n string&,\n const Slice&);\n\n typedef bool (*in_domain_func)(\n void*,\n Logger*,\n string&,\n const Slice&);\n\n typedef bool (*in_range_func)(\n void*,\n Logger*,\n string&,\n const Slice&);\n\n SliceTransformWrapper(\n string name,\n void* ctx,\n transform_func transfrom_callback,\n in_domain_func in_domain_callback,\n in_range_func in_range_callback):\n name(name),\n ctx(ctx),\n transfrom_callback(transfrom_callback),\n in_domain_callback(in_domain_callback),\n in_range_callback(in_range_callback)\n {}\n\n virtual const char* Name() const {\n return this->name.c_str();\n }\n\n virtual Slice Transform(const Slice& src) const {\n string error_msg;\n Slice val;\n\n val = this->transfrom_callback(\n this->ctx,\n this->info_log.get(),\n error_msg,\n src);\n\n if (error_msg.size()) {\n throw std::runtime_error(error_msg.c_str());\n }\n return val;\n }\n\n virtual bool InDomain(const Slice& src) const {\n string error_msg;\n bool val;\n\n val = this->in_domain_callback(\n this->ctx,\n this->info_log.get(),\n error_msg,\n src);\n\n if (error_msg.size()) {\n throw std::runtime_error(error_msg.c_str());\n }\n return val;\n }\n\n virtual bool InRange(const Slice& dst) const {\n string error_msg;\n bool val;\n\n val = this->in_range_callback(\n this->ctx,\n this->info_log.get(),\n error_msg,\n dst);\n\n if (error_msg.size()) {\n throw std::runtime_error(error_msg.c_str());\n }\n return val;\n }\n\n void set_info_log(std::shared_ptr info_log) {\n this->info_log = info_log;\n }\n\n private:\n string name;\n void* ctx;\n transform_func transfrom_callback;\n in_domain_func in_domain_callback;\n in_range_func in_range_callback;\n std::shared_ptr info_log;\n };\n}\n" }, { "path": "rocksdb/cpp/utils.hpp", "content": "#include \n\nnamespace py_rocks {\n template \n const T* vector_data(std::vector& v) {\n return v.data();\n }\n}\n" }, { "path": "rocksdb/cpp/write_batch_iter_helper.hpp", "content": "#pragma once\n\n#include \n#include \"rocksdb/write_batch.h\"\n\nnamespace py_rocks {\n\nclass RecordItemsHandler: public rocksdb::WriteBatch::Handler {\n public:\n enum Optype {PutRecord, MergeRecord, DeleteRecord};\n\n class BatchItem {\n public:\n BatchItem(\n const Optype& op,\n const rocksdb::Slice& key,\n const rocksdb::Slice& value):\n op(op),\n key(key),\n value(value)\n {}\n\n const Optype op;\n const rocksdb::Slice key;\n const rocksdb::Slice value;\n };\n\n typedef std::vector BatchItems;\n\n public:\n /* Items is filled during iteration. */\n RecordItemsHandler(BatchItems* items): items(items) {}\n\n void Put(const Slice& key, const Slice& value) {\n this->items->emplace_back(PutRecord, key, value);\n }\n\n void Merge(const Slice& key, const Slice& value) {\n this->items->emplace_back(MergeRecord, key, value);\n }\n\n virtual void Delete(const Slice& key) {\n this->items->emplace_back(DeleteRecord, key, rocksdb::Slice());\n } \n\n private:\n BatchItems* items;\n};\n\nrocksdb::Status\nget_batch_items(const rocksdb::WriteBatch* batch, RecordItemsHandler::BatchItems* items) {\n RecordItemsHandler handler(items);\n return batch->Iterate(&handler);\n}\n\n}\n" }, { "path": "rocksdb/db.pxd", "content": "cimport options\nfrom libc.stdint cimport uint64_t\nfrom status cimport Status\nfrom libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\nfrom libcpp.vector cimport vector\nfrom slice_ cimport Slice\nfrom snapshot cimport Snapshot\nfrom iterator cimport Iterator\n\ncdef extern from \"rocksdb/write_batch.h\" namespace \"rocksdb\":\n cdef cppclass WriteBatch:\n WriteBatch() nogil except+\n WriteBatch(string) nogil except+\n void Put(const Slice&, const Slice&) nogil except+\n void Merge(const Slice&, const Slice&) nogil except+\n void Delete(const Slice&) nogil except+\n void PutLogData(const Slice&) nogil except+\n void Clear() nogil except+\n const string& Data() nogil except+\n int Count() nogil except+\n\ncdef extern from \"cpp/write_batch_iter_helper.hpp\" namespace \"py_rocks\":\n cdef enum BatchItemOp \"RecordItemsHandler::Optype\":\n BatchItemOpPut \"py_rocks::RecordItemsHandler::Optype::PutRecord\"\n BatchItemOpMerge \"py_rocks::RecordItemsHandler::Optype::MergeRecord\"\n BatchItemOpDelte \"py_rocks::RecordItemsHandler::Optype::DeleteRecord\"\n\n cdef cppclass BatchItem \"py_rocks::RecordItemsHandler::BatchItem\":\n BatchItemOp op\n Slice key\n Slice value\n\n Status get_batch_items(WriteBatch* batch, vector[BatchItem]* items)\n\n\ncdef extern from \"rocksdb/db.h\" namespace \"rocksdb\":\n ctypedef uint64_t SequenceNumber\n\n cdef struct LiveFileMetaData:\n string name\n int level\n uint64_t size\n string smallestkey\n string largestkey\n SequenceNumber smallest_seqno\n SequenceNumber largest_seqno\n\n cdef cppclass Range:\n Range(const Slice&, const Slice&)\n\n cdef cppclass DB:\n Status Put(\n const options.WriteOptions&,\n const Slice&,\n const Slice&) nogil except+\n\n Status Delete(\n const options.WriteOptions&,\n const Slice&) nogil except+\n\n Status Merge(\n const options.WriteOptions&,\n const Slice&,\n const Slice&) nogil except+\n\n Status Write(\n const options.WriteOptions&,\n WriteBatch*) nogil except+\n\n Status Get(\n const options.ReadOptions&,\n const Slice&,\n string*) nogil except+\n\n vector[Status] MultiGet(\n const options.ReadOptions&,\n const vector[Slice]&,\n vector[string]*) nogil except+\n\n cpp_bool KeyMayExist(\n const options.ReadOptions&,\n Slice&,\n string*,\n cpp_bool*) nogil except+\n\n cpp_bool KeyMayExist(\n const options.ReadOptions&,\n Slice&,\n string*) nogil except+\n\n Iterator* NewIterator(\n const options.ReadOptions&) nogil except+\n\n const Snapshot* GetSnapshot() nogil except+\n\n void ReleaseSnapshot(const Snapshot*) nogil except+\n\n cpp_bool GetProperty(\n const Slice&,\n string*) nogil except+\n\n void GetApproximateSizes(\n const Range*\n int,\n uint64_t*) nogil except+\n\n Status CompactRange(\n const options.CompactRangeOptions&,\n const Slice*,\n const Slice*) nogil except+\n\n int NumberLevels() nogil except+\n int MaxMemCompactionLevel() nogil except+\n int Level0StopWriteTrigger() nogil except+\n const string& GetName() nogil except+\n const options.Options& GetOptions() nogil except+\n Status Flush(const options.FlushOptions&) nogil except+\n Status DisableFileDeletions() nogil except+\n Status EnableFileDeletions() nogil except+\n\n # TODO: Status GetSortedWalFiles(VectorLogPtr& files)\n # TODO: SequenceNumber GetLatestSequenceNumber()\n # TODO: Status GetUpdatesSince(\n # SequenceNumber seq_number,\n # unique_ptr[TransactionLogIterator]*)\n\n Status DeleteFile(string) nogil except+\n void GetLiveFilesMetaData(vector[LiveFileMetaData]*) nogil except+\n\n\n cdef Status DB_Open \"rocksdb::DB::Open\"(\n const options.Options&,\n const string&,\n DB**) nogil except+\n\n cdef Status DB_OpenForReadOnly \"rocksdb::DB::OpenForReadOnly\"(\n const options.Options&,\n const string&,\n DB**,\n cpp_bool) nogil except+\n\n cdef Status RepairDB(const string& dbname, const options.Options&)\n" }, { "path": "rocksdb/env.pxd", "content": "cdef extern from \"rocksdb/env.h\" namespace \"rocksdb\":\n cdef cppclass Env:\n Env()\n\n cdef Env* Env_Default \"rocksdb::Env::Default\"()\n" }, { "path": "rocksdb/errors.py", "content": "class NotFound(Exception):\n pass\n\nclass Corruption(Exception):\n pass\n\nclass NotSupported(Exception):\n pass\n\nclass InvalidArgument(Exception):\n pass\n\nclass RocksIOError(Exception):\n pass\n\nclass MergeInProgress(Exception):\n pass\n\nclass Incomplete(Exception):\n pass\n" }, { "path": "rocksdb/filter_policy.pxd", "content": "from libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\nfrom libc.string cimport const_char\nfrom slice_ cimport Slice\nfrom std_memory cimport shared_ptr\nfrom logger cimport Logger\n\ncdef extern from \"rocksdb/filter_policy.h\" namespace \"rocksdb\":\n cdef cppclass FilterPolicy:\n void CreateFilter(const Slice*, int, string*) nogil except+\n cpp_bool KeyMayMatch(const Slice&, const Slice&) nogil except+\n const_char* Name() nogil except+\n\n cdef extern const FilterPolicy* NewBloomFilterPolicy(int) nogil except+\n\nctypedef void (*create_filter_func)(\n void*,\n Logger*,\n string&,\n const Slice*,\n int,\n string*)\n\nctypedef cpp_bool (*key_may_match_func)(\n void*,\n Logger*,\n string&,\n const Slice&,\n const Slice&)\n\ncdef extern from \"cpp/filter_policy_wrapper.hpp\" namespace \"py_rocks\":\n cdef cppclass FilterPolicyWrapper:\n FilterPolicyWrapper(\n string,\n void*,\n create_filter_func,\n key_may_match_func) nogil except+\n\n void set_info_log(shared_ptr[Logger]) nogil except+\n" }, { "path": "rocksdb/interfaces.py", "content": "from abc import ABCMeta\nfrom abc import abstractmethod\n\n\nclass Comparator:\n __metaclass__ = ABCMeta\n\n @abstractmethod\n def compare(self, a, b):\n pass\n\n @abstractmethod\n def name(self):\n pass\n\n\nclass AssociativeMergeOperator:\n __metaclass__ = ABCMeta\n\n @abstractmethod\n def merge(self, key, existing_value, value):\n pass\n\n @abstractmethod\n def name(self):\n pass\n\n\nclass MergeOperator:\n __metaclass__ = ABCMeta\n\n @abstractmethod\n def full_merge(self, key, existing_value, operand_list):\n pass\n\n @abstractmethod\n def partial_merge(self, key, left_operand, right_operand):\n pass\n\n @abstractmethod\n def name(self):\n pass\n\n\nclass FilterPolicy:\n __metaclass__ = ABCMeta\n\n @abstractmethod\n def name(self):\n pass\n\n @abstractmethod\n def create_filter(self, keys):\n pass\n\n @abstractmethod\n def key_may_match(self, key, filter_):\n pass\n\nclass SliceTransform:\n __metaclass__ = ABCMeta\n\n @abstractmethod\n def name(self):\n pass\n\n @abstractmethod\n def transform(self, src):\n pass\n\n @abstractmethod\n def in_domain(self, src):\n pass\n\n @abstractmethod\n def in_range(self, dst):\n pass\n" }, { "path": "rocksdb/iterator.pxd", "content": "from libcpp cimport bool as cpp_bool\nfrom slice_ cimport Slice\nfrom status cimport Status\n\ncdef extern from \"rocksdb/iterator.h\" namespace \"rocksdb\":\n cdef cppclass Iterator:\n cpp_bool Valid() nogil except+\n void SeekToFirst() nogil except+\n void SeekToLast() nogil except+\n void Seek(const Slice&) nogil except+\n void Next() nogil except+\n void Prev() nogil except+\n Slice key() nogil except+\n Slice value() nogil except+\n Status status() nogil except+\n" }, { "path": "rocksdb/logger.pxd", "content": "cdef extern from \"rocksdb/env.h\" namespace \"rocksdb\":\n cdef cppclass Logger:\n pass\n\n void Log(Logger*, const char*, ...) nogil except+\n" }, { "path": "rocksdb/memtablerep.pxd", "content": "from libc.stdint cimport int32_t\n\ncdef extern from \"rocksdb/memtablerep.h\" namespace \"rocksdb\":\n cdef cppclass MemTableRepFactory:\n MemTableRepFactory()\n\n cdef MemTableRepFactory* NewHashSkipListRepFactory(size_t, int32_t, int32_t)\n cdef MemTableRepFactory* NewHashLinkListRepFactory(size_t)\n\ncdef extern from \"cpp/memtable_factories.hpp\" namespace \"py_rocks\":\n cdef MemTableRepFactory* NewVectorRepFactory(size_t)\n cdef MemTableRepFactory* NewSkipListFactory()\n" }, { "path": "rocksdb/merge_operator.pxd", "content": "from libcpp.string cimport string\nfrom libcpp cimport bool as cpp_bool\nfrom libcpp.deque cimport deque\nfrom slice_ cimport Slice\nfrom logger cimport Logger\n\ncdef extern from \"rocksdb/merge_operator.h\" namespace \"rocksdb\":\n cdef cppclass MergeOperator:\n pass\n\nctypedef cpp_bool (*merge_func)(\n void*,\n const Slice&,\n const Slice*,\n const Slice&,\n string*,\n Logger*)\n\nctypedef cpp_bool (*full_merge_func)(\n void* ctx,\n const Slice& key,\n const Slice* existing_value,\n const deque[string]& operand_list,\n string* new_value,\n Logger* logger)\n\nctypedef cpp_bool (*partial_merge_func)(\n void* ctx,\n const Slice& key,\n const Slice& left_op,\n const Slice& right_op,\n string* new_value,\n Logger* logger)\n\ncdef extern from \"cpp/merge_operator_wrapper.hpp\" namespace \"py_rocks\":\n cdef cppclass AssociativeMergeOperatorWrapper:\n AssociativeMergeOperatorWrapper(string, void*, merge_func) nogil except+\n\n cdef cppclass MergeOperatorWrapper:\n MergeOperatorWrapper(\n string,\n void*,\n void*,\n full_merge_func,\n partial_merge_func) nogil except+\n" }, { "path": "rocksdb/options.pxd", "content": "from libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\nfrom libcpp.vector cimport vector\nfrom libc.stdint cimport uint64_t\nfrom libc.stdint cimport uint32_t\nfrom std_memory cimport shared_ptr\nfrom comparator cimport Comparator\nfrom merge_operator cimport MergeOperator\nfrom logger cimport Logger\nfrom slice_ cimport Slice\nfrom snapshot cimport Snapshot\nfrom slice_transform cimport SliceTransform\nfrom table_factory cimport TableFactory\nfrom memtablerep cimport MemTableRepFactory\nfrom universal_compaction cimport CompactionOptionsUniversal\nfrom cache cimport Cache\n\ncdef extern from \"rocksdb/options.h\" namespace \"rocksdb\":\n ctypedef enum CompactionStyle:\n kCompactionStyleLevel\n kCompactionStyleUniversal\n\n ctypedef enum CompressionType:\n kNoCompression\n kSnappyCompression\n kZlibCompression\n kBZip2Compression\n kLZ4Compression\n kLZ4HCCompression\n\n ctypedef enum ReadTier:\n kReadAllTier\n kBlockCacheTier\n\n cdef cppclass Options:\n const Comparator* comparator\n shared_ptr[MergeOperator] merge_operator\n # TODO: compaction_filter\n # TODO: compaction_filter_factory\n cpp_bool create_if_missing\n cpp_bool error_if_exists\n cpp_bool paranoid_checks\n # TODO: env\n shared_ptr[Logger] info_log\n size_t write_buffer_size\n int max_write_buffer_number\n int min_write_buffer_number_to_merge\n int max_open_files\n CompressionType compression\n # TODO: compression_per_level\n # TODO: compression_opts\n shared_ptr[SliceTransform] prefix_extractor\n int num_levels\n int level0_file_num_compaction_trigger\n int level0_slowdown_writes_trigger\n int level0_stop_writes_trigger\n int max_mem_compaction_level\n uint64_t target_file_size_base\n int target_file_size_multiplier\n uint64_t max_bytes_for_level_base\n int max_bytes_for_level_multiplier\n vector[int] max_bytes_for_level_multiplier_additional\n int expanded_compaction_factor\n int source_compaction_factor\n int max_grandparent_overlap_factor\n # TODO: statistics\n cpp_bool disableDataSync\n cpp_bool use_fsync\n string db_log_dir\n string wal_dir\n uint64_t delete_obsolete_files_period_micros\n int max_background_compactions\n int max_background_flushes\n size_t max_log_file_size\n size_t log_file_time_to_roll\n size_t keep_log_file_num\n double soft_rate_limit\n double hard_rate_limit\n unsigned int rate_limit_delay_max_milliseconds\n uint64_t max_manifest_file_size\n int table_cache_numshardbits\n size_t arena_block_size\n # TODO: PrepareForBulkLoad()\n cpp_bool disable_auto_compactions\n uint64_t WAL_ttl_seconds\n uint64_t WAL_size_limit_MB\n size_t manifest_preallocation_size\n cpp_bool purge_redundant_kvs_while_flush\n cpp_bool allow_os_buffer\n cpp_bool allow_mmap_reads\n cpp_bool allow_mmap_writes\n cpp_bool is_fd_close_on_exec\n cpp_bool skip_log_error_on_recovery\n unsigned int stats_dump_period_sec\n cpp_bool advise_random_on_open\n # TODO: enum { NONE, NORMAL, SEQUENTIAL, WILLNEED } access_hint_on_compaction_start\n cpp_bool use_adaptive_mutex\n uint64_t bytes_per_sync\n cpp_bool verify_checksums_in_compaction\n CompactionStyle compaction_style\n CompactionOptionsUniversal compaction_options_universal\n cpp_bool filter_deletes\n uint64_t max_sequential_skip_in_iterations\n shared_ptr[MemTableRepFactory] memtable_factory\n shared_ptr[TableFactory] table_factory\n # TODO: table_properties_collectors\n cpp_bool inplace_update_support\n size_t inplace_update_num_locks\n shared_ptr[Cache] row_cache\n\n cdef cppclass WriteOptions:\n cpp_bool sync\n cpp_bool disableWAL\n\n cdef cppclass ReadOptions:\n cpp_bool verify_checksums\n cpp_bool fill_cache\n const Snapshot* snapshot\n ReadTier read_tier\n\n cdef cppclass FlushOptions:\n cpp_bool wait\n\n ctypedef enum BottommostLevelCompaction:\n blc_skip \"rocksdb::BottommostLevelCompaction::kSkip\"\n blc_is_filter \"rocksdb::BottommostLevelCompaction::kIfHaveCompactionFilter\"\n blc_force \"rocksdb::BottommostLevelCompaction::kForce\"\n\n cdef cppclass CompactRangeOptions:\n cpp_bool change_level\n int target_level\n uint32_t target_path_id\n BottommostLevelCompaction bottommost_level_compaction\n" }, { "path": "rocksdb/slice_.pxd", "content": "from libcpp.string cimport string\nfrom libcpp cimport bool as cpp_bool\n\ncdef extern from \"rocksdb/slice.h\" namespace \"rocksdb\":\n cdef cppclass Slice:\n Slice() nogil\n Slice(const char*, size_t) nogil\n Slice(const string&) nogil\n Slice(const char*) nogil\n\n const char* data() nogil\n size_t size() nogil\n cpp_bool empty() nogil\n char operator[](int) nogil\n void clear() nogil\n void remove_prefix(size_t) nogil\n string ToString() nogil\n string ToString(cpp_bool) nogil\n int compare(const Slice&) nogil\n cpp_bool starts_with(const Slice&) nogil\n" }, { "path": "rocksdb/slice_transform.pxd", "content": "from slice_ cimport Slice\nfrom libcpp.string cimport string\nfrom libcpp cimport bool as cpp_bool\nfrom logger cimport Logger\nfrom std_memory cimport shared_ptr\n\ncdef extern from \"rocksdb/slice_transform.h\" namespace \"rocksdb\":\n cdef cppclass SliceTransform:\n pass\n\nctypedef Slice (*transform_func)(\n void*,\n Logger*,\n string&,\n const Slice&)\n\nctypedef cpp_bool (*in_domain_func)(\n void*,\n Logger*,\n string&,\n const Slice&)\n\nctypedef cpp_bool (*in_range_func)(\n void*,\n Logger*,\n string&,\n const Slice&)\n\ncdef extern from \"cpp/slice_transform_wrapper.hpp\" namespace \"py_rocks\":\n cdef cppclass SliceTransformWrapper:\n SliceTransformWrapper(\n string name,\n void*,\n transform_func,\n in_domain_func,\n in_range_func) nogil except+\n void set_info_log(shared_ptr[Logger]) nogil except+\n" }, { "path": "rocksdb/snapshot.pxd", "content": "cdef extern from \"rocksdb/db.h\" namespace \"rocksdb\":\n cdef cppclass Snapshot:\n pass\n" }, { "path": "rocksdb/status.pxd", "content": "from libcpp cimport bool as cpp_bool\nfrom libcpp.string cimport string\n\ncdef extern from \"rocksdb/status.h\" namespace \"rocksdb\":\n cdef cppclass Status:\n Status()\n cpp_bool ok() nogil\n cpp_bool IsNotFound() nogil const\n cpp_bool IsCorruption() nogil const\n cpp_bool IsNotSupported() nogil const\n cpp_bool IsInvalidArgument() nogil const\n cpp_bool IsIOError() nogil const\n cpp_bool IsMergeInProgress() nogil const\n cpp_bool IsIncomplete() nogil const\n string ToString() nogil except+\n" }, { "path": "rocksdb/std_memory.pxd", "content": "cdef extern from \"\" namespace \"std\":\n cdef cppclass shared_ptr[T]:\n shared_ptr() nogil except+\n shared_ptr(T*) nogil except+\n void reset() nogil except+\n void reset(T*) nogil except+\n T* get() nogil except+\n" }, { "path": "rocksdb/table_factory.pxd", "content": "from libc.stdint cimport uint32_t\nfrom libcpp cimport bool as cpp_bool\nfrom std_memory cimport shared_ptr\n\nfrom cache cimport Cache\nfrom filter_policy cimport FilterPolicy\n\ncdef extern from \"rocksdb/table.h\" namespace \"rocksdb\":\n cdef cppclass TableFactory:\n TableFactory()\n\n ctypedef enum BlockBasedTableIndexType:\n kBinarySearch \"rocksdb::BlockBasedTableOptions::IndexType::kBinarySearch\"\n kHashSearch \"rocksdb::BlockBasedTableOptions::IndexType::kHashSearch\"\n\n ctypedef enum ChecksumType:\n kCRC32c\n kxxHash\n\n cdef cppclass BlockBasedTableOptions:\n BlockBasedTableOptions()\n BlockBasedTableIndexType index_type\n cpp_bool hash_index_allow_collision\n ChecksumType checksum\n cpp_bool no_block_cache\n size_t block_size\n int block_size_deviation\n int block_restart_interval\n cpp_bool whole_key_filtering\n shared_ptr[Cache] block_cache\n shared_ptr[Cache] block_cache_compressed\n shared_ptr[FilterPolicy] filter_policy\n\n cdef TableFactory* NewBlockBasedTableFactory(const BlockBasedTableOptions&)\n\n ctypedef enum EncodingType:\n kPlain\n kPrefix\n\n cdef cppclass PlainTableOptions:\n uint32_t user_key_len\n int bloom_bits_per_key\n double hash_table_ratio\n size_t index_sparseness\n size_t huge_page_tlb_size\n EncodingType encoding_type\n cpp_bool full_scan_mode\n cpp_bool store_index_in_file\n\n cdef TableFactory* NewPlainTableFactory(const PlainTableOptions&)\n" }, { "path": "rocksdb/tests/__init__.py", "content": "" }, { "path": "rocksdb/tests/test_db.py", "content": "import os\nimport shutil\nimport gc\nimport unittest\nimport rocksdb\nfrom itertools import takewhile\n\ndef int_to_bytes(ob):\n return str(ob).encode('ascii')\n\nclass TestHelper(object):\n def _clean(self):\n if os.path.exists('/tmp/test'):\n shutil.rmtree(\"/tmp/test\")\n\n def _close_db(self):\n del self.db\n gc.collect()\n\n\nclass TestDB(unittest.TestCase, TestHelper):\n def setUp(self):\n opts = rocksdb.Options(create_if_missing=True)\n self._clean()\n self.db = rocksdb.DB(\"/tmp/test\", opts)\n\n def tearDown(self):\n self._close_db()\n\n def test_options_used_twice(self):\n expected = \"Options object is already used by another DB\"\n with self.assertRaisesRegexp(Exception, expected):\n rocksdb.DB(\"/tmp/test2\", self.db.options)\n\n def test_unicode_path(self):\n name = b'/tmp/M\\xc3\\xbcnchen'.decode('utf8')\n rocksdb.DB(name, rocksdb.Options(create_if_missing=True))\n self.addCleanup(shutil.rmtree, name)\n self.assertTrue(os.path.isdir(name))\n\n def test_get_none(self):\n self.assertIsNone(self.db.get(b'xxx'))\n\n def test_put_get(self):\n self.db.put(b\"a\", b\"b\")\n self.assertEqual(b\"b\", self.db.get(b\"a\"))\n\n def test_multi_get(self):\n self.db.put(b\"a\", b\"1\")\n self.db.put(b\"b\", b\"2\")\n self.db.put(b\"c\", b\"3\")\n\n ret = self.db.multi_get([b'a', b'b', b'c'])\n ref = {b'a': b'1', b'c': b'3', b'b': b'2'}\n self.assertEqual(ref, ret)\n\n def test_delete(self):\n self.db.put(b\"a\", b\"b\")\n self.assertEqual(b\"b\", self.db.get(b\"a\"))\n self.db.delete(b\"a\")\n self.assertIsNone(self.db.get(b\"a\"))\n\n def test_write_batch(self):\n batch = rocksdb.WriteBatch()\n batch.put(b\"key\", b\"v1\")\n batch.delete(b\"key\")\n batch.put(b\"key\", b\"v2\")\n batch.put(b\"key\", b\"v3\")\n batch.put(b\"a\", b\"b\")\n\n self.db.write(batch)\n ref = {b'a': b'b', b'key': b'v3'}\n ret = self.db.multi_get([b'key', b'a'])\n self.assertEqual(ref, ret)\n\n def test_write_batch_iter(self):\n batch = rocksdb.WriteBatch()\n self.assertEqual([], list(batch))\n\n batch.put(b\"key1\", b\"v1\")\n batch.put(b\"key2\", b\"v2\")\n batch.put(b\"key3\", b\"v3\")\n batch.delete(b'a')\n batch.delete(b'key1')\n batch.merge(b'xxx', b'value')\n\n it = iter(batch)\n del batch\n ref = [\n ('Put', 'key1', 'v1'),\n ('Put', 'key2', 'v2'),\n ('Put', 'key3', 'v3'),\n ('Delete', 'a', ''),\n ('Delete', 'key1', ''),\n ('Merge', 'xxx', 'value')\n ]\n self.assertEqual(ref, list(it))\n\n\n def test_key_may_exists(self):\n self.db.put(b\"a\", b'1')\n\n self.assertEqual((False, None), self.db.key_may_exist(b\"x\"))\n self.assertEqual((False, None), self.db.key_may_exist(b'x', True))\n self.assertEqual((True, None), self.db.key_may_exist(b'a'))\n self.assertEqual((True, b'1'), self.db.key_may_exist(b'a', True))\n\n def test_iter_keys(self):\n for x in range(300):\n self.db.put(int_to_bytes(x), int_to_bytes(x))\n\n it = self.db.iterkeys()\n\n self.assertEqual([], list(it))\n\n it.seek_to_last()\n self.assertEqual([b'99'], list(it))\n\n ref = sorted([int_to_bytes(x) for x in range(300)])\n it.seek_to_first()\n self.assertEqual(ref, list(it))\n\n it.seek(b'90')\n ref = [\n b'90',\n b'91',\n b'92',\n b'93',\n b'94',\n b'95',\n b'96',\n b'97',\n b'98',\n b'99'\n ]\n self.assertEqual(ref, list(it))\n\n def test_iter_values(self):\n for x in range(300):\n self.db.put(int_to_bytes(x), int_to_bytes(x * 1000))\n\n it = self.db.itervalues()\n\n self.assertEqual([], list(it))\n\n it.seek_to_last()\n self.assertEqual([b'99000'], list(it))\n\n ref = sorted([int_to_bytes(x) for x in range(300)])\n ref = [int_to_bytes(int(x) * 1000) for x in ref]\n it.seek_to_first()\n self.assertEqual(ref, list(it))\n\n it.seek(b'90')\n ref = [int_to_bytes(x * 1000) for x in range(90, 100)]\n self.assertEqual(ref, list(it))\n\n def test_iter_items(self):\n for x in range(300):\n self.db.put(int_to_bytes(x), int_to_bytes(x * 1000))\n\n it = self.db.iteritems()\n\n self.assertEqual([], list(it))\n\n it.seek_to_last()\n self.assertEqual([(b'99', b'99000')], list(it))\n\n ref = sorted([int_to_bytes(x) for x in range(300)])\n ref = [(x, int_to_bytes(int(x) * 1000)) for x in ref]\n it.seek_to_first()\n self.assertEqual(ref, list(it))\n\n it.seek(b'90')\n ref = [(int_to_bytes(x), int_to_bytes(x * 1000)) for x in range(90, 100)]\n self.assertEqual(ref, list(it))\n\n def test_reverse_iter(self):\n for x in range(100):\n self.db.put(int_to_bytes(x), int_to_bytes(x * 1000))\n\n it = self.db.iteritems()\n it.seek_to_last()\n\n ref = reversed(sorted([int_to_bytes(x) for x in range(100)]))\n ref = [(x, int_to_bytes(int(x) * 1000)) for x in ref]\n\n self.assertEqual(ref, list(reversed(it)))\n\n def test_snapshot(self):\n self.db.put(b\"a\", b\"1\")\n self.db.put(b\"b\", b\"2\")\n\n snapshot = self.db.snapshot()\n self.db.put(b\"a\", b\"2\")\n self.db.delete(b\"b\")\n\n it = self.db.iteritems()\n it.seek_to_first()\n self.assertEqual({b'a': b'2'}, dict(it))\n\n it = self.db.iteritems(snapshot=snapshot)\n it.seek_to_first()\n self.assertEqual({b'a': b'1', b'b': b'2'}, dict(it))\n\n def test_get_property(self):\n for x in range(300):\n x = int_to_bytes(x)\n self.db.put(x, x)\n\n self.assertIsNotNone(self.db.get_property(b'rocksdb.stats'))\n self.assertIsNotNone(self.db.get_property(b'rocksdb.sstables'))\n self.assertIsNotNone(self.db.get_property(b'rocksdb.num-files-at-level0'))\n self.assertIsNone(self.db.get_property(b'does not exsits'))\n\n def test_compact_range(self):\n for x in range(10000):\n x = int_to_bytes(x)\n self.db.put(x, x)\n\n self.db.compact_range()\n\n\nclass AssocCounter(rocksdb.interfaces.AssociativeMergeOperator):\n def merge(self, key, existing_value, value):\n if existing_value:\n return (True, int_to_bytes(int(existing_value) + int(value)))\n return (True, value)\n\n def name(self):\n return b'AssocCounter'\n\n\nclass TestAssocMerge(unittest.TestCase, TestHelper):\n def setUp(self):\n opts = rocksdb.Options()\n opts.create_if_missing = True\n opts.merge_operator = AssocCounter()\n self._clean()\n self.db = rocksdb.DB('/tmp/test', opts)\n\n def tearDown(self):\n self._close_db()\n\n def test_merge(self):\n for x in range(1000):\n self.db.merge(b\"a\", int_to_bytes(x))\n self.assertEqual(sum(range(1000)), int(self.db.get(b'a')))\n\n\nclass FullCounter(rocksdb.interfaces.MergeOperator):\n def name(self):\n return b'fullcounter'\n\n def full_merge(self, key, existing_value, operand_list):\n ret = sum([int(x) for x in operand_list])\n if existing_value:\n ret += int(existing_value)\n\n return (True, int_to_bytes(ret))\n\n def partial_merge(self, key, left, right):\n return (True, int_to_bytes(int(left) + int(right)))\n\n\nclass TestFullMerge(unittest.TestCase, TestHelper):\n def setUp(self):\n opts = rocksdb.Options()\n opts.create_if_missing = True\n opts.merge_operator = FullCounter()\n self._clean()\n self.db = rocksdb.DB('/tmp/test', opts)\n\n def tearDown(self):\n self._close_db()\n\n def test_merge(self):\n for x in range(1000):\n self.db.merge(b\"a\", int_to_bytes(x))\n self.assertEqual(sum(range(1000)), int(self.db.get(b'a')))\n\n\nclass SimpleComparator(rocksdb.interfaces.Comparator):\n def name(self):\n return b'mycompare'\n\n def compare(self, a, b):\n a = int(a)\n b = int(b)\n if a < b:\n return -1\n if a == b:\n return 0\n if a > b:\n return 1\n\n\nclass TestComparator(unittest.TestCase, TestHelper):\n def setUp(self):\n opts = rocksdb.Options()\n opts.create_if_missing = True\n opts.comparator = SimpleComparator()\n self._clean()\n self.db = rocksdb.DB('/tmp/test', opts)\n\n def tearDown(self):\n self._close_db()\n\n def test_compare(self):\n for x in range(1000):\n self.db.put(int_to_bytes(x), int_to_bytes(x))\n\n self.assertEqual(b'300', self.db.get(b'300'))\n\nclass StaticPrefix(rocksdb.interfaces.SliceTransform):\n def name(self):\n return b'static'\n\n def transform(self, src):\n return (0, 5)\n\n def in_domain(self, src):\n return len(src) >= 5\n\n def in_range(self, dst):\n return len(dst) == 5\n\nclass TestPrefixExtractor(unittest.TestCase, TestHelper):\n def setUp(self):\n opts = rocksdb.Options(create_if_missing=True)\n opts.prefix_extractor = StaticPrefix()\n self._clean()\n self.db = rocksdb.DB('/tmp/test', opts)\n\n def tearDown(self):\n self._close_db()\n\n def _fill_db(self):\n for x in range(3000):\n keyx = hex(x)[2:].zfill(5).encode('utf8') + b'.x'\n keyy = hex(x)[2:].zfill(5).encode('utf8') + b'.y'\n keyz = hex(x)[2:].zfill(5).encode('utf8') + b'.z'\n self.db.put(keyx, b'x')\n self.db.put(keyy, b'y')\n self.db.put(keyz, b'z')\n\n\n def test_prefix_iterkeys(self):\n self._fill_db()\n self.assertEqual(b'x', self.db.get(b'00001.x'))\n self.assertEqual(b'y', self.db.get(b'00001.y'))\n self.assertEqual(b'z', self.db.get(b'00001.z'))\n\n it = self.db.iterkeys()\n it.seek(b'00002')\n\n ref = [b'00002.x', b'00002.y', b'00002.z']\n ret = takewhile(lambda key: key.startswith(b'00002'), it)\n self.assertEqual(ref, list(ret))\n\n def test_prefix_iteritems(self):\n self._fill_db()\n\n it = self.db.iteritems()\n it.seek(b'00002')\n\n ref = {b'00002.z': b'z', b'00002.y': b'y', b'00002.x': b'x'}\n ret = takewhile(lambda item: item[0].startswith(b'00002'), it)\n self.assertEqual(ref, dict(ret))\n" }, { "path": "rocksdb/tests/test_options.py", "content": "import unittest\nimport rocksdb\n\nclass TestFilterPolicy(rocksdb.interfaces.FilterPolicy):\n def create_filter(self, keys):\n return b'nix'\n\n def key_may_match(self, key, fil):\n return True\n\n def name(self):\n return b'testfilter'\n\nclass TestMergeOperator(rocksdb.interfaces.MergeOperator):\n def full_merge(self, *args, **kwargs):\n return (False, None)\n\n def partial_merge(self, *args, **kwargs):\n return (False, None)\n\n def name(self):\n return b'testmergeop'\n\nclass TestOptions(unittest.TestCase):\n def test_simple(self):\n opts = rocksdb.Options()\n self.assertEqual(True, opts.paranoid_checks)\n opts.paranoid_checks = False\n self.assertEqual(False, opts.paranoid_checks)\n\n self.assertIsNone(opts.merge_operator)\n ob = TestMergeOperator()\n opts.merge_operator = ob\n self.assertEqual(opts.merge_operator, ob)\n\n self.assertIsInstance(\n opts.comparator,\n rocksdb.BytewiseComparator)\n\n self.assertEqual('snappy_compression', opts.compression)\n opts.compression = rocksdb.CompressionType.no_compression\n self.assertEqual('no_compression', opts.compression)\n\n def test_block_options(self):\n rocksdb.BlockBasedTableFactory(\n block_size=4096,\n filter_policy=TestFilterPolicy(),\n block_cache=rocksdb.LRUCache(100))\n\n def test_unicode_path(self):\n name = b'/tmp/M\\xc3\\xbcnchen'.decode('utf8')\n opts = rocksdb.Options()\n opts.db_log_dir = name\n opts.wal_dir = name\n\n self.assertEqual(name, opts.db_log_dir)\n self.assertEqual(name, opts.wal_dir)\n\n def test_table_factory(self):\n opts = rocksdb.Options()\n self.assertIsNone(opts.table_factory)\n\n opts.table_factory = rocksdb.BlockBasedTableFactory()\n opts.table_factory = rocksdb.PlainTableFactory()\n\n def test_compaction_style(self):\n opts = rocksdb.Options()\n self.assertEqual('level', opts.compaction_style)\n\n opts.compaction_style = 'universal'\n self.assertEqual('universal', opts.compaction_style)\n\n opts.compaction_style = 'level'\n self.assertEqual('level', opts.compaction_style)\n\n with self.assertRaisesRegexp(Exception, 'Unknown compaction style'):\n opts.compaction_style = 'foo'\n\n def test_compaction_opts_universal(self):\n opts = rocksdb.Options()\n uopts = opts.compaction_options_universal\n self.assertEqual(-1, uopts['compression_size_percent'])\n self.assertEqual(200, uopts['max_size_amplification_percent'])\n self.assertEqual('total_size', uopts['stop_style'])\n self.assertEqual(1, uopts['size_ratio'])\n self.assertEqual(2, uopts['min_merge_width'])\n self.assertGreaterEqual(4294967295, uopts['max_merge_width'])\n\n new_opts = {'stop_style': 'similar_size', 'max_merge_width': 30}\n opts.compaction_options_universal = new_opts\n uopts = opts.compaction_options_universal\n\n self.assertEqual(-1, uopts['compression_size_percent'])\n self.assertEqual(200, uopts['max_size_amplification_percent'])\n self.assertEqual('similar_size', uopts['stop_style'])\n self.assertEqual(1, uopts['size_ratio'])\n self.assertEqual(2, uopts['min_merge_width'])\n self.assertEqual(30, uopts['max_merge_width'])\n\n def test_row_cache(self):\n opts = rocksdb.Options()\n self.assertIsNone(opts.row_cache)\n opts.row_cache = cache = rocksdb.LRUCache(2*1024*1024)\n self.assertEqual(cache, opts.row_cache)\n" }, { "path": "rocksdb/universal_compaction.pxd", "content": "cdef extern from \"rocksdb/universal_compaction.h\" namespace \"rocksdb\":\n\n ctypedef enum CompactionStopStyle:\n kCompactionStopStyleSimilarSize\n kCompactionStopStyleTotalSize\n\n cdef cppclass CompactionOptionsUniversal:\n CompactionOptionsUniversal()\n\n unsigned int size_ratio\n unsigned int min_merge_width\n unsigned int max_merge_width\n unsigned int max_size_amplification_percent\n int compression_size_percent\n CompactionStopStyle stop_style\n" }, { "path": "setup.py", "content": "from setuptools import setup\nfrom setuptools import find_packages\nfrom distutils.extension import Extension\n\ntry:\n from Cython.Build import cythonize\nexcept ImportError:\n def cythonize(extensions): return extensions\n sources = ['rocksdb/_rocksdb.cpp']\nelse:\n sources = ['rocksdb/_rocksdb.pyx']\n\nmod1 = Extension(\n 'rocksdb._rocksdb',\n sources,\n extra_compile_args=[\n '-std=c++11',\n '-O3',\n '-Wall',\n '-Wextra',\n '-Wconversion',\n '-fno-strict-aliasing'\n ],\n language='c++',\n libraries=[\n 'rocksdb',\n 'snappy',\n 'bz2',\n 'z'\n ]\n)\n\nsetup(\n name=\"pyrocksdb\",\n version='0.5',\n description=\"Python bindings for RocksDB\",\n keywords='rocksdb',\n author='Stephan Hofmockel',\n author_email=\"Use the github issues\",\n url=\"https://github.com/stephan-hof/pyrocksdb\",\n license='BSD License',\n install_requires=['setuptools'],\n package_dir={'rocksdb': 'rocksdb'},\n packages=find_packages('.'),\n ext_modules=cythonize([mod1]),\n test_suite='rocksdb.tests',\n include_package_data=True\n)\n" } ]