Showing preview only (5,290K chars total). Download the full file or copy to clipboard to get everything.
Repository: google/riegeli
Branch: master
Commit: a0a8dac780d1
Files: 498
Total size: 5.0 MB
Directory structure:
gitextract_1atzokxc/
├── .bazelrc
├── CONTRIBUTING.md
├── LICENSE
├── MANIFEST.in
├── MODULE.bazel
├── README.md
├── configure
├── doc/
│ ├── index.md
│ ├── record_writer_options.md
│ └── riegeli_records_file_format.md
├── python/
│ ├── BUILD
│ ├── MANIFEST.in
│ ├── README.md
│ ├── __init__.py
│ ├── build_pip_package.sh
│ ├── dummy_binary.py
│ ├── riegeli/
│ │ ├── BUILD
│ │ ├── BUILD.tpl
│ │ ├── __init__.py
│ │ ├── base/
│ │ │ ├── BUILD
│ │ │ ├── __init__.py
│ │ │ ├── riegeli_error.py
│ │ │ ├── utils.cc
│ │ │ └── utils.h
│ │ ├── bytes/
│ │ │ ├── BUILD
│ │ │ ├── python_reader.cc
│ │ │ ├── python_reader.h
│ │ │ ├── python_writer.cc
│ │ │ └── python_writer.h
│ │ ├── py_extension.bzl
│ │ ├── python_configure.bzl
│ │ ├── records/
│ │ │ ├── BUILD
│ │ │ ├── __init__.py
│ │ │ ├── examples/
│ │ │ │ ├── BUILD
│ │ │ │ ├── __init__.py
│ │ │ │ └── write_read_records.py
│ │ │ ├── record_position.cc
│ │ │ ├── record_position.h
│ │ │ ├── record_reader.cc
│ │ │ ├── record_writer.cc
│ │ │ ├── records_metadata.proto
│ │ │ ├── skipped_region.py
│ │ │ └── tests/
│ │ │ ├── BUILD
│ │ │ ├── __init__.py
│ │ │ ├── records_test.proto
│ │ │ └── records_test.py
│ │ └── tensorflow/
│ │ ├── BUILD
│ │ ├── __init__.py
│ │ ├── kernel_tests/
│ │ │ ├── __init__.py
│ │ │ └── riegeli_dataset_test.py
│ │ └── ops/
│ │ ├── __init__.py
│ │ └── riegeli_dataset_ops.py
│ └── setup.py
├── riegeli/
│ ├── .gitignore
│ ├── BUILD
│ ├── base/
│ │ ├── BUILD
│ │ ├── any.h
│ │ ├── any_initializer.h
│ │ ├── any_internal.h
│ │ ├── arithmetic.h
│ │ ├── assert.cc
│ │ ├── assert.h
│ │ ├── background_cleaning.cc
│ │ ├── background_cleaning.h
│ │ ├── binary_search.h
│ │ ├── buffer.cc
│ │ ├── buffer.h
│ │ ├── buffering.h
│ │ ├── byte_fill.cc
│ │ ├── byte_fill.h
│ │ ├── bytes_ref.h
│ │ ├── c_string_ref.h
│ │ ├── chain.cc
│ │ ├── chain.h
│ │ ├── chain_base.h
│ │ ├── chain_details.h
│ │ ├── closing_ptr.h
│ │ ├── compact_string.cc
│ │ ├── compact_string.h
│ │ ├── compare.h
│ │ ├── constexpr.h
│ │ ├── cord_iterator_span.cc
│ │ ├── cord_iterator_span.h
│ │ ├── cord_utils.cc
│ │ ├── cord_utils.h
│ │ ├── debug.cc
│ │ ├── debug.h
│ │ ├── dependency.h
│ │ ├── dependency_base.h
│ │ ├── dependency_manager.h
│ │ ├── errno_mapping.cc
│ │ ├── errno_mapping.h
│ │ ├── estimated_allocated_size.h
│ │ ├── external_data.cc
│ │ ├── external_data.h
│ │ ├── external_ref.h
│ │ ├── external_ref_base.h
│ │ ├── external_ref_support.h
│ │ ├── global.h
│ │ ├── hybrid_direct_common.h
│ │ ├── hybrid_direct_internal.h
│ │ ├── hybrid_direct_map.h
│ │ ├── hybrid_direct_set.h
│ │ ├── initializer.h
│ │ ├── initializer_internal.h
│ │ ├── intrusive_shared_ptr.h
│ │ ├── invoker.h
│ │ ├── iterable.h
│ │ ├── maker.h
│ │ ├── memory_estimator.cc
│ │ ├── memory_estimator.h
│ │ ├── moving_dependency.h
│ │ ├── new_aligned.h
│ │ ├── null_safe_memcpy.h
│ │ ├── object.cc
│ │ ├── object.h
│ │ ├── optional_compact_string.h
│ │ ├── options_parser.cc
│ │ ├── options_parser.h
│ │ ├── ownership.h
│ │ ├── parallelism.cc
│ │ ├── parallelism.h
│ │ ├── port.h
│ │ ├── recycling_pool.h
│ │ ├── ref_count.h
│ │ ├── reset.h
│ │ ├── shared_buffer.cc
│ │ ├── shared_buffer.h
│ │ ├── shared_ptr.h
│ │ ├── sized_shared_buffer.cc
│ │ ├── sized_shared_buffer.h
│ │ ├── stable_dependency.h
│ │ ├── status.cc
│ │ ├── status.h
│ │ ├── stream_utils.cc
│ │ ├── stream_utils.h
│ │ ├── string_ref.h
│ │ ├── string_utils.cc
│ │ ├── string_utils.h
│ │ ├── temporary_storage.h
│ │ ├── type_erased_ref.h
│ │ ├── type_id.h
│ │ ├── type_traits.h
│ │ ├── types.h
│ │ ├── unicode.cc
│ │ ├── unicode.h
│ │ └── uninitialized_vector.h
│ ├── brotli/
│ │ ├── BUILD
│ │ ├── brotli_allocator.cc
│ │ ├── brotli_allocator.h
│ │ ├── brotli_dictionary.cc
│ │ ├── brotli_dictionary.h
│ │ ├── brotli_reader.cc
│ │ ├── brotli_reader.h
│ │ ├── brotli_writer.cc
│ │ └── brotli_writer.h
│ ├── bytes/
│ │ ├── BUILD
│ │ ├── array_backward_writer.cc
│ │ ├── array_backward_writer.h
│ │ ├── array_writer.cc
│ │ ├── array_writer.h
│ │ ├── backward_writer.cc
│ │ ├── backward_writer.h
│ │ ├── buffer_options.cc
│ │ ├── buffer_options.h
│ │ ├── buffered_reader.cc
│ │ ├── buffered_reader.h
│ │ ├── buffered_writer.cc
│ │ ├── buffered_writer.h
│ │ ├── cfile_handle.cc
│ │ ├── cfile_handle.h
│ │ ├── cfile_internal.cc
│ │ ├── cfile_internal.h
│ │ ├── cfile_internal_for_cc.h
│ │ ├── cfile_reader.cc
│ │ ├── cfile_reader.h
│ │ ├── cfile_writer.cc
│ │ ├── cfile_writer.h
│ │ ├── chain_backward_writer.cc
│ │ ├── chain_backward_writer.h
│ │ ├── chain_reader.cc
│ │ ├── chain_reader.h
│ │ ├── chain_writer.cc
│ │ ├── chain_writer.h
│ │ ├── compact_string_writer.h
│ │ ├── copy_all.cc
│ │ ├── copy_all.h
│ │ ├── cord_backward_writer.cc
│ │ ├── cord_backward_writer.h
│ │ ├── cord_reader.cc
│ │ ├── cord_reader.h
│ │ ├── cord_writer.cc
│ │ ├── cord_writer.h
│ │ ├── fd_handle.cc
│ │ ├── fd_handle.h
│ │ ├── fd_internal.cc
│ │ ├── fd_internal.h
│ │ ├── fd_internal_for_cc.h
│ │ ├── fd_mmap_reader.cc
│ │ ├── fd_mmap_reader.h
│ │ ├── fd_reader.cc
│ │ ├── fd_reader.h
│ │ ├── fd_writer.cc
│ │ ├── fd_writer.h
│ │ ├── file_mode_string.cc
│ │ ├── file_mode_string.h
│ │ ├── iostream_internal.h
│ │ ├── istream_reader.cc
│ │ ├── istream_reader.h
│ │ ├── joining_reader.cc
│ │ ├── joining_reader.h
│ │ ├── limiting_backward_writer.cc
│ │ ├── limiting_backward_writer.h
│ │ ├── limiting_reader.cc
│ │ ├── limiting_reader.h
│ │ ├── limiting_writer.cc
│ │ ├── limiting_writer.h
│ │ ├── null_backward_writer.cc
│ │ ├── null_backward_writer.h
│ │ ├── null_writer.cc
│ │ ├── null_writer.h
│ │ ├── ostream_writer.cc
│ │ ├── ostream_writer.h
│ │ ├── path_ref.h
│ │ ├── position_shifting_backward_writer.cc
│ │ ├── position_shifting_backward_writer.h
│ │ ├── position_shifting_reader.cc
│ │ ├── position_shifting_reader.h
│ │ ├── position_shifting_writer.cc
│ │ ├── position_shifting_writer.h
│ │ ├── prefix_limiting_backward_writer.cc
│ │ ├── prefix_limiting_backward_writer.h
│ │ ├── prefix_limiting_reader.cc
│ │ ├── prefix_limiting_reader.h
│ │ ├── prefix_limiting_writer.cc
│ │ ├── prefix_limiting_writer.h
│ │ ├── pullable_reader.cc
│ │ ├── pullable_reader.h
│ │ ├── pushable_backward_writer.cc
│ │ ├── pushable_backward_writer.h
│ │ ├── pushable_writer.cc
│ │ ├── pushable_writer.h
│ │ ├── read_all.cc
│ │ ├── read_all.h
│ │ ├── reader.cc
│ │ ├── reader.h
│ │ ├── reader_cfile.cc
│ │ ├── reader_cfile.h
│ │ ├── reader_factory.cc
│ │ ├── reader_factory.h
│ │ ├── reader_istream.cc
│ │ ├── reader_istream.h
│ │ ├── resizable_writer.cc
│ │ ├── resizable_writer.h
│ │ ├── restricted_chain_writer.cc
│ │ ├── restricted_chain_writer.h
│ │ ├── splitting_writer.cc
│ │ ├── splitting_writer.h
│ │ ├── std_io.cc
│ │ ├── std_io.h
│ │ ├── string_reader.cc
│ │ ├── string_reader.h
│ │ ├── string_writer.cc
│ │ ├── string_writer.h
│ │ ├── stringify.h
│ │ ├── stringify_writer.h
│ │ ├── vector_writer.h
│ │ ├── wrapping_backward_writer.cc
│ │ ├── wrapping_backward_writer.h
│ │ ├── wrapping_reader.cc
│ │ ├── wrapping_reader.h
│ │ ├── wrapping_writer.cc
│ │ ├── wrapping_writer.h
│ │ ├── write.h
│ │ ├── write_int_internal.cc
│ │ ├── write_int_internal.h
│ │ ├── writer.cc
│ │ ├── writer.h
│ │ ├── writer_cfile.cc
│ │ ├── writer_cfile.h
│ │ ├── writer_ostream.cc
│ │ └── writer_ostream.h
│ ├── bzip2/
│ │ ├── BUILD
│ │ ├── bzip2_error.cc
│ │ ├── bzip2_error.h
│ │ ├── bzip2_reader.cc
│ │ ├── bzip2_reader.h
│ │ ├── bzip2_writer.cc
│ │ └── bzip2_writer.h
│ ├── chunk_encoding/
│ │ ├── BUILD
│ │ ├── README.md
│ │ ├── brotli_encoder_selection.cc
│ │ ├── brotli_encoder_selection.h
│ │ ├── chunk.cc
│ │ ├── chunk.h
│ │ ├── chunk_decoder.cc
│ │ ├── chunk_decoder.h
│ │ ├── chunk_encoder.cc
│ │ ├── chunk_encoder.h
│ │ ├── compressor.cc
│ │ ├── compressor.h
│ │ ├── compressor_options.cc
│ │ ├── compressor_options.h
│ │ ├── constants.h
│ │ ├── decompressor.cc
│ │ ├── decompressor.h
│ │ ├── deferred_encoder.cc
│ │ ├── deferred_encoder.h
│ │ ├── field_projection.h
│ │ ├── hash.cc
│ │ ├── hash.h
│ │ ├── simple_decoder.cc
│ │ ├── simple_decoder.h
│ │ ├── simple_encoder.cc
│ │ ├── simple_encoder.h
│ │ ├── transpose_decoder.cc
│ │ ├── transpose_decoder.h
│ │ ├── transpose_encoder.cc
│ │ ├── transpose_encoder.h
│ │ └── transpose_internal.h
│ ├── containers/
│ │ ├── BUILD
│ │ ├── chunked_sorted_string_set.cc
│ │ ├── chunked_sorted_string_set.h
│ │ ├── linear_sorted_string_set.cc
│ │ └── linear_sorted_string_set.h
│ ├── csv/
│ │ ├── BUILD
│ │ ├── csv_reader.cc
│ │ ├── csv_reader.h
│ │ ├── csv_record.cc
│ │ ├── csv_record.h
│ │ ├── csv_writer.cc
│ │ └── csv_writer.h
│ ├── digests/
│ │ ├── BUILD
│ │ ├── adler32_digester.cc
│ │ ├── adler32_digester.h
│ │ ├── crc32_digester.cc
│ │ ├── crc32_digester.h
│ │ ├── crc32c_digester.h
│ │ ├── digest_converter.h
│ │ ├── digester_handle.cc
│ │ ├── digester_handle.h
│ │ ├── digesting_reader.cc
│ │ ├── digesting_reader.h
│ │ ├── digesting_writer.cc
│ │ ├── digesting_writer.h
│ │ ├── highwayhash_digester.cc
│ │ ├── highwayhash_digester.h
│ │ ├── md5_digester.h
│ │ ├── openssl_digester.h
│ │ ├── sha1_digester.h
│ │ ├── sha256_digester.h
│ │ ├── sha512_256_digester.h
│ │ ├── sha512_digester.h
│ │ └── wrapping_digester.h
│ ├── endian/
│ │ ├── BUILD
│ │ ├── endian_reading.h
│ │ └── endian_writing.h
│ ├── gcs/
│ │ ├── BUILD
│ │ ├── gcs_internal.h
│ │ ├── gcs_object.cc
│ │ ├── gcs_object.h
│ │ ├── gcs_reader.cc
│ │ ├── gcs_reader.h
│ │ ├── gcs_writer.cc
│ │ └── gcs_writer.h
│ ├── lines/
│ │ ├── BUILD
│ │ ├── line_reading.cc
│ │ ├── line_reading.h
│ │ ├── line_writing.h
│ │ ├── newline.h
│ │ ├── text_reader.cc
│ │ ├── text_reader.h
│ │ ├── text_writer.cc
│ │ └── text_writer.h
│ ├── lz4/
│ │ ├── BUILD
│ │ ├── lz4_dictionary.cc
│ │ ├── lz4_dictionary.h
│ │ ├── lz4_reader.cc
│ │ ├── lz4_reader.h
│ │ ├── lz4_writer.cc
│ │ └── lz4_writer.h
│ ├── messages/
│ │ ├── BUILD
│ │ ├── context_projection.h
│ │ ├── dynamic_field_handler.h
│ │ ├── field_copier.h
│ │ ├── field_handler_map.h
│ │ ├── field_handlers.cc
│ │ ├── field_handlers.h
│ │ ├── map_entry_field.h
│ │ ├── message_wire_format.h
│ │ ├── parse_message.cc
│ │ ├── parse_message.h
│ │ ├── serialize_message.cc
│ │ ├── serialize_message.h
│ │ ├── serialized_message_assembler.cc
│ │ ├── serialized_message_assembler.h
│ │ ├── serialized_message_backward_writer.cc
│ │ ├── serialized_message_backward_writer.h
│ │ ├── serialized_message_internal.h
│ │ ├── serialized_message_reader.cc
│ │ ├── serialized_message_reader.h
│ │ ├── serialized_message_reader_internal.h
│ │ ├── serialized_message_writer.cc
│ │ ├── serialized_message_writer.h
│ │ ├── text_parse_message.cc
│ │ ├── text_parse_message.h
│ │ ├── text_print_message.cc
│ │ └── text_print_message.h
│ ├── ordered_varint/
│ │ ├── BUILD
│ │ ├── ordered_varint_internal.h
│ │ ├── ordered_varint_reading.cc
│ │ ├── ordered_varint_reading.h
│ │ ├── ordered_varint_writing.cc
│ │ └── ordered_varint_writing.h
│ ├── records/
│ │ ├── BUILD
│ │ ├── README.md
│ │ ├── block.h
│ │ ├── chunk_reader.cc
│ │ ├── chunk_reader.h
│ │ ├── chunk_writer.cc
│ │ ├── chunk_writer.h
│ │ ├── record_position.cc
│ │ ├── record_position.h
│ │ ├── record_reader.cc
│ │ ├── record_reader.h
│ │ ├── record_writer.cc
│ │ ├── record_writer.h
│ │ ├── records_metadata.proto
│ │ ├── skipped_region.cc
│ │ ├── skipped_region.h
│ │ └── tools/
│ │ ├── BUILD
│ │ ├── describe_riegeli_file.cc
│ │ ├── records_benchmark.cc
│ │ ├── riegeli_summary.proto
│ │ ├── tfrecord_recognizer.cc
│ │ └── tfrecord_recognizer.h
│ ├── snappy/
│ │ ├── BUILD
│ │ ├── framed/
│ │ │ ├── BUILD
│ │ │ ├── framed_snappy_reader.cc
│ │ │ ├── framed_snappy_reader.h
│ │ │ ├── framed_snappy_writer.cc
│ │ │ └── framed_snappy_writer.h
│ │ ├── hadoop/
│ │ │ ├── BUILD
│ │ │ ├── hadoop_snappy_reader.cc
│ │ │ ├── hadoop_snappy_reader.h
│ │ │ ├── hadoop_snappy_writer.cc
│ │ │ └── hadoop_snappy_writer.h
│ │ ├── snappy_reader.cc
│ │ ├── snappy_reader.h
│ │ ├── snappy_streams.cc
│ │ ├── snappy_streams.h
│ │ ├── snappy_writer.cc
│ │ └── snappy_writer.h
│ ├── tensorflow/
│ │ ├── BUILD
│ │ ├── io/
│ │ │ ├── BUILD
│ │ │ ├── file_reader.cc
│ │ │ ├── file_reader.h
│ │ │ ├── file_writer.cc
│ │ │ ├── file_writer.h
│ │ │ └── tstring_writer.h
│ │ ├── kernels/
│ │ │ └── riegeli_dataset_ops.cc
│ │ └── ops/
│ │ └── riegeli_dataset_ops.cc
│ ├── text/
│ │ ├── BUILD
│ │ ├── ascii_align.h
│ │ ├── concat.h
│ │ ├── join.h
│ │ ├── write_int.cc
│ │ └── write_int.h
│ ├── varint/
│ │ ├── BUILD
│ │ ├── varint_internal.h
│ │ ├── varint_reading.cc
│ │ ├── varint_reading.h
│ │ └── varint_writing.h
│ ├── xz/
│ │ ├── BUILD
│ │ ├── xz_error.cc
│ │ ├── xz_error.h
│ │ ├── xz_reader.cc
│ │ ├── xz_reader.h
│ │ ├── xz_writer.cc
│ │ └── xz_writer.h
│ ├── zlib/
│ │ ├── BUILD
│ │ ├── zlib_dictionary.h
│ │ ├── zlib_error.cc
│ │ ├── zlib_error.h
│ │ ├── zlib_reader.cc
│ │ ├── zlib_reader.h
│ │ ├── zlib_writer.cc
│ │ └── zlib_writer.h
│ └── zstd/
│ ├── BUILD
│ ├── zstd_dictionary.cc
│ ├── zstd_dictionary.h
│ ├── zstd_reader.cc
│ ├── zstd_reader.h
│ ├── zstd_writer.cc
│ └── zstd_writer.h
└── tf_dependency/
├── BUILD
├── BUILD.tpl
└── tf_configure.bzl
================================================
FILE CONTENTS
================================================
================================================
FILE: .bazelrc
================================================
# Enable Bzlmod by default.
common --enable_bzlmod
# Use C++17.
build --cxxopt=-std=c++17
build --host_cxxopt=-std=c++17
# Make Python protos faster by backing them with C++ protos.
# TODO: Reenable once protobuf releases
# https://github.com/protocolbuffers/protobuf/pull/22633
# i.e. in version > 32.0. Or possibly switch to upb.
# build --define=use_fast_cpp_protos=true
# Options from ./configure
# This is currently disabled because TensorFlow does not support bzlmod,
# hence Riegeli/TensorFlow bindings are broken anyway.
# import %workspace%/configure.bazelrc
================================================
FILE: CONTRIBUTING.md
================================================
# How to Contribute
We'd love to accept your patches and contributions to this project. There are
just a few small guidelines you need to follow.
## Contributor License Agreement
Contributions to this project must be accompanied by a Contributor License
Agreement. You (or your employer) retain the copyright to your contribution,
this simply gives us permission to use and redistribute your contributions as
part of the project. Head over to <https://cla.developers.google.com/> to see
your current agreements on file or to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted one
(even if it was for a different project), you probably don't need to do it
again.
## Code reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult
[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
information on using pull requests.
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: MANIFEST.in
================================================
include .bazelrc
include *.md
include LICENSE
include MANIFEST.in
include WORKSPACE
include configure
recursive-include doc *
recursive-include python *
recursive-include riegeli *
recursive-include third_party *
================================================
FILE: MODULE.bazel
================================================
module(
name = "riegeli",
repo_name = "com_google_riegeli",
)
bazel_dep(
name = "abseil-cpp",
version = "20260107.0",
repo_name = "com_google_absl",
)
bazel_dep(
name = "abseil-py",
version = "2.1.0",
repo_name = "absl_py",
)
bazel_dep(
name = "bazel_skylib",
version = "1.7.1",
)
bazel_dep(
name = "boringssl",
version = "0.0.0-20240530-2db0eb3",
)
bazel_dep(
name = "brotli",
version = "1.1.0",
repo_name = "org_brotli",
)
bazel_dep(
name = "bzip2",
version = "1.0.8",
)
bazel_dep(
name = "highwayhash",
version = "0.0.0-20240305-5ad3bf8.bcr.1",
)
bazel_dep(
name = "lz4",
version = "1.9.4",
)
bazel_dep(
name = "platforms",
version = "0.0.9",
)
bazel_dep(
name = "protobuf",
version = "33.2",
repo_name = "com_google_protobuf",
)
bazel_dep(
name = "rules_cc",
version = "0.1.2",
)
bazel_dep(
name = "rules_python",
version = "0.36.0",
)
bazel_dep(
name = "snappy",
version = "1.2.0",
)
bazel_dep(
name = "xz",
version = "5.4.5.bcr.1",
)
bazel_dep(
name = "zlib",
version = "1.3.1.bcr.3",
)
bazel_dep(
name = "zstd",
version = "1.5.6",
repo_name = "net_zstd",
)
bazel_dep(
name = "google_cloud_cpp",
version = "3.0.0-rc1",
)
# Configure hermetic Python toolchain
SUPPORTED_PYTHON_VERSIONS = [
"3.8",
"3.9",
"3.10",
"3.11",
"3.12",
]
DEFAULT_PYTHON_VERSION = SUPPORTED_PYTHON_VERSIONS[-1]
python = use_extension("@rules_python//python/extensions:python.bzl", "python")
[
python.toolchain(
is_default = version == DEFAULT_PYTHON_VERSION,
python_version = version,
)
for version in SUPPORTED_PYTHON_VERSIONS
]
================================================
FILE: README.md
================================================
# Riegeli
*Riegeli/records* is a file format for storing a sequence of string records,
typically serialized protocol buffers. It supports dense compression, fast
decoding, seeking, detection and optional skipping of data corruption, filtering
of proto message fields for even faster decoding, and parallel encoding.
See [documentation](https://github.com/google/riegeli/blob/master/doc/index.md).
# Status
Riegeli file format will only change in a backward compatible way (i.e. future
readers will understand current files, but current readers might not understand
files using future features).
Riegeli C++ API might change in incompatible ways.
================================================
FILE: configure
================================================
#!/bin/bash
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
PYTHON_BIN_PATH=`which python`
if [[ $PYTHON_BIN_PATH ]] && $PYTHON_BIN_PATH -c "import tensorflow" &>/dev/null; then
TF_CFLAGS=$($PYTHON_BIN_PATH -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))')
TF_LFLAGS=$($PYTHON_BIN_PATH -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))')
TF_HEADER_DIR=${TF_CFLAGS%% *}
TF_HEADER_DIR=${TF_HEADER_DIR#-I}
TF_SHARED_LIBRARY_DIR=${TF_LFLAGS%% *}
TF_SHARED_LIBRARY_DIR=${TF_SHARED_LIBRARY_DIR#-L}
TF_SHARED_LIBRARY_NAME=${TF_LFLAGS##* -l:}
else
TF_HEADER_DIR=
TF_SHARED_LIBRARY_DIR=
TF_SHARED_LIBRARY_NAME=
fi
{
printf 'build --action_env PYTHON_BIN_PATH="%s"\n' "$PYTHON_BIN_PATH"
printf 'build --action_env TF_HEADER_DIR="%s"\n' "$TF_HEADER_DIR"
printf 'build --action_env TF_SHARED_LIBRARY_DIR="%s"\n' "$TF_SHARED_LIBRARY_DIR"
printf 'build --action_env TF_SHARED_LIBRARY_NAME="%s"\n' "$TF_SHARED_LIBRARY_NAME"
} >configure.bazelrc
echo "Set up configure.bazelrc. Make sure to include it in your .bazelrc file."
================================================
FILE: doc/index.md
================================================
# Riegeli
*Riegeli/records* is a file format for storing a sequence of string records,
typically serialized protocol buffers. It supports dense compression, fast
decoding, seeking, detection and optional skipping of data corruption, filtering
of proto message fields for even faster decoding, and parallel encoding.
* [Specification of Riegeli/records file format](riegeli_records_file_format.md).
* [Specifying options for writing Riegeli/records files](record_writer_options.md).
================================================
FILE: doc/record_writer_options.md
================================================
# Specifying options for writing Riegeli/records files
Options for writing Riegeli/records files can be specified as a string:
```data
options ::= option? ("," option?)*
option ::=
"default" |
"transpose" (":" ("true" | "false"))? |
"uncompressed" |
"brotli" (":" brotli_level)? |
"zstd" (":" zstd_level)? |
"snappy" (":" snappy_level)? |
"window_log" ":" window_log |
"brotli_encoder" ":" ("rbrotli_or_cbrotli" | "cbrotli" | "rbrotli") |
"chunk_size" ":" chunk_size |
"bucket_fraction" ":" bucket_fraction |
"padding" (":" padding)? |
"initial_padding" (":" padding)? |
"final_padding" (":" padding)? |
"parallelism" ":" parallelism
brotli_level ::= integer in the range [0..11] (default 6)
zstd_level ::= integer in the range [-131072..22] (default 3)
snappy_level ::= integer in the range [1..2] (default 1)
window_log ::= "auto" or integer in the range [10..31]
chunk_size ::= "auto" or positive integer expressed as real with optional
suffix [BkKMGTPE]
bucket_fraction ::= real in the range [0..1]
padding ::= positive integer expressed as real with optional suffix [BkKMGTPE]
(default 64K)
parallelism ::= non-negative integer
```
An empty string is the same as `default`.
## `transpose`
If `true` (`transpose` is the same as `transpose:true`), records should be
serialized proto messages (but nothing will break if they are not). A chunk of
records will be processed in a way which allows for better compression.
If `false`, a chunk of records will be stored in a simpler format, directly or
with compression.
Default: `false`.
## Compression algorithms
### `uncompressed`
Changes compression algorithm to Uncompressed (turns compression off).
### `brotli`
Changes compression algorithm to [Brotli](https://github.com/google/brotli).
Sets compression level which tunes the tradeoff between compression density and
compression speed (higher = better density but slower).
`brotli_level` must be between 0 and 11. Default: `6`.
This is the default compression algorithm.
### `zstd`
Changes compression algorithm to [Zstd](https://facebook.github.io/zstd/). Sets
compression level which tunes the tradeoff between compression density and
compression speed (higher = better density but slower).
`zstd_level` must be between -131072 and 22. Level 0 is currently equivalent to
3. Default: 3.
### `snappy`
Changes compression algorithm to [Snappy](https://google.github.io/snappy/).
`snappy_level` must be between 1 and 2. Default: 1.
## `window_log`
Logarithm of the LZ77 sliding window size. This tunes the tradeoff between
compression density and memory usage (higher = better density but more memory).
Special value `auto` means to keep the default (`brotli`: 22, `zstd`: derived
from compression level and chunk size).
For `uncompressed` and `snappy`, `window_log` must be `auto`. For `brotli`,
`window_log` must be `auto` or between 10 and 30. For `zstd`, `window_log` must
be `auto` or between 10 and 30 in 32-bit build, 31 in 64-bit build.
Default: `auto`.
## `chunk_size`
Sets the desired uncompressed size of a chunk which groups messages to be
transposed, compressed, and written together.
A larger chunk size improves compression density; a smaller chunk size allows to
read pieces of the file independently with finer granularity, and reduces memory
usage of both writer and reader.
Special value `auto` means to keep the default (compressed: 1M, uncompressed:
4k).
Default: `auto`.
## `bucket_fraction`
Sets the desired uncompressed size of a bucket which groups values of several
fields of the given wire type to be compressed together, relative to the desired
chunk size, on the scale between 0.0 (compress each field separately) to 1.0
(put all fields of the same wire type in the same bucket.
This is meaningful if transpose and compression are enabled. A larger bucket
size improves compression density; a smaller bucket size makes reading with
projection faster, allowing to skip decompression of values of fields which are
not included.
Default 1.0.
## `padding`
If `padding > 1`, padding is written at the beginning, when flushing, and at the
end of the file, for the absolute position to reach a multiple of `padding`.
Consequences if `padding` is a multiple of 64KB:
1. Physical concatenation of separately written files yields a valid file
(setting metadata in subsequent files is wasteful but harmless).
2. Even if the existing file was corrupted or truncated, data appended to it
will be recoverable.
The cost is that up to `padding` bytes is wasted when padding is written.
`padding` is a shortcut for `set_initial_padding` with `set_final_padding`.
`padding` without the parameter assumes 64KB.
Default: 1 (no padding).
## `initial_padding`
If `initial_padding > 1`, padding is written at the beginning of the file, for
the absolute position to reach a multiple of `initial_padding`.
See `padding` for details.
`initial_padding` without the parameter assumes 64KB.
Default: 1 (no padding).
## `final_padding`
If `final_padding > 1`, padding is written when flushing and at the end of the
file, for the absolute position to reach a multiple of `final_padding`.
See `padding` for details.
`final_padding` without the parameter assumes 64KB.
Default: 1 (no padding).
## `parallelism`
Sets the maximum number of chunks being encoded in parallel in background.
Larger parallelism can increase throughput, up to a point where it no longer
matters; smaller parallelism reduces memory usage.
If `parallelism > 0`, chunks are written in background and reporting writing
errors is delayed.
Default: 0.
================================================
FILE: doc/riegeli_records_file_format.md
================================================
# Riegeli/records file format specification
## Summary
File contents are interpreted as a sequence of variable-sized *chunks,* where a
chunk encodes some number of *records.* A record can be any byte sequence but
Riegeli has special support for the common case where it is a serialized proto
message.
In order to support seeking and recovery after data corruption, the sequence of
chunks is interrupted by a *block header* at every multiple of the block size
which is 64 KiB. After the block header the interrupted chunk continues.
A record can be identified by the position of the chunk beginning and the index
of the record within the chunk. A record can also be identified by a number
resembling a file position, defined as the sum of the chunk beginning and the
record index.
## Conventions
Numbers in block headers and chunk headers are encoded as unsigned Little-Endian
integers.
Hashes are 64-bit [HighwayHash](https://github.com/google/highwayhash) values
with the key {0x2f696c6567656952, 0x0a7364726f636572, 0x2f696c6567656952,
0x0a7364726f636572} ('Riegeli/', 'records\n', 'Riegeli/', 'records\n').
## Block header
A block header allows to locate the chunk that the block header interrupts.
Block headers can interrupt a chunk at arbitrary points, including in the middle
of the chunk header.
If a block header lies exactly between chunks, it is considered to interrupt the
next chunk; this includes the situation at the beginning of the file. In this
case the chunk formally begins at the beginning of the block, even though it
contains no bytes before the block header.
* Block header (24 bytes):
* `header_hash` (8 bytes) — hash of the rest of the header
(`previous_chunk` and `next_chunk`)
* `previous_chunk` (8 bytes) — distance from the beginning of the chunk
interrupted by this block header to the beginning of the block
* `next_chunk` (8 bytes) — distance from the beginning of the block to the
end of the chunk interrupted by this block header
If `header_hash` does not match, then this block header is corrupted and must be
ignored. Block headers can be skipped during sequential file reading, they are
useful only for seeking and for error recovery.
## Chunk
A chunk must not begin inside nor immediately after a block header.
* Chunk header (40 bytes):
* `header_hash` (8 bytes) — hash of the rest of the header (`data_size` up
to and including `decoded_data_size`)
* `data_size` (8 bytes) — size of `data` (excluding intervening block
headers)
* `data_hash` (8 bytes) — hash of `data`
* `chunk_type` (1 byte) — determines how to interpret `data`
* `num_records` (7 bytes) — number of records after decoding
* `decoded_data_size` (8 bytes) — sum of record sizes after decoding
* `data` (`data_size` bytes) — encoded records or other data
* `padding` — ignored (usually filled with zeros by the encoder)
If `header_hash` does not match, header contents cannot be trusted; if skipping
over corruption is desired, a valid chunk should be located using block headers.
If `data_hash` does not match, `data` is corrupted; if skipping over corruption
is desired, the chunk must be ignored.
The size of `padding` is the minimum size which satisfies the following
constraints:
* The chunk (including chunk header, `data`, `padding`, and intervening block
headers) has at least as many bytes as `num_records`.
* The chunk does not end inside nor immediately after a block header.
If `num_records` is 0, `decoded_data_size` has a meaning depending on the chunk
type.
*Rationale:*
*The presence of `padding` allows to assign unique numbers resembling file
positions to records.*
*`decoded_data_size` is stored in the chunk header, instead of being implied by
or stored in `data`, to help decoders decide how many chunks to potentially read
ahead.*
## Chunk data
Some parts of chunk data are compressed. The compression format is generally
specified as `compression_type` (byte):
* 0 — none
* 0x62 ('b') — [Brotli](https://github.com/google/brotli)
* 0x7a ('z') — [Zstd](https://facebook.github.io/zstd/)
* 0x73 ('s') — [Snappy](https://google.github.io/snappy/)
Any compressed block is prefixed with its decompressed size (varint64) unless
`compression_type` is 0.
*Rationale:*
*Knowing the decompressed size can make easier for the decoder to decompress
data into a preallocated array.*
### File signature
`chunk_type` is 0x73 ('s').
A file signature chunk must be present at the beginning of the file. It may also
be present elsewhere, in which case it encodes no records and is ignored.
`data_size`, `num_records`, and `decoded_data_size` must be 0.
This makes the first 64 bytes of a Riegeli/records file fixed:
```data
83 af 70 d1 0d 88 4a 3f 00 00 00 00 00 00 00 00
40 00 00 00 00 00 00 00 91 ba c2 3c 92 87 e1 a9
00 00 00 00 00 00 00 00 e1 9f 13 c0 e9 b1 c3 72
73 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
```
### File metadata
`chunk_type` is 0x6d ('m').
A file metadata chunk provides information describing the records. Metadata are
not necessary to read the records but might be helpful to interpret their
contents.
If present, metadata should be written immediately after file signature.
The chunk is encoded like a transposed chunk with a single record containing a
serialized `RecordsMetadata` proto message, except that `chunk_type` is
different and `num_records` is 0.
### Padding chunk
`chunk_type` is 0x70 ('p').
A padding chunk encodes no records and only occupies file space.
`num_records` and `decoded_data_size` must be 0. `data` is ignored (usually
filled with zeros by the encoder).
This can be used for more efficient file concatenation (bringing the file offset
modulo `kBlockSize` to 0 allows for physical concatenation of files without
examining their contents), or for syncing to a file system which requires a
particular file offset granularity in order for the sync to be effective.
### Simple chunk with records
`chunk_type` is 0x72 ('r').
Simple chunks store record sizes and concatenated record contents in two
buffers, possibly compressed.
The format:
* `compression_type` (byte) — compression type for sizes and values
* `compressed_sizes_size` (varint64) — size of `compressed_sizes`
* `compressed_sizes` (`compressed_sizes_size` bytes) - compressed buffer with
record sizes
* `compressed_values` (the rest of `data`) — compressed buffer with record
values
`compressed_sizes`, after decompression, contains `num_records` varint64s: the
size of each record.
`compressed_values`, after decompression, contains `decoded_data_size` bytes:
concatenation of record values.
### Transposed chunk with records
`chunk_type` is 0x74 ('t').
TODO: Document this.
## Properties of the file format
* Data corruption anywhere is detected whenever the hash allows this, and it
causes only a local data loss of up to a chunk (if chunk data are damaged)
or block (if chunk header is damaged).
* It is possible to open for append and write more records, even without
reading the original file contents; the original file size must be taken
into account though.
* Seeking to the chunk closest to the given file position requires a seek +
small read, then iterating through chunk headers in a block.
## Implementation notes
The following formulas clarify how certain field values and positions can be
computed.
Constants for fixed sizes:
```c++
kBlockSize = 1 << 16;
kBlockHeaderSize = 24;
kUsableBlockSize = kBlockSize - kBlockHeaderSize;
kChunkHeaderSize = 40;
```
Constraints for chunk boundary distances in a block header:
```c++
previous_chunk % kBlockSize < kUsableBlockSize &&
next_chunk > 0 &&
(next_chunk - 1) % kBlockSize >= kBlockHeaderSize
```
End position of a chunk which begins at `chunk_begin`:
```c++
NumOverheadBlocks(pos, size) =
(size + (pos + kUsableBlockSize - 1) % kBlockSize) / kUsableBlockSize;
AddWithOverhead(pos, size) =
pos + size + NumOverheadBlocks(pos, size) * kBlockHeaderSize;
// Equivalent implementation using unsigned arithmetic modulo 1 << 64:
// RemainingInBlock(pos) = (-pos) % kBlockSize;
RemainingInBlock(pos) = kBlockSize - 1 - (pos + kBlockSize - 1) % kBlockSize;
SaturatingSub(a, b) = a > b ? a - b : 0;
// 0 -> 0, 1..25 -> 25, 26 -> 26, ..., 64K -> 64K, 64K+1..64K+25 -> 64K+25 etc.
RoundUpToPossibleChunkBoundary(pos) =
pos + SaturatingSub(RemainingInBlock(pos), kUsableBlockSize - 1);
chunk_end = max(AddWithOverhead(chunk_begin, kChunkHeaderSize + data_size),
RoundUpToPossibleChunkBoundary(chunk_begin + num_records));
```
Fields of a block header at `block_begin` which interrupts a chunk at
`chunk_begin`:
```c++
prev_chunk = block_begin - chunk_begin;
next_chunk = chunk_end - block_begin;
```
================================================
FILE: python/BUILD
================================================
load("@rules_python//python:defs.bzl", "py_binary")
package(default_visibility = ["//visibility:private"])
licenses(["notice"])
# These dependencies are gathered in a py_binary, instead of directly in
# sh_binary data, so that bazel links __init__.py files to runfiles.
py_binary(
name = "dummy_binary",
srcs = ["dummy_binary.py"],
srcs_version = "PY3",
deps = [
"//python/riegeli",
"//python/riegeli/tensorflow:riegeli_dataset_ops",
],
)
sh_binary(
name = "build_pip_package",
srcs = ["build_pip_package.sh"],
data = [
"MANIFEST.in",
"README.md",
"setup.py",
":dummy_binary",
],
)
================================================
FILE: python/MANIFEST.in
================================================
recursive-include riegeli *.py
================================================
FILE: python/README.md
================================================
# Riegeli
*Riegeli/records* is a file format for storing a sequence of string records,
typically serialized protocol buffers. It supports dense compression, fast
decoding, seeking, detection and optional skipping of data corruption, filtering
of proto message fields for even faster decoding, and parallel encoding.
See [documentation](https://github.com/google/riegeli/blob/master/doc/index.md).
# Status
Riegeli file format will only change in a backward compatible way (i.e. future
readers will understand current files, but current readers might not understand
files using future features).
Riegeli C++ API might change in incompatible ways.
================================================
FILE: python/__init__.py
================================================
================================================
FILE: python/build_pip_package.sh
================================================
#!/bin/bash
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Builds a pip package for riegeli.
#
# Usage (where DEST is a where to write the output, e.g. ~/riegeli-dist):
# $ bazel build -c opt python:build_pip_package
# $ bazel-bin/python/build_pip_package --dest DEST --sdist --bdist
set -e
function is_absolute {
[[ "$1" = /* ]] || [[ "$1" =~ ^[a-zA-Z]:[/\\].* ]]
}
function real_path() {
if is_absolute "$1"; then
printf "%s" "$1"
else
printf "%s/%s" "$PWD" "${1#./}"
fi
}
function build_sdist() {
local dest=$1
python python/setup.py sdist --dist-dir "$dest"
}
function build_bdist() {
local dest=$1
cd bazel-bin/python/build_pip_package.runfiles/com_google_riegeli/python
python setup.py bdist_wheel --dist-dir "$dest"
cd -
}
function main() {
local dest=
local sdist=false
local bdist=false
while [[ $# -gt 0 ]]; do
if [[ $1 == --dest ]]; then
shift
dest=$(real_path "$1")
elif [[ $1 == --sdist ]]; then
sdist=true
elif [[ $1 == --bdist ]]; then
bdist=true
else
printf "Unknown flag: %s\n" "$1" >&2
exit 1
fi
shift
done
if [[ -z $dest ]]; then
printf "Missing required flag: --dest DIRECTORY\n" >&2
exit 1
fi
if [[ $sdist != true ]] && [[ $bdist != true ]]; then
printf "Nothing to do: missing --sdist or --bdist\n" >&2
exit 1
fi
mkdir -p -- "$dest"
if [[ $sdist = true ]]; then
build_sdist "$dest"
fi
if [[ $bdist = true ]]; then
build_bdist "$dest"
fi
}
main "$@"
================================================
FILE: python/dummy_binary.py
================================================
================================================
FILE: python/riegeli/BUILD
================================================
# Riegeli, file format for storing a sequence of records.
load("@rules_python//python:defs.bzl", "py_library")
package(
default_visibility = ["//visibility:public"],
features = ["header_modules"],
)
licenses(["notice"])
exports_files(["LICENSE"])
py_library(
name = "riegeli",
srcs = ["__init__.py"],
imports = [".."],
deps = [
"//python/riegeli/base:riegeli_error",
"//python/riegeli/records:record_position",
"//python/riegeli/records:record_reader",
"//python/riegeli/records:record_writer",
"//python/riegeli/records:records_metadata_py_pb2",
"//python/riegeli/records:skipped_region",
],
)
================================================
FILE: python/riegeli/BUILD.tpl
================================================
load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
licenses(["restricted"])
package(
default_visibility = ["//visibility:public"],
features = ["header_modules"],
)
toolchain(
name = "toolchain",
toolchain = ":py_runtime_pair",
toolchain_type = "@bazel_tools//tools/python:toolchain_type",
)
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
# See https://docs.python.org/3/extending/windows.html
cc_import(
name = "python_lib",
interface_library = select({
":windows": ":python_import_lib",
# A placeholder for Unix platforms which makes --no_build happy.
"//conditions:default": "not-existing.lib",
}),
system_provided = 1,
)
cc_library(
name = "python_headers",
hdrs = [":python_include"],
deps = select({
":windows": [":python_lib"],
"//conditions:default": [],
}),
includes = ["python_include"],
)
cc_library(
name = "numpy_headers",
hdrs = [":numpy_include"],
includes = ["numpy_include"],
)
config_setting(
name = "windows",
values = {"cpu": "x64_windows"},
visibility = ["//visibility:public"],
)
%{PYTHON_RUNTIME_PAIR}
%{PYTHON_INCLUDE_GENRULE}
%{NUMPY_INCLUDE_GENRULE}
%{PYTHON_IMPORT_LIB_GENRULE}
================================================
FILE: python/riegeli/__init__.py
================================================
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Writes or reads Riegeli/records files."""
from riegeli.base import riegeli_error
from riegeli.records import record_position
from riegeli.records import record_reader
from riegeli.records import record_writer
from riegeli.records import records_metadata_pb2
from riegeli.records import skipped_region
__all__ = (
'RiegeliError',
'CancelledError',
'UnknownError',
'InvalidArgumentError',
'DeadlineExceededError',
'NotFoundError',
'AlreadyExistsError',
'PermissionDeniedError',
'UnauthenticatedError',
'ResourceExhaustedError',
'FailedPreconditionError',
'AbortedError',
'OutOfRangeError',
'UnimplementedError',
'InternalError',
'UnavailableError',
'DataLossError',
'FlushType',
'RecordPosition',
'SkippedRegion',
'RecordsMetadata',
'set_record_type',
'RecordWriter',
'EXISTENCE_ONLY',
'get_record_type',
'RecordReader',
)
# pylint: disable=invalid-name
RiegeliError = riegeli_error.RiegeliError
CancelledError = riegeli_error.CancelledError
UnknownError = riegeli_error.UnknownError
InvalidArgumentError = riegeli_error.InvalidArgumentError
DeadlineExceededError = riegeli_error.DeadlineExceededError
NotFoundError = riegeli_error.NotFoundError
AlreadyExistsError = riegeli_error.AlreadyExistsError
PermissionDeniedError = riegeli_error.PermissionDeniedError
UnauthenticatedError = riegeli_error.UnauthenticatedError
ResourceExhaustedError = riegeli_error.ResourceExhaustedError
FailedPreconditionError = riegeli_error.FailedPreconditionError
AbortedError = riegeli_error.AbortedError
OutOfRangeError = riegeli_error.OutOfRangeError
UnimplementedError = riegeli_error.UnimplementedError
InternalError = riegeli_error.InternalError
UnavailableError = riegeli_error.UnavailableError
DataLossError = riegeli_error.DataLossError
RecordPosition = record_position.RecordPosition
SkippedRegion = skipped_region.SkippedRegion
RecordsMetadata = records_metadata_pb2.RecordsMetadata
FlushType = record_writer.FlushType
set_record_type = record_writer.set_record_type
RecordWriter = record_writer.RecordWriter
EXISTENCE_ONLY = record_reader.EXISTENCE_ONLY
get_record_type = record_reader.get_record_type
RecordReader = record_reader.RecordReader
================================================
FILE: python/riegeli/base/BUILD
================================================
load("@rules_cc//cc:defs.bzl", "cc_library")
load("@rules_python//python:defs.bzl", "py_library")
package(
default_visibility = ["//python/riegeli:__subpackages__"],
features = ["header_modules"],
)
licenses(["notice"])
cc_library(
name = "utils",
srcs = ["utils.cc"],
hdrs = ["utils.h"],
data = [":riegeli_error"], # Python module imported from C++.
# utils.cc has #define before #include to influence what the included
# files provide.
features = ["-use_header_modules"],
deps = [
"//riegeli/base:arithmetic",
"//riegeli/base:assert",
"//riegeli/base:chain",
"//riegeli/base:compare",
"//riegeli/base:types",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:string_view",
"@com_google_absl//absl/types:span",
"@rules_python//python/cc:current_py_cc_headers",
],
)
py_library(
name = "riegeli_error",
srcs = ["riegeli_error.py"],
)
================================================
FILE: python/riegeli/base/__init__.py
================================================
================================================
FILE: python/riegeli/base/riegeli_error.py
================================================
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__all__ = (
'RiegeliError',
'CancelledError',
'UnknownError',
'InvalidArgumentError',
'DeadlineExceededError',
'NotFoundError',
'AlreadyExistsError',
'PermissionDeniedError',
'UnauthenticatedError',
'ResourceExhaustedError',
'FailedPreconditionError',
'AbortedError',
'OutOfRangeError',
'UnimplementedError',
'InternalError',
'UnavailableError',
'DataLossError',
)
class RiegeliError(Exception):
"""Base class of errors reported by Google APIs.
Sometimes multiple error codes may apply. Services should return the most
specific error code that applies. For example, prefer `OutOfRangeError` over
`FailedPreconditionError` if both codes apply. Similarly prefer
`NotFoundError` or `AlreadyExistsError` over `FailedPreconditionError`.
Attributes:
code: Error code classifying the error, matching C++ StatusCode.
"""
class CancelledError(RiegeliError):
"""The operation was cancelled, typically by the caller."""
code = 1
class UnknownError(RiegeliError):
"""Unknown error.
For example, this error may be returned when a Status value received from
another address space belongs to an error-space that is not known in this
address space. Also errors raised by APIs that do not return enough error
information may be converted to this error.
"""
code = 2
class InvalidArgumentError(RiegeliError):
"""The client specified an invalid argument.
Note that this differs from `FailedPreconditionError`. `InvalidArgumentError`
indicates arguments that are problematic regardless of the state of the system
(e.g., a malformed file name).
"""
code = 3
class DeadlineExceededError(RiegeliError):
"""The deadline expired before the operation could complete.
For operations that change the state of the system, this error may be returned
even if the operation has completed successfully. or example, a successful
response from a server could have been delayed long enough for the deadline to
expire.
"""
code = 4
class NotFoundError(RiegeliError):
"""Some requested entity (e.g., file or directory) was not found.
Note to server developers: if a request is denied for an entire class of
users, such as gradual feature rollout or undocumented allowlist,
`NotFoundError` may be used. If a request is denied for some users within a
class of users, such as user-based access control, `PermissionDeniedError`
must be used.
"""
code = 5
class AlreadyExistsError(RiegeliError):
"""The entity that a client attempted to create already exists."""
code = 6
class PermissionDeniedError(RiegeliError):
"""The caller does not have permission to execute the specified operation.
`PermissionDeniedError` must not be used for rejections caused by exhausting
some resource (use `ResourceExhaustedError` instead for those errors).
`PermissionDeniedError` must not be used if the caller can not be identified
(use `UnauthenticatedError` instead for those errors). This error code does
not imply the request is valid or the requested entity exists or satisfies
other pre-conditions.
"""
code = 7
class UnauthenticatedError(RiegeliError):
"""No valid authentication credentials for the operation."""
code = 16
class ResourceExhaustedError(RiegeliError):
"""Some resource has been exhausted.
Perhaps a per-user quota, or perhaps the entire file system is out of
space.
"""
code = 8
class FailedPreconditionError(RiegeliError):
"""Failed precondition.
The operation was rejected because the system is not in a state required for
the operation's execution. For example, the directory to be deleted is
non-empty, an rmdir operation is applied to a non-directory, etc.
A litmus test that may help a service implementor in deciding between
`FailedPreconditionError`, `AbortedError`, and `UnavailableError`:
(a) Use `UnavailableError` if the client can retry just the failing call.
(b) Use `AbortedError` if the client should retry at a higher-level (e.g.,
when a client-specified test-and-set fails, indicating the client should
restart a read-modify-write sequence).
(c) Use `FailedPreconditionError` if the client should not retry until the
system state has been explicitly fixed. E.g., if an "rmdir" fails because
the directory is non-empty, `FailedPreconditionError` should be returned
since the client should not retry unless the files are deleted from the
directory.
"""
code = 9
class AbortedError(RiegeliError):
"""The operation was aborted.
Typically due to a concurrency issue such as a sequencer check failure or
transaction abort.
See litmus test at `FailedPreconditionError` for deciding between
`FailedPreconditionError`, `AbortedError`, and `UnavailableError`.
"""
code = 10
class OutOfRangeError(RiegeliError):
"""The operation was attempted past the valid range.
E.g., seeking or reading past end-of-file.
Unlike `InvalidArgumentError`, this error indicates a problem that may be
fixed if the system state changes. For example, a 32-bit file system will
generate `InvalidArgumentError` if asked to read at an offset that is not in
the range [0,2^32-1], but it will generate `OutOfRangeError` if asked to read
from an offset past the current file size.
There is a fair bit of overlap between `FailedPreconditionError` and
`OutOfRangeError`. We recommend using `OutOfRangeError` (the more specific
error) when it applies so that callers who are iterating through a space can
easily look for an `OutOfRangeError` error to detect when they are done.
"""
code = 11
class UnimplementedError(RiegeliError):
"""The operation is not implemented.
Or is not supported/enabled in this service.
"""
code = 12
class InternalError(RiegeliError):
"""Internal errors.
This means that some invariants expected by the underlying system have been
broken. This error code is reserved for serious errors.
"""
code = 13
class UnavailableError(RiegeliError):
"""The service is currently unavailable.
This is most likely a transient condition, which can be corrected by retrying
with a backoff.
See litmus test at `FailedPreconditionError` for deciding between
`FailedPreconditionError`, `AbortedError`, and `UnavailableError`.
"""
code = 14
class DataLossError(RiegeliError):
"""Unrecoverable data loss or corruption."""
code = 15
================================================
FILE: python/riegeli/base/utils.cc
================================================
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#define PY_SSIZE_T_CLEAN
#include <Python.h>
// clang-format: do not reorder the above include.
#include "python/riegeli/base/utils.h"
// clang-format: do not reorder the above include.
#include <stddef.h>
#include <stdint.h>
#include <limits>
#include <optional>
#include <string>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/base/optimization.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/types/span.h"
#include "riegeli/base/arithmetic.h"
#include "riegeli/base/assert.h"
#include "riegeli/base/chain.h"
#include "riegeli/base/compare.h"
#include "riegeli/base/types.h"
namespace riegeli::python {
Exception& Exception::operator=(const Exception& that) noexcept {
PythonLock lock;
Py_XINCREF(that.type_.get());
type_.reset(that.type_.get());
Py_XINCREF(that.value_.get());
value_.reset(that.value_.get());
Py_XINCREF(that.traceback_.get());
traceback_.reset(that.traceback_.get());
return *this;
}
Exception Exception::Fetch() {
PythonLock::AssertHeld();
PyObject* type;
PyObject* value;
PyObject* traceback;
PyErr_Fetch(&type, &value, &traceback);
PyErr_NormalizeException(&type, &value, &traceback);
return Exception(type, value, traceback);
}
PyObject* Exception::Restore() const& {
PythonLock::AssertHeld();
Py_XINCREF(type_.get());
Py_XINCREF(value_.get());
Py_XINCREF(traceback_.get());
PyErr_Restore(type_.get(), value_.get(), traceback_.get());
return nullptr;
}
PyObject* Exception::Restore() && {
PythonLock::AssertHeld();
PyErr_Restore(type_.release(), value_.release(), traceback_.release());
return nullptr;
}
std::string Exception::message() const {
if (ok()) return "OK";
PythonLock lock;
RIEGELI_ASSERT(PyExceptionClass_Check(type_.get()))
<< "Expected an exception class, not " << Py_TYPE(type_.get())->tp_name;
std::string message = PyExceptionClass_Name(type_.get());
if (value_ == nullptr) return message;
const PythonPtr str_result(PyObject_Str(value_.get()));
if (ABSL_PREDICT_FALSE(str_result == nullptr)) {
PyErr_Clear();
absl::StrAppend(&message, ": <str() failed>");
return message;
}
StrOrBytes str;
if (ABSL_PREDICT_FALSE(!str.FromPython(str_result.get()))) {
PyErr_Clear();
absl::StrAppend(&message, ": <StrOrBytes::FromPython() failed>");
return message;
}
if (!absl::string_view(str).empty()) {
absl::StrAppend(&message, ": ", absl::string_view(str));
}
return message;
}
void SetRiegeliError(const absl::Status& status) {
RIEGELI_ASSERT(!status.ok())
<< "Failed precondition of SetRiegeliError(): status not failed";
PythonLock::AssertHeld();
PythonPtr message = StringToPython(status.message());
if (ABSL_PREDICT_FALSE(message == nullptr)) return;
PyObject* type;
switch (status.code()) {
#define HANDLE_CODE(name) \
case absl::StatusCode::k##name: { \
static constexpr ImportedConstant k##name##Error( \
"riegeli.base.riegeli_error", #name "Error"); \
if (ABSL_PREDICT_FALSE(!k##name##Error.Verify())) return; \
type = k##name##Error.get(); \
} break
// clang-format off
HANDLE_CODE(Cancelled);
default:
HANDLE_CODE(Unknown);
HANDLE_CODE(InvalidArgument);
HANDLE_CODE(DeadlineExceeded);
HANDLE_CODE(NotFound);
HANDLE_CODE(AlreadyExists);
HANDLE_CODE(PermissionDenied);
HANDLE_CODE(Unauthenticated);
HANDLE_CODE(ResourceExhausted);
HANDLE_CODE(FailedPrecondition);
HANDLE_CODE(Aborted);
HANDLE_CODE(OutOfRange);
HANDLE_CODE(Unimplemented);
HANDLE_CODE(Internal);
HANDLE_CODE(Unavailable);
HANDLE_CODE(DataLoss);
// clang-format on
#undef HANDLE_CODE
}
Py_INCREF(type);
PyErr_Restore(type, message.release(), nullptr);
}
namespace py_internal {
namespace {
// A linked list of all objects of type `StaticObject` which have `value_`
// allocated, chained by their `next_` fields. This is used to free the objects
// on Python interpreter shutdown.
const StaticObject* all_static_objects = nullptr;
} // namespace
void FreeStaticObjectsImpl() {
const StaticObject* static_object =
std::exchange(all_static_objects, nullptr);
while (static_object != nullptr) {
static_object->value_ = nullptr;
static_object = std::exchange(static_object->next_, nullptr);
}
}
// `extern "C"` sets the C calling convention for compatibility with the Python
// API. `static` avoids making symbols public, as `extern "C"` trumps anonymous
// namespace.
extern "C" {
static void FreeStaticObjects() { FreeStaticObjectsImpl(); }
} // extern "C"
void StaticObject::RegisterThis() const {
PythonLock::AssertHeld();
if (all_static_objects == nullptr) {
// This is the first registered `StaticObject` since `Py_Initialize()`.
Py_AtExit(FreeStaticObjects);
}
next_ = std::exchange(all_static_objects, this);
}
bool ImportedCapsuleBase::ImportValue() const {
// For some reason `PyImport_ImportModule()` is sometimes required before
// `PyCapsule_Import()` for a module with a nested name.
const size_t dot = absl::string_view(capsule_name_).rfind('.');
RIEGELI_ASSERT_NE(dot, absl::string_view::npos)
<< "Capsule name does not contain a dot: " << capsule_name_;
RIEGELI_CHECK(
PyImport_ImportModule(std::string(capsule_name_, dot).c_str()) != nullptr)
<< Exception::Fetch().message();
value_ = PyCapsule_Import(capsule_name_, false);
return value_ != nullptr;
}
} // namespace py_internal
bool Identifier::AllocateValue() const {
value_ = StringToPython(name_).release();
if (ABSL_PREDICT_FALSE(value_ == nullptr)) return false;
PyUnicode_InternInPlace(&value_);
RegisterThis();
return true;
}
bool ImportedConstant::AllocateValue() const {
const PythonPtr module_name = StringToPython(module_name_);
if (ABSL_PREDICT_FALSE(module_name == nullptr)) return false;
const PythonPtr module(PyImport_Import(module_name.get()));
if (ABSL_PREDICT_FALSE(module == nullptr)) return false;
const PythonPtr attr_name = StringToPython(attr_name_);
if (ABSL_PREDICT_FALSE(attr_name == nullptr)) return false;
value_ = PyObject_GetAttr(module.get(), attr_name.get());
if (ABSL_PREDICT_FALSE(value_ == nullptr)) return false;
RegisterThis();
return true;
}
bool ExportCapsule(PyObject* module, const char* capsule_name,
const void* ptr) {
PythonPtr capsule(
PyCapsule_New(const_cast<void*>(ptr), capsule_name, nullptr));
if (ABSL_PREDICT_FALSE(capsule == nullptr)) return false;
const size_t dot = absl::string_view(capsule_name).rfind('.');
RIEGELI_ASSERT_NE(dot, absl::string_view::npos)
<< "Capsule name does not contain a dot: " << capsule_name;
RIEGELI_ASSERT(PyModule_Check(module))
<< "Expected a module, not " << Py_TYPE(module)->tp_name;
RIEGELI_ASSERT_EQ(absl::string_view(PyModule_GetName(module)),
absl::string_view(capsule_name, dot))
<< "Module name mismatch";
if (ABSL_PREDICT_FALSE(PyModule_AddObject(module, capsule_name + dot + 1,
capsule.release()) < 0)) {
return false;
}
return true;
}
MemoryView::~MemoryView() {
if (object_ != nullptr && Py_REFCNT(object_.get()) > 1) {
PyObject* value;
PyObject* type;
PyObject* traceback;
PyErr_Fetch(&value, &type, &traceback);
ReleaseInternal();
PyErr_Restore(value, type, traceback);
}
}
PyObject* MemoryView::ToPython(absl::string_view value) {
RIEGELI_ASSERT_EQ(object_, nullptr)
<< "Failed precondition of MemoryView::ToPython(): "
"called more than once";
object_.reset(PyMemoryView_FromMemory(const_cast<char*>(value.data()),
IntCast<Py_ssize_t>(value.size()),
PyBUF_READ));
return object_.get();
}
PyObject* MemoryView::MutableToPython(absl::Span<char> value) {
RIEGELI_ASSERT_EQ(object_, nullptr)
<< "Failed precondition of MemoryView::MutableToPython(): "
"called more than once";
object_.reset(PyMemoryView_FromMemory(
value.data(), IntCast<Py_ssize_t>(value.size()), PyBUF_WRITE));
return object_.get();
}
bool MemoryView::Release() {
bool release_ok = true;
if (object_ != nullptr && Py_REFCNT(object_.get()) > 1) {
release_ok = ReleaseInternal();
}
object_.reset();
return release_ok;
}
inline bool MemoryView::ReleaseInternal() {
static constexpr Identifier id_release("release");
const PythonPtr release_result(
PyObject_CallMethodObjArgs(object_.get(), id_release.get(), nullptr));
return release_result != nullptr;
}
bool StrOrBytes::FromPython(PyObject* object ABSL_ATTRIBUTE_LIFETIME_BOUND) {
RIEGELI_ASSERT_EQ(data_.data(), nullptr)
<< "Failed precondition of StrOrBytes::FromPython(): "
"called more than once";
if (PyUnicode_Check(object)) {
Py_ssize_t length;
const char* data = PyUnicode_AsUTF8AndSize(object, &length);
if (ABSL_PREDICT_FALSE(data == nullptr)) return false;
data_ = absl::string_view(data, IntCast<size_t>(length));
return true;
} else if (ABSL_PREDICT_FALSE(!PyBytes_Check(object))) {
PyErr_Format(PyExc_TypeError, "Expected str or bytes, not %s",
Py_TYPE(object)->tp_name);
return false;
}
data_ = absl::string_view(PyBytes_AS_STRING(object),
IntCast<size_t>(PyBytes_GET_SIZE(object)));
return true;
}
PythonPtr ChainToPython(const Chain& value) {
PythonPtr bytes(
PyBytes_FromStringAndSize(nullptr, IntCast<Py_ssize_t>(value.size())));
if (ABSL_PREDICT_FALSE(bytes == nullptr)) return nullptr;
value.CopyTo(PyBytes_AS_STRING(bytes.get()));
return bytes;
}
std::optional<Chain> ChainFromPython(PyObject* object) {
Py_buffer buffer;
if (ABSL_PREDICT_FALSE(PyObject_GetBuffer(object, &buffer, PyBUF_CONTIG_RO) <
0)) {
return std::nullopt;
}
Chain result(absl::string_view(static_cast<const char*>(buffer.buf),
IntCast<size_t>(buffer.len)));
PyBuffer_Release(&buffer);
return result;
}
PythonPtr SizeToPython(size_t value) {
if (ABSL_PREDICT_FALSE(value >
std::numeric_limits<unsigned long long>::max())) {
PyErr_Format(PyExc_OverflowError, "Size out of range: %zu", value);
return nullptr;
}
return PythonPtr(
PyLong_FromUnsignedLongLong(IntCast<unsigned long long>(value)));
}
std::optional<size_t> SizeFromPython(PyObject* object) {
const PythonPtr index(PyNumber_Index(object));
if (ABSL_PREDICT_FALSE(index == nullptr)) return std::nullopt;
RIEGELI_ASSERT(PyLong_Check(index.get()))
<< "PyNumber_Index() returned an unexpected type: "
<< Py_TYPE(index.get())->tp_name;
unsigned long long index_value = PyLong_AsUnsignedLongLong(index.get());
if (ABSL_PREDICT_FALSE(index_value == static_cast<unsigned long long>(-1)) &&
PyErr_Occurred()) {
return std::nullopt;
}
if (ABSL_PREDICT_FALSE(index_value > std::numeric_limits<size_t>::max())) {
PyErr_Format(PyExc_OverflowError, "Size out of range: %llu", index_value);
return std::nullopt;
}
return IntCast<size_t>(index_value);
}
PythonPtr PositionToPython(Position value) {
if (ABSL_PREDICT_FALSE(value >
std::numeric_limits<unsigned long long>::max())) {
PyErr_Format(PyExc_OverflowError, "Position out of range: %ju",
uintmax_t{value});
return nullptr;
}
return PythonPtr(
PyLong_FromUnsignedLongLong(IntCast<unsigned long long>(value)));
}
std::optional<Position> PositionFromPython(PyObject* object) {
const PythonPtr index(PyNumber_Index(object));
if (ABSL_PREDICT_FALSE(index == nullptr)) return std::nullopt;
RIEGELI_ASSERT(PyLong_Check(index.get()))
<< "PyNumber_Index() returned an unexpected type: "
<< Py_TYPE(index.get())->tp_name;
const unsigned long long index_value = PyLong_AsUnsignedLongLong(index.get());
if (ABSL_PREDICT_FALSE(index_value == static_cast<unsigned long long>(-1)) &&
PyErr_Occurred()) {
return std::nullopt;
}
if (ABSL_PREDICT_FALSE(index_value > std::numeric_limits<Position>::max())) {
PyErr_Format(PyExc_OverflowError, "Position out of range: %llu",
index_value);
return std::nullopt;
}
return IntCast<Position>(index_value);
}
PythonPtr PartialOrderingToPython(PartialOrdering ordering) {
if (ordering == PartialOrdering::unordered) {
return Py_INCREF(Py_None), PythonPtr(Py_None);
}
return PythonPtr(PyLong_FromLong(ordering < 0 ? -1 : ordering == 0 ? 0 : 1));
}
std::optional<PartialOrdering> PartialOrderingFromPython(PyObject* object) {
if (object == Py_None) return PartialOrdering::unordered;
const long long_value = PyLong_AsLong(object);
if (ABSL_PREDICT_FALSE(long_value == -1) && PyErr_Occurred()) {
return std::nullopt;
}
return riegeli::Compare(long_value, 0);
}
} // namespace riegeli::python
================================================
FILE: python/riegeli/base/utils.h
================================================
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef PYTHON_RIEGELI_BASE_UTILS_H_
#define PYTHON_RIEGELI_BASE_UTILS_H_
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#include <Python.h>
// clang-format: do not reorder the above include.
#include <stddef.h>
#include <memory>
#include <new>
#include <optional>
#include <string>
#include <type_traits>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/base/optimization.h"
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
#include "absl/types/span.h"
#include "riegeli/base/arithmetic.h"
#include "riegeli/base/assert.h"
#include "riegeli/base/chain.h"
#include "riegeli/base/compare.h"
#include "riegeli/base/types.h"
namespace riegeli::python {
// Ensures that Python GIL is locked. Reentrant.
//
// Same as `PyGILState_Ensure()` / `PyGILState_Release()`.
class PythonLock {
public:
static void AssertHeld() {
RIEGELI_ASSERT(PyGILState_Check()) << "Python GIL was assumed to be held";
}
PythonLock() { gstate_ = PyGILState_Ensure(); }
PythonLock(const PythonLock&) = delete;
PythonLock& operator=(const PythonLock&) = delete;
~PythonLock() { PyGILState_Release(gstate_); }
private:
PyGILState_STATE gstate_;
};
// Unlocks Python GIL, allowing non-Python threads to run.
//
// Same as `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS`.
class PythonUnlock {
public:
PythonUnlock() {
PythonLock::AssertHeld();
tstate_ = PyEval_SaveThread();
}
PythonUnlock(const PythonUnlock&) = delete;
PythonUnlock& operator=(const PythonUnlock&) = delete;
~PythonUnlock() { PyEval_RestoreThread(tstate_); }
private:
PyThreadState* tstate_;
};
// Apply a function with Python GIL unlocked, allowing non-Python threads to
// run.
//
// Same as `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS`.
template <typename Function>
std::invoke_result_t<Function> PythonUnlocked(Function&& f) {
PythonUnlock unlock;
return std::forward<Function>(f)();
}
// Owned `PyObject` which assumes that Python GIL is held.
struct Deleter {
template <typename T>
void operator()(T* ptr) const {
PythonLock::AssertHeld();
Py_DECREF(ptr);
}
};
using PythonPtr = std::unique_ptr<PyObject, Deleter>;
// Owned `PyObject` which does not assume that Python GIL is held.
struct LockingDeleter {
template <typename T>
void operator()(T* ptr) const {
PythonLock lock;
Py_DECREF(ptr);
}
};
using PythonPtrLocking = std::unique_ptr<PyObject, LockingDeleter>;
// Allows a C++ object to be safely embedded in a Python object allocated with
// `PyType_GenericAlloc()`.
//
// `PythonWrapped<T>` is similar to `std::optional<T>`, but:
// * `PythonWrapped<T>` is POD.
// * `PythonWrapped<T>` supports only a subset of `std::optional<T>` API.
// * `PythonWrapped<T>` filled with zero bytes is valid and absent
// (`PyType_GenericAlloc()` fills the Python object with zero bytes).
// * `PythonWrapped<T>` should be explicitly `reset()` in the implementation of
// `tp_dealloc` (there is no C++ destructor).
template <typename T>
class PythonWrapped {
public:
static_assert(alignof(T) <= alignof(max_align_t),
"PythonWrapped does not support overaligned types");
template <typename... Args>
ABSL_ATTRIBUTE_REINITIALIZES void emplace(Args&&... args) {
if (has_value_) {
get()->~T();
} else {
has_value_ = true;
}
new (storage_) T(std::forward<Args>(args)...);
}
ABSL_ATTRIBUTE_REINITIALIZES void reset() {
if (has_value_) {
get()->~T();
has_value_ = false;
}
}
bool has_value() const { return has_value_; }
T* get() ABSL_ATTRIBUTE_LIFETIME_BOUND {
RIEGELI_ASSERT(has_value_) << "Object uninitialized";
return std::launder(reinterpret_cast<T*>(storage_));
}
const T* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
RIEGELI_ASSERT(has_value_) << "Object uninitialized";
return std::launder(reinterpret_cast<const T*>(storage_));
}
T& operator*() ABSL_ATTRIBUTE_LIFETIME_BOUND { return *get(); }
const T& operator*() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return *get(); }
T* operator->() ABSL_ATTRIBUTE_LIFETIME_BOUND { return get(); }
const T* operator->() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return get(); }
bool Verify() const {
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(!has_value())) {
PyErr_SetString(PyExc_ValueError, "Object uninitialized");
return false;
}
return true;
}
private:
bool has_value_;
alignas(T) char storage_[sizeof(T)];
};
// Represents an optional Python exception being raised.
class Exception {
public:
// No exception.
Exception() = default;
Exception(const Exception& that) noexcept;
Exception& operator=(const Exception& that) noexcept;
Exception(Exception&& that) = default;
Exception& operator=(Exception&& that) = default;
// Fetches the active Python exception.
static Exception Fetch();
// Restores the active Python exception.
PyObject* Restore() const&;
PyObject* Restore() &&;
bool ok() const { return type_ == nullptr; }
std::string message() const;
// For implementing `tp_traverse` of objects containing `Exception`.
int Traverse(visitproc visit, void* arg);
private:
// Steals references.
explicit Exception(PyObject* type, PyObject* value, PyObject* traceback)
: type_(type), value_(value), traceback_(traceback) {}
PythonPtrLocking type_;
PythonPtrLocking value_;
PythonPtrLocking traceback_;
};
// Translate a failed status to the active Python exception, a class extending
// `RiegeliError`.
void SetRiegeliError(const absl::Status& status);
namespace py_internal {
// Lazily initialized pointer to a Python object, persisting until interpreter
// shutdown.
class StaticObject {
protected:
mutable PyObject* value_ = nullptr;
mutable const StaticObject* next_ = nullptr;
// Register this object in a global list of static objects. This must be
// called when `value_` is allocated.
void RegisterThis() const;
private:
friend void FreeStaticObjectsImpl();
};
// Template parameter independent part of `ImportedCapsule`.
class ImportedCapsuleBase {
public:
// Forces importing the value, returning `false` on failures (with Python
// exception set).
//
// If `Verify()` returns `true`, `get()` does not die.
bool Verify() const {
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(value_ == nullptr)) return ImportValue();
return true;
}
protected:
explicit constexpr ImportedCapsuleBase(const char* capsule_name)
: capsule_name_(capsule_name) {}
~ImportedCapsuleBase() = default;
bool ImportValue() const;
mutable void* value_ = nullptr;
private:
const char* capsule_name_;
};
} // namespace py_internal
// Creates a Python string (type `str`) which persists until interpreter
// shutdown. This is useful for attribute or method names in
// `PyObject_GetAttr()` or `PyObject_CallMethodObjArgs()`.
//
// An instance of `Identifier` should be allocated statically:
// ```
// static constexpr Identifier id_write("write");
// ```
//
// Then `id_write.get()` is a borrowed reference to the Python object.
class Identifier : public py_internal::StaticObject {
public:
explicit constexpr Identifier(absl::string_view name) : name_(name) {}
// Forces allocating the value, returning `false` on failures (with Python
// exception set).
//
// If `Verify()` returns `true`, `get()` does not die.
bool Verify() const {
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(value_ == nullptr)) return AllocateValue();
return true;
}
// Returns the value, allocating it on the first call. Dies on failure
// (use `Verify()` to prevent this).
PyObject* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(value_ == nullptr)) {
RIEGELI_CHECK(AllocateValue()) << Exception::Fetch().message();
}
return value_;
}
private:
bool AllocateValue() const;
absl::string_view name_;
};
// Imports a Python module and gets its attribute, which persists until
// interpreter shutdown.
//
// An instance of `ImportedConstant` should be allocated statically:
// ```
// static constexpr ImportedConstant kRiegeliError(
// "riegeli.base.riegeli_error", "RiegeliError");
// ```
//
// Then `kRiegeliError.get()` is a borrowed reference to the Python object.
class ImportedConstant : public py_internal::StaticObject {
public:
explicit constexpr ImportedConstant(absl::string_view module_name,
absl::string_view attr_name)
: module_name_(module_name), attr_name_(attr_name) {}
// Forces importing the value, returning `false` on failures (with Python
// exception set).
//
// If `Verify()` returns `true`, `get()` does not die.
bool Verify() const {
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(value_ == nullptr)) return AllocateValue();
return true;
}
// Returns the value, importing it on the first call. Dies on failure
// (use `Verify()` to prevent this).
PyObject* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(value_ == nullptr)) {
RIEGELI_CHECK(AllocateValue()) << Exception::Fetch().message();
}
return value_;
}
private:
bool AllocateValue() const;
absl::string_view module_name_;
absl::string_view attr_name_;
};
// Exports a Python capsule containing a C++ pointer, which should be valid
// forever, by adding it to the given module.
//
// `capsule_name` must be "module_name.attr_name" with `module_name`
// corresponding to `PyModule_GetName(module)`.
//
// Returns `false` on failure (with Python exception set).
bool ExportCapsule(PyObject* module, const char* capsule_name, const void* ptr);
// Imports a Python capsule and gets its stored pointer, which persists forever.
//
// `capsule_name must` be "module_name.attr_name".
//
// An instance of `ImportedCapsule` should be allocated statically:
// ```
// static constexpr ImportedCapsule<RecordPositionApi> kRecordPositionApi(
// "riegeli.records.record_position._CPPAPI");
// ```
//
// Then `kRecordPositionApi.get()` is a pointer stored in the capsule.
template <typename T>
class ImportedCapsule : public py_internal::ImportedCapsuleBase {
public:
explicit constexpr ImportedCapsule(const char* capsule_name)
: ImportedCapsuleBase(capsule_name) {}
// Returns the value, importing it on the first call. Dies on failure
// (use `Verify()` to prevent this).
const T* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(value_ == nullptr)) {
RIEGELI_CHECK(ImportValue()) << Exception::Fetch().message();
}
return static_cast<const T*>(value_);
}
const T& operator*() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return *get(); }
const T* operator->() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return get(); }
};
// Converts C++ `long` to a Python `int` object.
//
// Returns `nullptr` on failure (with Python exception set).
inline PythonPtr IntToPython(long value) {
return PythonPtr(PyLong_FromLong(value));
}
// Converts C++ `absl::string_view` to a Python `bytes` object.
//
// Returns `nullptr` on failure (with Python exception set).
inline PythonPtr BytesToPython(absl::string_view value) {
return PythonPtr(PyBytes_FromStringAndSize(
value.data(), IntCast<Py_ssize_t>(value.size())));
}
// Converts C++ array of bytes to a Python `memoryview` object.
//
// Memory is shared. The C++ memory must be valid as long as the Python object
// is needed.
class MemoryView {
public:
MemoryView() = default;
MemoryView(const MemoryView&) = delete;
MemoryView& operator=(const MemoryView&) = delete;
// Calls `Release()`, ignoring its result, without disturbing the Python
// exception state.
~MemoryView();
// Creates and returns a read-only `memoryview` object.
//
// Returns `nullptr` on failure (with Python exception set).
//
// `ToPython()` or `MutableToPython()` must be called at most once for each
// `MemoryView` object.
PyObject* ToPython(absl::string_view value);
// Creates and returns a mutable `memoryview` object.
//
// Returns `nullptr` on failure (with Python exception set).
//
// `ToPython()` or `MutableToPython()` must be called at most once for each
// `MemoryView` object.
PyObject* MutableToPython(absl::Span<char> value);
// If a reference to the `memoryview` has been stored elsewhere, calls
// `memoryview.release()` to mark the `memoryview` as invalid.
//
// Returns `false` on failure (with Python exception set).
bool Release();
private:
bool ReleaseInternal();
PythonPtr object_;
};
// Refers to internals of a Python `bytes`-like object, using the buffer
// protocol.
class BytesLike {
public:
BytesLike() noexcept { buffer_.obj = nullptr; }
BytesLike(const BytesLike&) = delete;
BytesLike& operator=(const BytesLike&) = delete;
~BytesLike() {
PythonLock::AssertHeld();
if (buffer_.obj != nullptr) PyBuffer_Release(&buffer_);
}
// Converts from a Python object.
//
// Returns `false` on failure (with Python exception set).
//
// Must be called at most once for each `BytesLike` object.
bool FromPython(PyObject* object) {
RIEGELI_ASSERT_EQ(buffer_.obj, nullptr)
<< "Failed precondition of BytesLike::FromPython(): "
"called more than once";
return PyObject_GetBuffer(object, &buffer_, PyBUF_CONTIG_RO) == 0;
}
// Returns the binary contents.
/*implicit*/ operator absl::string_view() const
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return absl::string_view(static_cast<const char*>(buffer_.buf),
IntCast<size_t>(buffer_.len));
}
private:
Py_buffer buffer_;
};
// Converts C++ `absl::string_view` to a Python `str` object. Unicode is
// converted from UTF-8.
//
// Returns `nullptr` on failure (with Python exception set).
inline PythonPtr StringToPython(absl::string_view value) {
return PythonPtr(PyUnicode_FromStringAndSize(
value.data(), IntCast<Py_ssize_t>(value.size())));
}
// Refers to internals of a Python object representing text. Valid Python
// objects are `str` or `bytes`. Unicode is converted to UTF-8.
class StrOrBytes {
public:
StrOrBytes() noexcept {}
StrOrBytes(const StrOrBytes&) = delete;
StrOrBytes& operator=(const StrOrBytes&) = delete;
// Converts from a Python object.
//
// Returns `false` on failure (with Python exception set).
//
// Must be called at most once for each `StrOrBytes` object.
bool FromPython(PyObject* object ABSL_ATTRIBUTE_LIFETIME_BOUND);
// Returns the text contents.
/*implicit*/ operator absl::string_view() const { return data_; }
private:
absl::string_view data_;
};
// Converts C++ `Chain` to a Python `bytes` object.
//
// Returns `nullptr` on failure (with Python exception set).
PythonPtr ChainToPython(const Chain& value);
// Converts a Python `bytes`-like object to C++ `Chain`, using the buffer
// protocol.
//
// Returns `std::nullopt` on failure (with Python exception set).
std::optional<Chain> ChainFromPython(PyObject* object);
// Converts C++ `size_t` to a Python `int` object.
//
// Returns `nullptr` on failure (with Python exception set).
PythonPtr SizeToPython(size_t value);
// Converts a Python object to C++ `size_t`. Valid Python objects are the same
// as for slicing: `int` or objects supporting `__index__()`.
//
// Returns `std::nullopt` on failure (with Python exception set).
std::optional<size_t> SizeFromPython(PyObject* object);
// Converts C++ `Position` to a Python `int` object.
//
// Returns `nullptr` on failure (with Python exception set).
PythonPtr PositionToPython(Position value);
// Converts a Python object to C++ `Position`. Valid Python objects are the same
// as for slicing: `int` or objects supporting `__index__()`.
//
// Returns `std::nullopt` on failure (with Python exception set).
std::optional<Position> PositionFromPython(PyObject* object);
// Converts C++ `PartialOrdering` to a Python `None` (for `unordered`) or `int`
// object (-1 for `less`, 0 for `equivalent`, or 1 for `greater`).
//
// Returns `nullptr` on failure (with Python exception set).
PythonPtr PartialOrderingToPython(PartialOrdering ordering);
// Converts a Python object to C++ `PartialOrdering`. Valid Python objects are
// `int` (compared with 0) or `None`.
//
// Returns `std::nullopt` on failure (with Python exception set).
std::optional<PartialOrdering> PartialOrderingFromPython(PyObject* object);
// Implementation details follow.
inline Exception::Exception(const Exception& that) noexcept { *this = that; }
inline int Exception::Traverse(visitproc visit, void* arg) {
Py_VISIT(type_.get());
Py_VISIT(value_.get());
Py_VISIT(traceback_.get());
return 0;
}
} // namespace riegeli::python
#endif // PYTHON_RIEGELI_BASE_UTILS_H_
================================================
FILE: python/riegeli/bytes/BUILD
================================================
load("@rules_cc//cc:defs.bzl", "cc_library")
package(
default_visibility = ["//python/riegeli:__subpackages__"],
features = ["header_modules"],
)
licenses(["notice"])
cc_library(
name = "python_reader",
srcs = ["python_reader.cc"],
hdrs = ["python_reader.h"],
# python_reader.cc has #define before #include to influence what the
# included files provide.
features = ["-use_header_modules"],
deps = [
"//python/riegeli/base:utils",
"//riegeli/base:arithmetic",
"//riegeli/base:assert",
"//riegeli/base:global",
"//riegeli/base:object",
"//riegeli/base:types",
"//riegeli/bytes:buffer_options",
"//riegeli/bytes:buffered_reader",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/numeric:bits",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:string_view",
"@com_google_absl//absl/types:span",
"@rules_python//python/cc:current_py_cc_headers",
],
)
cc_library(
name = "python_writer",
srcs = ["python_writer.cc"],
hdrs = ["python_writer.h"],
# python_writer.cc has #define before #include to influence what the
# included files provide.
features = ["-use_header_modules"],
deps = [
"//python/riegeli/base:utils",
"//riegeli/base:arithmetic",
"//riegeli/base:assert",
"//riegeli/base:global",
"//riegeli/base:object",
"//riegeli/base:types",
"//riegeli/bytes:buffer_options",
"//riegeli/bytes:buffered_writer",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/numeric:bits",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:string_view",
"@rules_python//python/cc:current_py_cc_headers",
],
)
================================================
FILE: python/riegeli/bytes/python_reader.cc
================================================
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#define PY_SSIZE_T_CLEAN
#include <Python.h>
// clang-format: do not reorder the above include.
#include "python/riegeli/bytes/python_reader.h"
// clang-format: do not reorder the above include.
#include <stddef.h>
#include <cstring>
#include <limits>
#include <optional>
#include "absl/base/optimization.h"
#include "absl/numeric/bits.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/types/span.h"
#include "python/riegeli/base/utils.h"
#include "riegeli/base/arithmetic.h"
#include "riegeli/base/assert.h"
#include "riegeli/base/global.h"
#include "riegeli/base/types.h"
#include "riegeli/bytes/buffered_reader.h"
namespace riegeli::python {
PythonReader::PythonReader(PyObject* src, Options options)
: BufferedReader(options.buffer_options()), owns_src_(options.owns_src()) {
PythonLock::AssertHeld();
Py_INCREF(src);
src_.reset(src);
if (options.assumed_pos() != std::nullopt) {
set_limit_pos(*options.assumed_pos());
// `supports_random_access_` is left as `false`.
random_access_status_ = Global([] {
return absl::UnimplementedError(
"PythonReader::Options::assumed_pos() excludes random access");
});
} else {
static constexpr Identifier id_seekable("seekable");
const PythonPtr seekable_result(
PyObject_CallMethodObjArgs(src_.get(), id_seekable.get(), nullptr));
if (ABSL_PREDICT_FALSE(seekable_result == nullptr)) {
FailOperation("seekable()");
return;
}
const int seekable_is_true = PyObject_IsTrue(seekable_result.get());
if (ABSL_PREDICT_FALSE(seekable_is_true < 0)) {
FailOperation("PyObject_IsTrue() after seekable()");
return;
}
if (seekable_is_true == 0) {
// Random access is not supported. Assume 0 as the initial position.
// `supports_random_access_` is left as `false`.
random_access_status_ = Global([] {
return absl::UnimplementedError(
"seekable() is False which excludes random access");
});
return;
}
static constexpr Identifier id_tell("tell");
const PythonPtr tell_result(
PyObject_CallMethodObjArgs(src_.get(), id_tell.get(), nullptr));
if (ABSL_PREDICT_FALSE(tell_result == nullptr)) {
FailOperation("tell()");
return;
}
const std::optional<Position> file_pos =
PositionFromPython(tell_result.get());
if (ABSL_PREDICT_FALSE(file_pos == std::nullopt)) {
FailOperation("PositionFromPython() after tell()");
return;
}
set_limit_pos(*file_pos);
supports_random_access_ = true;
}
BeginRun();
}
void PythonReader::Done() {
BufferedReader::Done();
random_access_status_ = absl::OkStatus();
if (owns_src_ && src_ != nullptr) {
PythonLock lock;
static constexpr Identifier id_close("close");
const PythonPtr close_result(
PyObject_CallMethodObjArgs(src_.get(), id_close.get(), nullptr));
if (ABSL_PREDICT_FALSE(close_result == nullptr)) FailOperation("close()");
}
}
inline bool PythonReader::FailOperation(absl::string_view operation) {
RIEGELI_ASSERT(is_open())
<< "Failed precondition of PythonReader::FailOperation(): "
"Object closed";
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(!ok())) {
// Ignore this error because `PythonReader` already failed.
PyErr_Clear();
return false;
}
exception_ = Exception::Fetch();
return Fail(absl::UnknownError(
absl::StrCat(operation, " failed: ", exception_.message())));
}
bool PythonReader::ReadInternal(size_t min_length, size_t max_length,
char* dest) {
RIEGELI_ASSERT_GT(min_length, 0u)
<< "Failed precondition of BufferedReader::ReadInternal(): "
"nothing to read";
RIEGELI_ASSERT_GE(max_length, min_length)
<< "Failed precondition of BufferedReader::ReadInternal(): "
"max_length < min_length";
RIEGELI_ASSERT_OK(*this)
<< "Failed precondition of BufferedReader::ReadInternal()";
PythonLock lock;
// Find a read function to use, preferring in order: `readinto1()`,
// `readinto()`, `read1()`, `read()`.
if (ABSL_PREDICT_FALSE(read_function_ == nullptr)) {
static constexpr Identifier id_readinto1("readinto1");
read_function_.reset(PyObject_GetAttr(src_.get(), id_readinto1.get()));
read_function_name_ = "readinto1()";
if (read_function_ == nullptr) {
if (ABSL_PREDICT_FALSE(!PyErr_ExceptionMatches(PyExc_AttributeError))) {
return FailOperation(read_function_name_);
}
PyErr_Clear();
static constexpr Identifier id_readinto("readinto");
read_function_.reset(PyObject_GetAttr(src_.get(), id_readinto.get()));
read_function_name_ = "readinto()";
if (read_function_ == nullptr) {
if (ABSL_PREDICT_FALSE(!PyErr_ExceptionMatches(PyExc_AttributeError))) {
return FailOperation(read_function_name_);
}
PyErr_Clear();
use_bytes_ = true;
static constexpr Identifier id_read1("read1");
read_function_.reset(PyObject_GetAttr(src_.get(), id_read1.get()));
read_function_name_ = "read1()";
if (read_function_ == nullptr) {
if (ABSL_PREDICT_FALSE(
!PyErr_ExceptionMatches(PyExc_AttributeError))) {
return FailOperation(read_function_name_);
}
PyErr_Clear();
static constexpr Identifier id_read("read");
read_function_.reset(PyObject_GetAttr(src_.get(), id_read.get()));
read_function_name_ = "read()";
if (ABSL_PREDICT_FALSE(read_function_ == nullptr)) {
return FailOperation(read_function_name_);
}
}
}
}
}
for (;;) {
if (ABSL_PREDICT_FALSE(limit_pos() ==
std::numeric_limits<Position>::max())) {
return FailOverflow();
}
const size_t length_to_read = UnsignedMin(
max_length, std::numeric_limits<Position>::max() - limit_pos(),
absl::bit_floor(size_t{std::numeric_limits<Py_ssize_t>::max()}));
size_t length_read;
if (!use_bytes_) {
PythonPtr read_result;
{
// Prefer using `readinto1()` or `readinto()` to avoid copying memory.
MemoryView memory_view;
PyObject* const memory_view_object =
memory_view.MutableToPython(absl::MakeSpan(dest, length_to_read));
if (ABSL_PREDICT_FALSE(memory_view_object == nullptr)) {
return FailOperation("MemoryView::MutableToPython()");
}
read_result.reset(PyObject_CallFunctionObjArgs(
read_function_.get(), memory_view_object, nullptr));
if (ABSL_PREDICT_FALSE(read_result == nullptr)) {
return FailOperation(read_function_name_);
}
if (ABSL_PREDICT_FALSE(!memory_view.Release())) {
return FailOperation("MemoryView::Release()");
}
}
const std::optional<size_t> length_read_opt =
SizeFromPython(read_result.get());
if (ABSL_PREDICT_FALSE(length_read_opt == std::nullopt)) {
return FailOperation(
absl::StrCat("SizeFromPython() after ", read_function_name_));
}
length_read = *length_read_opt;
if (ABSL_PREDICT_FALSE(length_read == 0)) return false;
if (ABSL_PREDICT_FALSE(length_read > max_length)) {
return Fail(absl::InternalError(
absl::StrCat(read_function_name_, " read more than requested")));
}
} else {
const PythonPtr length(SizeToPython(length_to_read));
if (ABSL_PREDICT_FALSE(length == nullptr)) {
return FailOperation("SizeToPython()");
}
const PythonPtr read_result(PyObject_CallFunctionObjArgs(
read_function_.get(), length.get(), nullptr));
if (ABSL_PREDICT_FALSE(read_result == nullptr)) {
return FailOperation(read_function_name_);
}
Py_buffer buffer;
if (ABSL_PREDICT_FALSE(PyObject_GetBuffer(read_result.get(), &buffer,
PyBUF_CONTIG_RO) < 0)) {
return FailOperation(
absl::StrCat("PyObject_GetBuffer() after ", read_function_name_));
}
if (ABSL_PREDICT_FALSE(buffer.len == 0)) {
PyBuffer_Release(&buffer);
return false;
}
if (ABSL_PREDICT_FALSE(IntCast<size_t>(buffer.len) > max_length)) {
PyBuffer_Release(&buffer);
return Fail(absl::InternalError(
absl::StrCat(read_function_name_, " read more than requested")));
}
std::memcpy(dest, buffer.buf, IntCast<size_t>(buffer.len));
length_read = IntCast<size_t>(buffer.len);
PyBuffer_Release(&buffer);
}
move_limit_pos(length_read);
if (length_read >= min_length) return true;
dest += length_read;
min_length -= length_read;
max_length -= length_read;
}
}
bool PythonReader::SeekBehindBuffer(Position new_pos) {
RIEGELI_ASSERT(new_pos < start_pos() || new_pos > limit_pos())
<< "Failed precondition of BufferedReader::SeekBehindBuffer(): "
"position in the buffer, use Seek() instead";
RIEGELI_ASSERT_EQ(start_to_limit(), 0u)
<< "Failed precondition of BufferedReader::SeekBehindBuffer(): "
"buffer not empty";
if (ABSL_PREDICT_FALSE(!PythonReader::SupportsRandomAccess())) {
if (ABSL_PREDICT_FALSE(new_pos < start_pos())) {
if (ok()) Fail(random_access_status_);
return false;
}
return BufferedReader::SeekBehindBuffer(new_pos);
}
if (ABSL_PREDICT_FALSE(!ok())) return false;
PythonLock lock;
if (new_pos > limit_pos()) {
// Seeking forwards.
const std::optional<Position> size = SizeInternal();
if (ABSL_PREDICT_FALSE(size == std::nullopt)) return false;
if (ABSL_PREDICT_FALSE(new_pos > *size)) {
// File ends.
set_limit_pos(*size);
return false;
}
}
set_limit_pos(new_pos);
const PythonPtr file_pos = PositionToPython(limit_pos());
if (ABSL_PREDICT_FALSE(file_pos == nullptr)) {
return FailOperation("PositionToPython()");
}
static constexpr Identifier id_seek("seek");
const PythonPtr seek_result(PyObject_CallMethodObjArgs(
src_.get(), id_seek.get(), file_pos.get(), nullptr));
if (ABSL_PREDICT_FALSE(seek_result == nullptr)) {
return FailOperation("seek()");
}
return true;
}
inline std::optional<Position> PythonReader::SizeInternal() {
RIEGELI_ASSERT_OK(*this)
<< "Failed precondition of PythonReader::SizeInternal()";
RIEGELI_ASSERT(PythonReader::SupportsRandomAccess())
<< "Failed precondition of PythonReader::SizeInternal(): "
"random access not supported";
PythonLock::AssertHeld();
absl::string_view operation;
const PythonPtr file_pos = PositionToPython(0);
if (ABSL_PREDICT_FALSE(file_pos == nullptr)) {
FailOperation("PositionToPython()");
return std::nullopt;
}
const PythonPtr whence = IntToPython(2); // `io.SEEK_END`
if (ABSL_PREDICT_FALSE(whence == nullptr)) {
FailOperation("IntToPython()");
return std::nullopt;
}
static constexpr Identifier id_seek("seek");
PythonPtr result(PyObject_CallMethodObjArgs(
src_.get(), id_seek.get(), file_pos.get(), whence.get(), nullptr));
if (result.get() == Py_None) {
// Python2 `file.seek()` returns `None`, so `tell()` is needed to get the
// new position. Python2 is dead, but some classes still behave like that.
static constexpr Identifier id_tell("tell");
result.reset(
PyObject_CallMethodObjArgs(src_.get(), id_tell.get(), nullptr));
operation = "tell()";
} else {
// `io.IOBase.seek()` returns the new position.
operation = "seek()";
}
if (ABSL_PREDICT_FALSE(result == nullptr)) {
FailOperation(operation);
return std::nullopt;
}
const std::optional<Position> size = PositionFromPython(result.get());
if (ABSL_PREDICT_FALSE(size == std::nullopt)) {
FailOperation(absl::StrCat("PositionFromPython() after ", operation));
return std::nullopt;
}
return *size;
}
std::optional<Position> PythonReader::SizeImpl() {
if (ABSL_PREDICT_FALSE(!PythonReader::SupportsRandomAccess())) {
if (ok()) Fail(random_access_status_);
return std::nullopt;
}
if (ABSL_PREDICT_FALSE(!ok())) return std::nullopt;
PythonLock lock;
const std::optional<Position> size = SizeInternal();
if (ABSL_PREDICT_FALSE(size == std::nullopt)) return std::nullopt;
const PythonPtr file_pos = PositionToPython(limit_pos());
if (ABSL_PREDICT_FALSE(file_pos == nullptr)) {
FailOperation("PositionToPython()");
return std::nullopt;
}
static constexpr Identifier id_seek("seek");
const PythonPtr seek_result(PyObject_CallMethodObjArgs(
src_.get(), id_seek.get(), file_pos.get(), nullptr));
if (ABSL_PREDICT_FALSE(seek_result == nullptr)) {
FailOperation("seek()");
return std::nullopt;
}
return *size;
}
} // namespace riegeli::python
================================================
FILE: python/riegeli/bytes/python_reader.h
================================================
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef PYTHON_RIEGELI_BYTES_PYTHON_READER_H_
#define PYTHON_RIEGELI_BYTES_PYTHON_READER_H_
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#include <Python.h>
// clang-format: do not reorder the above include.
#include <stddef.h>
#include <optional>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
#include "python/riegeli/base/utils.h"
#include "riegeli/base/object.h"
#include "riegeli/base/types.h"
#include "riegeli/bytes/buffer_options.h"
#include "riegeli/bytes/buffered_reader.h"
namespace riegeli::python {
// A `Reader` which reads from a Python binary I/O stream.
//
// The stream must support:
// * `close()` - for `Close()` if `Options::owns_src()`
// * `readinto1(memoryview)` or
// `readinto(memoryview)` or
// `read1(int)` or
// `read(int)`
// * `seekable()`
// * `seek(int[, int])` - for `Seek()` or `Size()`
// * `tell()` - for `Seek()` or `Size()`
//
// `PythonReader` supports random access if
// `Options::assumed_pos() == std::nullopt` and the stream supports random
// access (this is checked by calling `seekable()`).
//
// Warning: if random access is not supported and the stream is not owned,
// it will have an unpredictable amount of extra data consumed because of
// buffering.
class PythonReader : public BufferedReader {
public:
class Options : public BufferOptionsBase<Options> {
public:
Options() noexcept {}
// If `true`, `PythonReader::Close()` closes the stream.
//
// Default: `false`.
Options& set_owns_src(bool owns_src) & ABSL_ATTRIBUTE_LIFETIME_BOUND {
owns_src_ = owns_src;
return *this;
}
Options&& set_owns_src(bool owns_src) && ABSL_ATTRIBUTE_LIFETIME_BOUND {
return std::move(set_owns_src(owns_src));
}
bool owns_src() const { return owns_src_; }
// If `std::nullopt`, the current position reported by `pos()` corresponds
// to the current stream position if possible, otherwise 0 is assumed as the
// initial position. Random access is supported if the stream supports
// random access.
//
// If not `std::nullopt`, this position is assumed initially, to be reported
// by `pos()`. It does not need to correspond to the current stream
// position. Random access is not supported.
//
// Default: `std::nullopt`.
Options& set_assumed_pos(std::optional<Position> assumed_pos) &
ABSL_ATTRIBUTE_LIFETIME_BOUND {
assumed_pos_ = assumed_pos;
return *this;
}
Options&& set_assumed_pos(std::optional<Position> assumed_pos) &&
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return std::move(set_assumed_pos(assumed_pos));
}
std::optional<Position> assumed_pos() const { return assumed_pos_; }
private:
bool owns_src_ = false;
std::optional<Position> assumed_pos_;
};
// Creates a closed `PythonReader`.
explicit PythonReader(Closed) noexcept : BufferedReader(kClosed) {}
// Will read from `src`.
explicit PythonReader(PyObject* src, Options options = Options());
PythonReader(PythonReader&& that) noexcept;
PythonReader& operator=(PythonReader&& that) noexcept;
// Returns a borrowed reference to the stream being read from.
PyObject* src() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return src_.get(); }
const Exception& exception() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return exception_;
}
bool ToleratesReadingAhead() override {
return BufferedReader::ToleratesReadingAhead() ||
PythonReader::SupportsRandomAccess();
}
bool SupportsRandomAccess() override { return supports_random_access_; }
// For implementing `tp_traverse` of objects containing `PythonReader`.
int Traverse(visitproc visit, void* arg);
protected:
void Done() override;
bool ReadInternal(size_t min_length, size_t max_length, char* dest) override;
bool SeekBehindBuffer(Position new_pos) override;
std::optional<Position> SizeImpl() override;
private:
ABSL_ATTRIBUTE_COLD bool FailOperation(absl::string_view operation);
std::optional<Position> SizeInternal();
PythonPtrLocking src_;
bool owns_src_ = false;
bool supports_random_access_ = false;
absl::Status random_access_status_;
Exception exception_;
PythonPtrLocking read_function_;
absl::string_view read_function_name_;
bool use_bytes_ = false;
};
inline PythonReader::PythonReader(PythonReader&& that) noexcept
: BufferedReader(static_cast<BufferedReader&&>(that)),
src_(std::move(that.src_)),
owns_src_(that.owns_src_),
supports_random_access_(
std::exchange(that.supports_random_access_, false)),
random_access_status_(std::move(that.random_access_status_)),
exception_(std::move(that.exception_)),
read_function_(std::move(that.read_function_)),
read_function_name_(that.read_function_name_),
use_bytes_(that.use_bytes_) {}
inline PythonReader& PythonReader::operator=(PythonReader&& that) noexcept {
BufferedReader::operator=(static_cast<BufferedReader&&>(that));
src_ = std::move(that.src_);
owns_src_ = that.owns_src_;
supports_random_access_ = std::exchange(that.supports_random_access_, false);
random_access_status_ = std::move(that.random_access_status_);
exception_ = std::move(that.exception_);
read_function_ = std::move(that.read_function_);
read_function_name_ = that.read_function_name_;
use_bytes_ = that.use_bytes_;
return *this;
}
inline int PythonReader::Traverse(visitproc visit, void* arg) {
Py_VISIT(src_.get());
Py_VISIT(read_function_.get());
return exception_.Traverse(visit, arg);
}
} // namespace riegeli::python
#endif // PYTHON_RIEGELI_BYTES_PYTHON_READER_H_
================================================
FILE: python/riegeli/bytes/python_writer.cc
================================================
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#define PY_SSIZE_T_CLEAN
#include <Python.h>
// clang-format: do not reorder the above include.
#include "python/riegeli/bytes/python_writer.h"
// clang-format: do not reorder the above include.
#include <stddef.h>
#include <limits>
#include <optional>
#include "absl/base/attributes.h"
#include "absl/base/optimization.h"
#include "absl/numeric/bits.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "python/riegeli/base/utils.h"
#include "riegeli/base/arithmetic.h"
#include "riegeli/base/assert.h"
#include "riegeli/base/global.h"
#include "riegeli/base/types.h"
#include "riegeli/bytes/buffered_writer.h"
namespace riegeli::python {
PythonWriter::PythonWriter(PyObject* dest, Options options)
: BufferedWriter(options.buffer_options()),
owns_dest_(options.owns_dest()) {
PythonLock::AssertHeld();
Py_INCREF(dest);
dest_.reset(dest);
if (options.assumed_pos() != std::nullopt) {
set_start_pos(*options.assumed_pos());
// `supports_random_access_` is left as `false`.
random_access_status_ = Global([] {
return absl::UnimplementedError(
"PythonWriter::Options::assumed_pos() excludes random access");
});
} else {
static constexpr Identifier id_seekable("seekable");
const PythonPtr seekable_result(
PyObject_CallMethodObjArgs(dest_.get(), id_seekable.get(), nullptr));
if (ABSL_PREDICT_FALSE(seekable_result == nullptr)) {
FailOperation("seekable()");
return;
}
const int seekable_is_true = PyObject_IsTrue(seekable_result.get());
if (ABSL_PREDICT_FALSE(seekable_is_true < 0)) {
FailOperation("PyObject_IsTrue() after seekable()");
return;
}
if (seekable_is_true == 0) {
// Random access is not supported. Assume 0 as the initial position.
// `supports_random_access_` is left as `false`.
random_access_status_ = Global([] {
return absl::UnimplementedError(
"seekable() is False which excludes random access");
});
return;
}
static constexpr Identifier id_tell("tell");
const PythonPtr tell_result(
PyObject_CallMethodObjArgs(dest_.get(), id_tell.get(), nullptr));
if (ABSL_PREDICT_FALSE(tell_result == nullptr)) {
FailOperation("tell()");
return;
}
const std::optional<Position> file_pos =
PositionFromPython(tell_result.get());
if (ABSL_PREDICT_FALSE(file_pos == std::nullopt)) {
FailOperation("PositionFromPython() after tell()");
return;
}
set_start_pos(*file_pos);
supports_random_access_ = true;
}
BeginRun();
}
void PythonWriter::Done() {
BufferedWriter::Done();
random_access_status_ = absl::OkStatus();
if (owns_dest_ && dest_ != nullptr) {
PythonLock lock;
static constexpr Identifier id_close("close");
const PythonPtr close_result(
PyObject_CallMethodObjArgs(dest_.get(), id_close.get(), nullptr));
if (ABSL_PREDICT_FALSE(close_result == nullptr)) FailOperation("close()");
}
}
inline bool PythonWriter::FailOperation(absl::string_view operation) {
RIEGELI_ASSERT(is_open())
<< "Failed precondition of PythonWriter::FailOperation(): "
"Object closed";
PythonLock::AssertHeld();
if (ABSL_PREDICT_FALSE(!ok())) {
// Ignore this error because `PythonWriter` already failed.
PyErr_Clear();
return false;
}
exception_ = Exception::Fetch();
return Fail(absl::UnknownError(
absl::StrCat(operation, " failed: ", exception_.message())));
}
bool PythonWriter::WriteInternal(absl::string_view src) {
RIEGELI_ASSERT(!src.empty())
<< "Failed precondition of BufferedWriter::WriteInternal(): "
"nothing to write";
RIEGELI_ASSERT_OK(*this)
<< "Failed precondition of BufferedWriter::WriteInternal()";
if (ABSL_PREDICT_FALSE(src.size() >
std::numeric_limits<Position>::max() - start_pos())) {
return FailOverflow();
}
PythonLock lock;
if (ABSL_PREDICT_FALSE(write_function_ == nullptr)) {
static constexpr Identifier id_write("write");
write_function_.reset(PyObject_GetAttr(dest_.get(), id_write.get()));
if (ABSL_PREDICT_FALSE(write_function_ == nullptr)) {
return FailOperation("write()");
}
}
do {
const size_t length_to_write = UnsignedMin(
src.size(),
absl::bit_floor(size_t{std::numeric_limits<Py_ssize_t>::max()}));
size_t length_written;
{
PythonPtr write_result;
if (!use_bytes_) {
// Prefer passing a `memoryview` to avoid copying memory.
MemoryView memory_view;
PyObject* const memory_view_object = memory_view.ToPython(
absl::string_view(src.data(), length_to_write));
if (ABSL_PREDICT_FALSE(memory_view_object == nullptr)) {
return FailOperation("MemoryView::ToPython()");
}
write_result.reset(PyObject_CallFunctionObjArgs(
write_function_.get(), memory_view_object, nullptr));
if (ABSL_PREDICT_FALSE(write_result == nullptr)) {
if (!PyErr_ExceptionMatches(PyExc_TypeError)) {
return FailOperation("write()");
}
PyErr_Clear();
use_bytes_ = true;
}
if (ABSL_PREDICT_FALSE(!memory_view.Release())) {
return FailOperation("MemoryView::Release()");
}
}
if (use_bytes_) {
// `write()` does not support `memoryview`. Use `bytes`.
const PythonPtr bytes = BytesToPython(src.substr(0, length_to_write));
if (ABSL_PREDICT_FALSE(bytes == nullptr)) {
return FailOperation("BytesToPython()");
}
write_result.reset(PyObject_CallFunctionObjArgs(write_function_.get(),
bytes.get(), nullptr));
if (ABSL_PREDICT_FALSE(write_result == nullptr)) {
return FailOperation("write()");
}
}
if (write_result.get() == Py_None) {
// Python2 `file.write()` returns `None`, and would raise an exception
// if less than the full length had been written. Python2 is dead, but
// some classes still behave like that.
length_written = length_to_write;
} else {
// `io.IOBase.write()` returns the length written.
const std::optional<size_t> length_written_opt =
SizeFromPython(write_result.get());
if (ABSL_PREDICT_FALSE(length_written_opt == std::nullopt)) {
return FailOperation("SizeFromPython() after write()");
}
length_written = *length_written_opt;
}
}
if (ABSL_PREDICT_FALSE(length_written > length_to_write)) {
return Fail(absl::InternalError("write() wrote more than requested"));
}
move_start_pos(length_written);
src.remove_prefix(length_written);
} while (!src.empty());
return true;
}
bool PythonWriter::FlushImpl(FlushType flush_type) {
if (ABSL_PREDICT_FALSE(!BufferedWriter::FlushImpl(flush_type))) return false;
switch (flush_type) {
case FlushType::kFromObject:
if (!owns_dest_) return true;
ABSL_FALLTHROUGH_INTENDED;
case FlushType::kFromProcess:
case FlushType::kFromMachine:
PythonLock lock;
static constexpr Identifier id_flush("flush");
const PythonPtr flush_result(
PyObject_CallMethodObjArgs(dest_.get(), id_flush.get(), nullptr));
if (ABSL_PREDICT_FALSE(flush_result == nullptr)) {
return FailOperation("flush()");
}
return true;
}
RIEGELI_ASSUME_UNREACHABLE()
<< "Unknown flush type: " << static_cast<int>(flush_type);
}
bool PythonWriter::SeekBehindBuffer(Position new_pos) {
RIEGELI_ASSERT_NE(new_pos, pos())
<< "Failed precondition of BufferedWriter::SeekBehindBuffer(): "
"position unchanged, use Seek() instead";
RIEGELI_ASSERT_EQ(start_to_limit(), 0u)
<< "Failed precondition of BufferedWriter::SeekBehindBuffer(): "
"buffer not empty";
if (ABSL_PREDICT_FALSE(!PythonWriter::SupportsRandomAccess())) {
if (ok()) Fail(random_access_status_);
return false;
}
PythonLock lock;
if (new_pos > start_pos()) {
// Seeking forwards.
const std::optional<Position> size = SizeInternal();
if (ABSL_PREDICT_FALSE(size == std::nullopt)) return false;
if (ABSL_PREDICT_FALSE(new_pos > *size)) {
// File ends.
set_start_pos(*size);
return false;
}
}
set_start_pos(new_pos);
const PythonPtr file_pos = PositionToPython(start_pos());
if (ABSL_PREDICT_FALSE(file_pos == nullptr)) {
return FailOperation("PositionToPython()");
}
static constexpr Identifier id_seek("seek");
const PythonPtr seek_result(PyObject_CallMethodObjArgs(
dest_.get(), id_seek.get(), file_pos.get(), nullptr));
if (ABSL_PREDICT_FALSE(seek_result == nullptr)) {
return FailOperation("seek()");
}
return true;
}
inline std::optional<Position> PythonWriter::SizeInternal() {
RIEGELI_ASSERT_OK(*this)
<< "Failed precondition of PythonWriter::SizeInternal()";
RIEGELI_ASSERT(PythonWriter::SupportsRandomAccess())
<< "Failed precondition of PythonWriter::SizeInternal(): "
"random access not supported";
RIEGELI_ASSERT_EQ(start_to_limit(), 0u)
<< "Failed precondition of PythonWriter::SizeInternal(): "
"buffer not empty";
PythonLock::AssertHeld();
absl::string_view operation;
const PythonPtr file_pos = PositionToPython(0);
if (ABSL_PREDICT_FALSE(file_pos == nullptr)) {
FailOperation("PositionToPython()");
return std::nullopt;
}
const PythonPtr whence = IntToPython(2); // `io.SEEK_END`
if (ABSL_PREDICT_FALSE(whence == nullptr)) {
FailOperation("IntToPython()");
return std::nullopt;
}
static constexpr Identifier id_seek("seek");
PythonPtr result(PyObject_CallMethodObjArgs(
dest_.get(), id_seek.get(), file_pos.get(), whence.get(), nullptr));
if (result.get() == Py_None) {
// Python2 `file.seek()` returns `None`. Python2 is dead, but some classes
// still behave like that.
static constexpr Identifier id_tell("tell");
result.reset(
PyObject_CallMethodObjArgs(dest_.get(), id_tell.get(), nullptr));
operation = "tell()";
} else {
// `io.IOBase.seek()` returns the new position.
operation = "seek()";
}
if (ABSL_PREDICT_FALSE(result == nullptr)) {
FailOperation(operation);
return std::nullopt;
}
const std::optional<Position> size = PositionFromPython(result.get());
if (ABSL_PREDICT_FALSE(size == std::nullopt)) {
FailOperation(absl::StrCat("PositionFromPython() after ", operation));
return std::nullopt;
}
return *size;
}
std::optional<Position> PythonWriter::SizeBehindBuffer() {
RIEGELI_ASSERT_EQ(start_to_limit(), 0u)
<< "Failed precondition of BufferedWriter::SizeBehindBuffer(): "
"buffer not empty";
if (ABSL_PREDICT_FALSE(!PythonWriter::SupportsRandomAccess())) {
if (ok()) Fail(random_access_status_);
return std::nullopt;
}
if (ABSL_PREDICT_FALSE(!ok())) return std::nullopt;
PythonLock lock;
const std::optional<Position> size = SizeInternal();
if (ABSL_PREDICT_FALSE(size == std::nullopt)) return std::nullopt;
const PythonPtr file_pos = PositionToPython(start_pos());
if (ABSL_PREDICT_FALSE(file_pos == nullptr)) {
FailOperation("PositionToPython()");
return std::nullopt;
}
static constexpr Identifier id_seek("seek");
const PythonPtr seek_result(PyObject_CallMethodObjArgs(
dest_.get(), id_seek.get(), file_pos.get(), nullptr));
if (ABSL_PREDICT_FALSE(seek_result == nullptr)) {
FailOperation("seek()");
return std::nullopt;
}
return *size;
}
bool PythonWriter::TruncateBehindBuffer(Position new_size) {
RIEGELI_ASSERT_EQ(start_to_limit(), 0u)
<< "Failed precondition of BufferedWriter::TruncateBehindBuffer(): "
"buffer not empty";
if (ABSL_PREDICT_FALSE(!PythonWriter::SupportsRandomAccess())) {
if (ok()) Fail(random_access_status_);
return false;
}
if (ABSL_PREDICT_FALSE(!ok())) return false;
PythonLock lock;
const std::optional<Position> size = SizeInternal();
if (ABSL_PREDICT_FALSE(size == std::nullopt)) return false;
if (ABSL_PREDICT_FALSE(new_size > *size)) {
// File ends.
set_start_pos(*size);
return false;
}
{
const PythonPtr file_pos = PositionToPython(new_size);
if (ABSL_PREDICT_FALSE(file_pos == nullptr)) {
return FailOperation("PositionToPython()");
}
static constexpr Identifier id_seek("seek");
const PythonPtr seek_result(PyObject_CallMethodObjArgs(
dest_.get(), id_seek.get(), file_pos.get(), nullptr));
if (ABSL_PREDICT_FALSE(seek_result == nullptr)) {
return FailOperation("seek()");
}
}
set_start_pos(new_size);
static constexpr Identifier id_truncate("truncate");
const PythonPtr truncate_result(
PyObject_CallMethodObjArgs(dest_.get(), id_truncate.get(), nullptr));
if (ABSL_PREDICT_FALSE(truncate_result == nullptr)) {
return FailOperation("truncate()");
}
return true;
}
} // namespace riegeli::python
================================================
FILE: python/riegeli/bytes/python_writer.h
================================================
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef PYTHON_RIEGELI_BYTES_PYTHON_WRITER_H_
#define PYTHON_RIEGELI_BYTES_PYTHON_WRITER_H_
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#include <Python.h>
// clang-format: do not reorder the above include.
#include <optional>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
#include "python/riegeli/base/utils.h"
#include "riegeli/base/object.h"
#include "riegeli/base/types.h"
#include "riegeli/bytes/buffer_options.h"
#include "riegeli/bytes/buffered_writer.h"
namespace riegeli::python {
// A `Writer` which writes to a Python binary I/O stream.
//
// The stream must support:
// * `close()` - for `Close()` if `Options::owns_dest()`
// * `write(bytes)`
// * `flush()` - for `Flush()`
// * `seekable()`
// * `seek(int[, int])` - for `Seek()`, `Size()`, or `Truncate()`
// * `tell()` - for `Seek()`, `Size()`, or `Truncate()`
// * `truncate()` - for `Truncate()`
//
// `PythonWriter` supports random access if
// `Options::assumed_pos() == std::nullopt` and the stream supports random
// access (this is checked by calling `seekable()`).
class PythonWriter : public BufferedWriter {
public:
class Options : public BufferOptionsBase<Options> {
public:
Options() noexcept {}
// If `true`, `PythonWriter::Close()` closes the stream, and
// `PythonWriter::Flush(flush_type)` flushes the stream even if `flush_type`
// is `FlushType::kFromObject`.
//
// Default: `false`.
Options& set_owns_dest(bool owns_dest) & ABSL_ATTRIBUTE_LIFETIME_BOUND {
owns_dest_ = owns_dest;
return *this;
}
Options&& set_owns_dest(bool owns_dest) && ABSL_ATTRIBUTE_LIFETIME_BOUND {
return std::move(set_owns_dest(owns_dest));
}
bool owns_dest() const { return owns_dest_; }
// If `std::nullopt`, the current position reported by `pos()` corresponds
// to the current stream position if possible, otherwise 0 is assumed as the
// initial position. Random access is supported if the stream supports
// random access.
//
// If not `std::nullopt`, this position is assumed initially, to be reported
// by `pos()`. It does not need to correspond to the current stream
// position. Random access is not supported.
//
// Default: `std::nullopt`.
Options& set_assumed_pos(std::optional<Position> assumed_pos) &
ABSL_ATTRIBUTE_LIFETIME_BOUND {
assumed_pos_ = assumed_pos;
return *this;
}
Options&& set_assumed_pos(std::optional<Position> assumed_pos) &&
ABSL_ATTRIBUTE_LIFETIME_BOUND {
return std::move(set_assumed_pos(assumed_pos));
}
std::optional<Position> assumed_pos() const { return assumed_pos_; }
private:
bool owns_dest_ = false;
std::optional<Position> assumed_pos_;
};
// Creates a closed `PythonWriter`.
explicit PythonWriter(Closed) noexcept : BufferedWriter(kClosed) {}
// Will write to `dest`.
explicit PythonWriter(PyObject* dest, Options options = Options());
PythonWriter(PythonWriter&& that) noexcept;
PythonWriter& operator=(PythonWriter&& that) noexcept;
// Returns a borrowed reference to the stream being written to.
PyObject* dest() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return dest_.get(); }
const Exception& exception() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
return exception_;
}
bool SupportsRandomAccess() override { return supports_random_access_; }
// For implementing `tp_traverse` of objects containing `PythonWriter`.
int Traverse(visitproc visit, void* arg);
protected:
void Done() override;
bool WriteInternal(absl::string_view src) override;
bool FlushImpl(FlushType flush_type) override;
bool SeekBehindBuffer(Position new_pos) override;
std::optional<Position> SizeBehindBuffer() override;
bool TruncateBehindBuffer(Position new_size) override;
private:
ABSL_ATTRIBUTE_COLD bool FailOperation(absl::string_view operation);
std::optional<Position> SizeInternal();
PythonPtrLocking dest_;
bool owns_dest_ = false;
bool supports_random_access_ = false;
absl::Status random_access_status_;
Exception exception_;
PythonPtrLocking write_function_;
bool use_bytes_ = false;
};
inline PythonWriter::PythonWriter(PythonWriter&& that) noexcept
: BufferedWriter(static_cast<BufferedWriter&&>(that)),
dest_(std::move(that.dest_)),
owns_dest_(that.owns_dest_),
supports_random_access_(
std::exchange(that.supports_random_access_, false)),
random_access_status_(std::move(that.random_access_status_)),
exception_(std::move(that.exception_)),
write_function_(std::move(that.write_function_)),
use_bytes_(that.use_bytes_) {}
inline PythonWriter& PythonWriter::operator=(PythonWriter&& that) noexcept {
BufferedWriter::operator=(static_cast<BufferedWriter&&>(that));
dest_ = std::move(that.dest_);
owns_dest_ = that.owns_dest_;
supports_random_access_ = std::exchange(that.supports_random_access_, false);
random_access_status_ = std::move(that.random_access_status_);
exception_ = std::move(that.exception_);
write_function_ = std::move(that.write_function_);
use_bytes_ = that.use_bytes_;
return *this;
}
inline int PythonWriter::Traverse(visitproc visit, void* arg) {
Py_VISIT(dest_.get());
Py_VISIT(write_function_.get());
return exception_.Traverse(visit, arg);
}
} // namespace riegeli::python
#endif // PYTHON_RIEGELI_BYTES_PYTHON_WRITER_H_
================================================
FILE: python/riegeli/py_extension.bzl
================================================
"""Supports writing Python modules in C++."""
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
load("@rules_python//python:defs.bzl", "py_library")
def py_extension(
name = None,
srcs = None,
hdrs = None,
data = None,
features = None,
visibility = None,
deps = None):
"""Creates a Python module implemented in C++.
Python modules can depend on a py_extension. Other py_extensions can depend
on a generated C++ library named with "_cc" suffix.
Args:
name: Name for this target.
srcs: C++ source files.
hdrs: C++ header files, for other py_extensions which depend on this.
data: Files needed at runtime. This may include Python libraries.
features: Passed to cc_library.
visibility: Controls which rules can depend on this.
deps: Other C++ libraries that this library depends upon.
"""
cc_library_name = name + "_cc"
cc_binary_name = name + ".so"
cc_library(
name = cc_library_name,
srcs = srcs,
hdrs = hdrs,
data = data,
features = features,
visibility = visibility,
deps = deps,
alwayslink = True,
)
cc_binary(
name = cc_binary_name,
linkshared = True,
linkstatic = True,
visibility = ["//visibility:private"],
deps = [cc_library_name],
)
py_library(
name = name,
data = [cc_binary_name],
visibility = visibility,
)
================================================
FILE: python/riegeli/python_configure.bzl
================================================
"""Repository rule for Python autoconfiguration.
`python_configure` depends on the following environment variables:
* `PYTHON_BIN_PATH`: location of python binary.
* `PYTHON_LIB_PATH`: Location of python libraries.
"""
_BAZEL_SH = "BAZEL_SH"
_PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
_PYTHON_LIB_PATH = "PYTHON_LIB_PATH"
_TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO"
def _tpl(repository_ctx, tpl, substitutions = {}, out = None):
if not out:
out = tpl
repository_ctx.template(
out,
Label("//python/riegeli:{}.tpl".format(tpl)),
substitutions,
)
def _fail(msg):
"""Outputs failure message when auto configuration fails."""
red = "\033[0;31m"
no_color = "\033[0m"
fail("{}Python Configuration Error:{} {}\n".format(red, no_color, msg))
def _is_windows(repository_ctx):
"""Returns true if the host operating system is Windows."""
os_name = repository_ctx.os.name.lower()
return "windows" in os_name
def _execute(
repository_ctx,
cmdline,
error_msg = None,
error_details = None,
empty_stdout_fine = False):
"""Executes an arbitrary shell command.
Args:
repository_ctx: the repository_ctx object
cmdline: list of strings, the command to execute
error_msg: string, a summary of the error if the command fails
error_details: string, details about the error or steps to fix it
empty_stdout_fine: bool, if True, an empty stdout result is fine,
otherwise it's an error
Return:
the result of repository_ctx.execute(cmdline)
"""
result = repository_ctx.execute(cmdline)
if result.stderr or not (empty_stdout_fine or result.stdout):
_fail("\n".join([
error_msg.strip() if error_msg else "Repository command failed",
result.stderr.strip(),
error_details if error_details else "",
]))
return result
def _read_dir(repository_ctx, src_dir):
"""Returns a string with all files in a directory.
Finds all files inside a directory, traversing subfolders and following
symlinks. The returned string contains the full path of all files
separated by line breaks.
"""
if _is_windows(repository_ctx):
src_dir = src_dir.replace("/", "\\")
find_result = _execute(
repository_ctx,
["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"],
empty_stdout_fine = True,
)
# src_files will be used in genrule.outs where the paths must
# use forward slashes.
result = find_result.stdout.replace("\\", "/")
else:
find_result = _execute(
repository_ctx,
["find", src_dir, "-follow", "-type", "f"],
empty_stdout_fine = True,
)
result = find_result.stdout
return result
def _genrule(src_dir, genrule_name, command, outs):
"""Returns a string with a genrule.
Genrule executes the given command and produces the given outputs.
"""
return (
"genrule(\n" +
' name = "{}",\n' +
" outs = [\n" +
"{}\n" +
" ],\n" +
' cmd = """\n' +
"{}\n" +
' """,\n' +
")\n"
).format(genrule_name, outs, command)
def _norm_path(path):
"""Returns a path with '/' and removes the trailing slash."""
return path.replace("\\", "/").rstrip("/")
def _symlink_genrule_for_dir(
repository_ctx,
src_dir,
dest_dir,
genrule_name,
src_files = [],
dest_files = []):
"""Returns a genrule to symlink (or copy if on Windows) a set of files.
If src_dir is passed, files will be read from the given directory; otherwise
we assume files are in src_files and dest_files
"""
if src_dir != None:
src_dir = _norm_path(src_dir)
dest_dir = _norm_path(dest_dir)
files = "\n".join(
sorted(_read_dir(repository_ctx, src_dir).splitlines()),
)
# Create a list with the src_dir stripped to use for outputs.
dest_files = files.replace(src_dir, "").splitlines()
src_files = files.splitlines()
command = []
outs = []
for i in range(len(dest_files)):
if dest_files[i] != "":
# If we have only one file to link we do not want to use the
# dest_dir, as $(@D) will include the full path to the file.
dest = "$(@D)/{}{}".format(
dest_dir if len(dest_files) != 1 else "",
dest_files[i],
)
# Copy the headers to create a sandboxable setup.
cmd = "cp -f"
command.append('{} "{}" "{}"'.format(cmd, src_files[i], dest))
outs.append(' "{}{}",'.format(dest_dir, dest_files[i]))
genrule = _genrule(
src_dir,
genrule_name,
" && ".join(command),
"\n".join(outs),
)
return genrule
def _get_python_bin(repository_ctx):
"""Gets the python bin path."""
python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH)
if python_bin != None:
return python_bin
python_bin_path = repository_ctx.which("python")
if python_bin_path != None:
return str(python_bin_path)
_fail(("Cannot find python in PATH, please make sure " +
"python is installed and add its directory in PATH, " +
"or --define {}='/something/else'.\nPATH={}").format(
_PYTHON_BIN_PATH,
repository_ctx.os.environ.get("PATH", ""),
))
def _get_bash_bin(repository_ctx):
"""Gets the bash bin path."""
bash_bin = repository_ctx.os.environ.get(_BAZEL_SH)
if bash_bin != None:
return bash_bin
bash_bin_path = repository_ctx.which("bash")
if bash_bin_path != None:
return str(bash_bin_path)
_fail(("Cannot find bash in PATH, please make sure " +
"bash is installed and add its directory in PATH, " +
"or --define {}='/path/to/bash'.\nPATH={}").format(
_BAZEL_SH,
repository_ctx.os.environ.get("PATH", ""),
))
def _get_python_runtime_pair(repository_ctx, python_bin):
"""Builds a py_runtime_pair definition."""
return (
"py_runtime_pair(\n" +
' name = "py_runtime_pair",\n' +
" py2_runtime = None,\n" +
" py3_runtime = \":py3_runtime\",\n" +
")\n" +
"\n" +
"py_runtime(\n" +
' name = "py3_runtime",\n' +
' interpreter_path = "{}",\n' +
' python_version = "PY3",\n' +
")\n"
).format(python_bin)
def _get_python_lib(repository_ctx, python_bin):
"""Gets the python lib path."""
python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH)
if python_lib != None:
return python_lib
print_lib = ("<<END\n" +
"import site\n" +
"import os\n" +
"\n" +
"try:\n" +
" input = raw_input\n" +
"except NameError:\n" +
" pass\n" +
"\n" +
"python_paths = []\n" +
"if os.getenv('PYTHONPATH') is not None:\n" +
" python_paths = os.getenv('PYTHONPATH').split(':')\n" +
"try:\n" +
" library_paths = site.getsitepackages()\n" +
"except AttributeError:\n" +
" from distutils.sysconfig import get_python_lib\n" +
" library_paths = [get_python_lib()]\n" +
"all_paths = set(python_paths + library_paths)\n" +
"paths = []\n" +
"for path in all_paths:\n" +
" if os.path.isdir(path):\n" +
" paths.append(path)\n" +
"if len(paths) >= 1:\n" +
" print(paths[0])\n" +
"END")
cmd = "{} - {}".format(python_bin, print_lib)
result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
return result.stdout.strip("\n")
def _check_python_lib(repository_ctx, python_lib):
"""Checks the python lib path."""
cmd = 'test -d "{}" -a -x "{}"'.format(python_lib, python_lib)
result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
if result.return_code == 1:
_fail("Invalid python library path: {}".format(python_lib))
def _check_python_bin(repository_ctx, python_bin):
"""Checks the python bin path."""
cmd = '[[ -x "{}" ]] && [[ ! -d "{}" ]]'.format(python_bin, python_bin)
result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
if result.return_code == 1:
_fail(("--define {}='{}' is not executable. " +
"Is it the python binary?").format(
_PYTHON_BIN_PATH,
python_bin,
))
def _get_python_include(repository_ctx, python_bin):
"""Gets the python include path."""
result = _execute(
repository_ctx,
[
python_bin,
"-c",
"import importlib; " +
"import importlib.util; " +
"print(importlib.import_module('distutils.sysconfig').get_python_inc() " +
"if importlib.util.find_spec('distutils.sysconfig') " +
"else importlib.import_module('sysconfig').get_path('include'))",
],
error_msg = "Problem getting python include path.",
error_details = ("Is the Python binary path set up right? " +
"(See ./configure or {}.) " +
"Is distutils installed?").format(_PYTHON_BIN_PATH),
)
return result.stdout.splitlines()[0]
def _get_python_import_lib_name(repository_ctx, python_bin):
"""Gets Python import library name (pythonXY.lib) on Windows."""
result = _execute(
repository_ctx,
[
python_bin,
"-c",
"import sys; " +
'print("python{}{}.lib".format(' +
"sys.version_info.major, sys.version_info.minor))",
],
error_msg = "Problem getting python import library.",
error_details = ("Is the Python binary path set up right? " +
"(See ./configure or {}.) ").format(_PYTHON_BIN_PATH),
)
return result.stdout.splitlines()[0]
def _get_numpy_include(repository_ctx, python_bin):
"""Gets the numpy include path."""
return _execute(
repository_ctx,
[
python_bin,
"-c",
"import numpy; print(numpy.get_include())",
],
error_msg = "Problem getting numpy include path.",
error_details = "Is numpy installed?",
).stdout.splitlines()[0]
def _create_local_python_repository(repository_ctx):
"""Creates the repository containing files set up to build with Python."""
python_bin = _get_python_bin(repository_ctx)
_check_python_bin(repository_ctx, python_bin)
python_runtime_pair = _get_python_runtime_pair(repository_ctx, python_bin)
python_lib = _get_python_lib(repository_ctx, python_bin)
_check_python_lib(repository_ctx, python_lib)
python_include = _get_python_include(repository_ctx, python_bin)
numpy_include = _get_numpy_include(repository_ctx, python_bin) + "/numpy"
python_include_rule = _symlink_genrule_for_dir(
repository_ctx,
python_include,
"python_include",
"python_include",
)
python_import_lib_genrule = ""
# To build Python C/C++ extension on Windows, we need to link to python
# import library pythonXY.lib
# See https://docs.python.org/3/extending/windows.html
if _is_windows(repository_ctx):
python_include = _norm_path(python_include)
python_import_lib_name = _get_python_import_lib_name(
repository_ctx,
python_bin,
)
python_import_lib_src = "{}/libs/{}".format(
python_include.rsplit("/", 1)[0],
python_import_lib_name,
)
python_import_lib_genrule = _symlink_genrule_for_dir(
repository_ctx,
None,
"",
"python_import_lib",
[python_import_lib_src],
[python_import_lib_name],
)
numpy_include_rule = _symlink_genrule_for_dir(
repository_ctx,
numpy_include,
"numpy_include/numpy",
"numpy_include",
)
_tpl(repository_ctx, "BUILD", {
"%{PYTHON_RUNTIME_PAIR}": python_runtime_pair,
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
"%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule,
})
def _create_remote_python_repository(repository_ctx, remote_config_repo):
"""Creates pointers to a remotely configured repo set up to build with Python.
"""
repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
def _python_autoconf_impl(repository_ctx):
"""Implementation of the python_autoconf repository rule."""
if _TF_PYTHON_CONFIG_REPO in repository_ctx.os.environ:
_create_remote_python_repository(
repository_ctx,
repository_ctx.os.environ[_TF_PYTHON_CONFIG_REPO],
)
else:
_create_local_python_repository(repository_ctx)
python_configure = repository_rule(
implementation = _python_autoconf_impl,
environ = [
_BAZEL_SH,
_PYTHON_BIN_PATH,
_PYTHON_LIB_PATH,
_TF_PYTHON_CONFIG_REPO,
],
)
"""Detects and configures the local Python.
Add the following to your WORKSPACE FILE:
```python
python_configure(name = "local_config_python")
```
Args:
name: A unique name for this workspace rule.
"""
================================================
FILE: python/riegeli/records/BUILD
================================================
load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
load("@rules_python//python:defs.bzl", "py_library")
load("//python/riegeli:py_extension.bzl", "py_extension")
package(
default_visibility = ["//python/riegeli:__subpackages__"],
features = ["header_modules"],
)
licenses(["notice"])
py_extension(
name = "record_reader",
srcs = ["record_reader.cc"],
# Python modules imported from C++.
data = [
":records_metadata_py_pb2",
":skipped_region",
"@com_google_protobuf//:protobuf_python",
],
# record_reader.cc has #define before #include to influence what the
# included files provide.
features = ["-use_header_modules"],
deps = [
":record_position_cc",
"//python/riegeli/base:utils",
"//python/riegeli/bytes:python_reader",
"//riegeli/base:arithmetic",
"//riegeli/base:assert",
"//riegeli/base:chain",
"//riegeli/base:compare",
"//riegeli/base:types",
"//riegeli/chunk_encoding:field_projection",
"//riegeli/records:record_position",
"//riegeli/records:record_reader",
"//riegeli/records:skipped_region",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:string_view",
"@rules_python//python/cc:current_py_cc_headers",
],
)
py_extension(
name = "record_writer",
srcs = ["record_writer.cc"],
# record_writer.cc has #define before #include to influence what the
# included files provide.
features = ["-use_header_modules"],
deps = [
":record_position_cc",
"//python/riegeli/base:utils",
"//python/riegeli/bytes:python_writer",
"//riegeli/base:assert",
"//riegeli/base:chain",
"//riegeli/base:types",
"//riegeli/records:record_writer",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings:string_view",
"@rules_python//python/cc:current_py_cc_headers",
],
)
py_extension(
name = "record_position",
srcs = ["record_position.cc"],
hdrs = ["record_position.h"],
# record_position.cc has #define before #include to influence what the
# included files provide.
features = ["-use_header_modules"],
deps = [
"//python/riegeli/base:utils",
"//riegeli/base:arithmetic",
"//riegeli/base:types",
"//riegeli/records:record_position",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/hash",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:string_view",
"@rules_python//python/cc:current_py_cc_headers",
],
)
py_library(
name = "skipped_region",
srcs = ["skipped_region.py"],
)
proto_library(
name = "records_metadata_proto",
srcs = ["records_metadata.proto"],
deps = ["@com_google_protobuf//:descriptor_proto"],
)
py_proto_library(
name = "records_metadata_py_pb2",
deps = [":records_metadata_proto"],
)
================================================
FILE: python/riegeli/records/__init__.py
================================================
================================================
FILE: python/riegeli/records/examples/BUILD
================================================
load("@rules_python//python:defs.bzl", "py_binary")
package(features = ["header_modules"])
licenses(["notice"])
py_binary(
name = "write_read_records",
srcs = ["write_read_records.py"],
deps = [
"//python/riegeli",
"//python/riegeli/records/tests:records_test_py_pb2",
],
)
================================================
FILE: python/riegeli/records/examples/__init__.py
================================================
================================================
FILE: python/riegeli/records/examples/write_read_records.py
================================================
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Simple example which writes and reads a Riegeli/records file."""
import io
import riegeli
from riegeli.records.tests import records_test_pb2
def sample_string(i, size):
piece = f'{i} '.encode()
result = piece * -(-size // len(piece)) # len(result) >= size
return result[:size]
def sample_message(i, size):
return records_test_pb2.SimpleMessage(id=i, payload=sample_string(i, size))
def write_records(filename):
print('Writing', filename)
metadata = riegeli.RecordsMetadata()
riegeli.set_record_type(metadata, records_test_pb2.SimpleMessage)
with riegeli.RecordWriter(
io.FileIO(filename, mode='wb'), options='transpose', metadata=metadata
) as writer:
writer.write_messages(sample_message(i, 100) for i in range(100))
def read_records(filename):
print('Reading', filename)
with riegeli.RecordReader(
io.FileIO(filename, mode='rb'),
field_projection=[[
records_test_pb2.SimpleMessage.DESCRIPTOR.fields_by_name['id'].number
]],
) as reader:
print(
' '.join(
str(record.id)
for record in reader.read_messages(records_test_pb2.SimpleMessage)
)
)
def main():
filename = '/tmp/riegeli_example'
write_records(filename)
read_records(filename)
if __name__ == '__main__':
main()
================================================
FILE: python/riegeli/records/record_position.cc
================================================
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#define PY_SSIZE_T_CLEAN
#include <Python.h>
// clang-format: do not reorder the above include.
#include "python/riegeli/records/record_position.h"
// clang-format: do not reorder the above include.
#include <stdint.h>
#include <limits>
#include <memory>
#include <optional>
#include <utility>
#include "absl/base/optimization.h"
#include "absl/hash/hash.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "python/riegeli/base/utils.h"
#include "riegeli/base/arithmetic.h"
#include "riegeli/base/types.h"
#include "riegeli/records/record_position.h"
namespace riegeli::python {
namespace {
struct PyRecordPositionObject {
// clang-format off
PyObject_HEAD
static_assert(true, ""); // clang-format workaround.
// clang-format on
PythonWrapped<FutureRecordPosition> record_position;
};
extern PyTypeObject PyRecordPosition_Type;
// `extern "C"` sets the C calling convention for compatibility with the Python
// API. `static` avoids making symbols public, as `extern "C"` trumps anonymous
// namespace.
extern "C" {
static void RecordPositionDestructor(PyRecordPositionObject* self) {
PythonUnlocked([&] { self->record_position.reset(); });
Py_TYPE(self)->tp_free(self);
}
static PyRecordPositionObject* RecordPositionNew(PyTypeObject* cls,
PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"chunk_begin", "record_index",
nullptr};
PyObject* chunk_begin_arg;
PyObject* record_index_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "OO:RecordPosition", const_cast<char**>(keywords),
&chunk_begin_arg, &record_index_arg))) {
return nullptr;
}
const std::optional<Position> chunk_begin =
PositionFromPython(chunk_begin_arg);
if (ABSL_PREDICT_FALSE(chunk_begin == std::nullopt)) return nullptr;
const std::optional<Position> record_index =
PositionFromPython(record_index_arg);
if (ABSL_PREDICT_FALSE(record_index == std::nullopt)) return nullptr;
if (ABSL_PREDICT_FALSE(*chunk_begin > std::numeric_limits<uint64_t>::max()) ||
ABSL_PREDICT_FALSE(*record_index >
std::numeric_limits<uint64_t>::max() - *chunk_begin)) {
PyErr_Format(PyExc_OverflowError, "RecordPosition overflow: %llu/%llu",
static_cast<unsigned long long>(*chunk_begin),
static_cast<unsigned long long>(*record_index));
return nullptr;
}
std::unique_ptr<PyRecordPositionObject, Deleter> self(
reinterpret_cast<PyRecordPositionObject*>(cls->tp_alloc(cls, 0)));
if (ABSL_PREDICT_FALSE(self == nullptr)) return nullptr;
self->record_position.emplace(RecordPosition(
IntCast<uint64_t>(*chunk_begin), IntCast<uint64_t>(*record_index)));
return self.release();
}
static PyObject* RecordPositionChunkBegin(PyRecordPositionObject* self,
void* closure) {
const RecordPosition pos =
PythonUnlocked([&] { return self->record_position->get(); });
return PositionToPython(pos.chunk_begin()).release();
}
static PyObject* RecordPositionRecordIndex(PyRecordPositionObject* self,
void* closure) {
const RecordPosition pos =
PythonUnlocked([&] { return self->record_position->get(); });
return PositionToPython(pos.record_index()).release();
}
static PyObject* RecordPositionNumeric(PyRecordPositionObject* self,
void* closure) {
const RecordPosition pos =
PythonUnlocked([&] { return self->record_position->get(); });
return PositionToPython(pos.numeric()).release();
}
static PyObject* RecordPositionCompare(PyObject* a, PyObject* b, int op) {
if (ABSL_PREDICT_FALSE(!PyObject_TypeCheck(a, &PyRecordPosition_Type)) ||
ABSL_PREDICT_FALSE(!PyObject_TypeCheck(b, &PyRecordPosition_Type))) {
Py_INCREF(Py_NotImplemented);
return Py_NotImplemented;
}
RecordPosition a_pos;
RecordPosition b_pos;
PythonUnlocked([&] {
a_pos =
reinterpret_cast<PyRecordPositionObject*>(a)->record_position->get();
b_pos =
reinterpret_cast<PyRecordPositionObject*>(b)->record_position->get();
});
switch (op) {
case Py_EQ:
return PyBool_FromLong(a_pos == b_pos);
case Py_NE:
return PyBool_FromLong(a_pos != b_pos);
case Py_LT:
return PyBool_FromLong(a_pos < b_pos);
case Py_GT:
return PyBool_FromLong(a_pos > b_pos);
case Py_LE:
return PyBool_FromLong(a_pos <= b_pos);
case Py_GE:
return PyBool_FromLong(a_pos >= b_pos);
default:
Py_INCREF(Py_NotImplemented);
return Py_NotImplemented;
}
}
static Py_hash_t RecordPositionHash(PyRecordPositionObject* self) {
const RecordPosition pos =
PythonUnlocked([&] { return self->record_position->get(); });
Py_hash_t hash = static_cast<Py_hash_t>(absl::Hash<RecordPosition>()(pos));
if (ABSL_PREDICT_FALSE(hash == -1)) hash = -2;
return hash;
}
static PyObject* RecordPositionStr(PyRecordPositionObject* self) {
const RecordPosition pos =
PythonUnlocked([&] { return self->record_position->get(); });
return StringToPython(pos.ToString()).release();
}
static PyRecordPositionObject* RecordPositionFromStr(PyTypeObject* cls,
PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"serialized", nullptr};
PyObject* serialized_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:from_str", const_cast<char**>(keywords),
&serialized_arg))) {
return nullptr;
}
StrOrBytes serialized;
if (ABSL_PREDICT_FALSE(!serialized.FromPython(serialized_arg))) {
return nullptr;
}
RecordPosition pos;
if (ABSL_PREDICT_FALSE(!pos.FromString(serialized))) {
PyErr_SetString(PyExc_ValueError, "RecordPosition.from_str() failed");
return nullptr;
}
std::unique_ptr<PyRecordPositionObject, Deleter> self(
reinterpret_cast<PyRecordPositionObject*>(cls->tp_alloc(cls, 0)));
if (ABSL_PREDICT_FALSE(self == nullptr)) return nullptr;
self->record_position.emplace(pos);
return self.release();
}
static PyObject* RecordPositionRepr(PyRecordPositionObject* self) {
const RecordPosition pos =
PythonUnlocked([&] { return self->record_position->get(); });
return StringToPython(absl::StrCat("RecordPosition(", pos.chunk_begin(), ", ",
pos.record_index(), ")"))
.release();
}
static PyObject* RecordPositionToBytes(PyRecordPositionObject* self,
PyObject* args) {
const RecordPosition pos =
PythonUnlocked([&] { return self->record_position->get(); });
return BytesToPython(pos.ToBytes()).release();
}
static PyRecordPositionObject* RecordPositionFromBytes(PyTypeObject* cls,
PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"serialized", nullptr};
PyObject* serialized_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:from_bytes", const_cast<char**>(keywords),
&serialized_arg))) {
return nullptr;
}
BytesLike serialized;
if (ABSL_PREDICT_FALSE(!serialized.FromPython(serialized_arg))) {
return nullptr;
}
RecordPosition pos;
if (ABSL_PREDICT_FALSE(!pos.FromBytes(serialized))) {
PyErr_SetString(PyExc_ValueError, "RecordPosition.from_bytes() failed");
return nullptr;
}
std::unique_ptr<PyRecordPositionObject, Deleter> self(
reinterpret_cast<PyRecordPositionObject*>(cls->tp_alloc(cls, 0)));
if (ABSL_PREDICT_FALSE(self == nullptr)) return nullptr;
self->record_position.emplace(pos);
return self.release();
}
} // extern "C"
const PyMethodDef RecordPositionMethods[] = {
{"from_str", reinterpret_cast<PyCFunction>(RecordPositionFromStr),
METH_VARARGS | METH_KEYWORDS | METH_CLASS,
R"doc(
from_str(type, serialized: str | bytes) -> RecordPosition
Parses RecordPosition from its text format.
Args:
serialized: Text string to parse.
)doc"},
{"to_bytes", reinterpret_cast<PyCFunction>(RecordPositionToBytes),
METH_NOARGS,
R"doc(
to_bytes(self) -> bytes
Returns the RecordPosition serialized to its binary format.
Serialized byte strings have the same natural order as the corresponding
positions.
)doc"},
{"from_bytes", reinterpret_cast<PyCFunction>(RecordPositionFromBytes),
METH_VARARGS | METH_KEYWORDS | METH_CLASS, R"doc(
from_bytes(
type, serialized: bytes | bytearray | memoryview) -> RecordPosition
Parses RecordPosition from its binary format.
Serialized byte strings have the same natural order as the corresponding
positions.
Args:
serialized: Byte string to parse.
)doc"},
{nullptr, nullptr, 0, nullptr},
};
const PyGetSetDef RecordPositionGetSet[] = {
{const_cast<char*>("chunk_begin"),
reinterpret_cast<getter>(RecordPositionChunkBegin), nullptr,
const_cast<char*>(R"doc(
chunk_begin: int
File position of the beginning of the chunk containing the given record.
)doc"),
nullptr},
{const_cast<char*>("record_index"),
reinterpret_cast<getter>(RecordPositionRecordIndex), nullptr,
const_cast<char*>(R"doc(
record_index: int
Index of the record within the chunk.
)doc"),
nullptr},
{const_cast<char*>("numeric"),
reinterpret_cast<getter>(RecordPositionNumeric), nullptr,
const_cast<char*>(R"doc(
numeric: int
Converts RecordPosition to an integer scaled between 0 and file size.
Distinct RecordPositions of a valid file have distinct numeric values.
)doc"),
nullptr},
{nullptr, nullptr, nullptr, nullptr, nullptr}};
PyTypeObject PyRecordPosition_Type = {
// clang-format off
PyVarObject_HEAD_INIT(&PyType_Type, 0)
// clang-format on
"riegeli.records.record_position.RecordPosition", // tp_name
sizeof(PyRecordPositionObject), // tp_basicsize
0, // tp_itemsize
reinterpret_cast<destructor>(RecordPositionDestructor), // tp_dealloc
#if PY_VERSION_HEX >= 0x03080000
0, // tp_vectorcall_offset
#else
nullptr, // tp_print
#endif
nullptr, // tp_getattr
nullptr, // tp_setattr
nullptr, // tp_as_async
reinterpret_cast<reprfunc>(RecordPositionRepr), // tp_repr
nullptr, // tp_as_number
nullptr, // tp_as_sequence
nullptr, // tp_as_mapping
reinterpret_cast<hashfunc>(RecordPositionHash), // tp_hash
nullptr, // tp_call
reinterpret_cast<reprfunc>(RecordPositionStr), // tp_str
nullptr, // tp_getattro
nullptr, // tp_setattro
nullptr, // tp_as_buffer
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags
R"doc(
RecordPosition(chunk_begin: int, record_index: int) -> RecordPosition
Represents a position in a Riegeli/records file.
There are two ways of expressing positions, both strictly monotonic:
* RecordPosition - Faster for seeking.
* int - Scaled between 0 and file size.
RecordPosition can be converted to int by the numeric property.
Working with RecordPosition is recommended, unless it is needed to seek to an
approximate position interpolated along the file, e.g. for splitting the file
into shards, or unless the position must be expressed as an integer from the
range [0, file_size] in order to fit into a preexisting API.
Both RecordReader and RecordWriter return positions. A position from
RecordWriter can act as a future: accessing its contents for the first time
might block, waiting for pending operations to complete.
)doc", // tp_doc
nullptr, // tp_traverse
nullptr, // tp_clear
RecordPositionCompare, // tp_richcompare
0, // tp_weaklistoffset
nullptr, // tp_iter
nullptr, // tp_iternext
const_cast<PyMethodDef*>(RecordPositionMethods), // tp_methods
nullptr, // tp_members
const_cast<PyGetSetDef*>(RecordPositionGetSet), // tp_getset
nullptr, // tp_base
nullptr, // tp_dict
nullptr, // tp_descr_get
nullptr, // tp_descr_set
0, // tp_dictoffset
nullptr, // tp_init
nullptr, // tp_alloc
reinterpret_cast<newfunc>(RecordPositionNew), // tp_new
nullptr, // tp_free
nullptr, // tp_is_gc
nullptr, // tp_bases
nullptr, // tp_mro
nullptr, // tp_cache
nullptr, // tp_subclasses
nullptr, // tp_weaklist
nullptr, // tp_del
0, // tp_version_tag
nullptr, // tp_finalize
};
PythonPtr RecordPositionToPython(FutureRecordPosition value) {
PythonPtr self(PyRecordPosition_Type.tp_alloc(&PyRecordPosition_Type, 0));
if (ABSL_PREDICT_FALSE(self == nullptr)) return nullptr;
reinterpret_cast<PyRecordPositionObject*>(self.get())
->record_position.emplace(std::move(value));
return self;
}
std::optional<RecordPosition> RecordPositionFromPython(PyObject* object) {
if (ABSL_PREDICT_FALSE(!PyObject_TypeCheck(object, &PyRecordPosition_Type))) {
PyErr_Format(PyExc_TypeError, "Expected RecordPosition, not %s",
Py_TYPE(object)->tp_name);
return std::nullopt;
}
return PythonUnlocked([&] {
return reinterpret_cast<PyRecordPositionObject*>(object)
->record_position->get();
});
}
const char* const kModuleName = "riegeli.records.record_position";
const char kModuleDoc[] =
R"doc(Represents a position in a Riegeli/records file.)doc";
PyModuleDef kModuleDef = {
PyModuleDef_HEAD_INIT,
kModuleName, // m_name
kModuleDoc, // m_doc
-1, // m_size
nullptr, // m_methods
nullptr, // m_slots
nullptr, // m_traverse
nullptr, // m_clear
nullptr, // m_free
};
PyObject* InitModule() {
if (ABSL_PREDICT_FALSE(PyType_Ready(&PyRecordPosition_Type) < 0)) {
return nullptr;
}
PythonPtr module(PyModule_Create(&kModuleDef));
if (ABSL_PREDICT_FALSE(module == nullptr)) return nullptr;
Py_INCREF(&PyRecordPosition_Type);
if (ABSL_PREDICT_FALSE(PyModule_AddObject(module.get(), "RecordPosition",
reinterpret_cast<PyObject*>(
&PyRecordPosition_Type)) < 0)) {
return nullptr;
}
static constexpr RecordPositionApi kRecordPositionApi = {
RecordPositionToPython,
RecordPositionFromPython,
};
if (ABSL_PREDICT_FALSE(!ExportCapsule(
module.get(), kRecordPositionCapsuleName, &kRecordPositionApi))) {
return nullptr;
}
return module.release();
}
} // namespace
PyMODINIT_FUNC PyInit_record_position() { return InitModule(); }
} // namespace riegeli::python
================================================
FILE: python/riegeli/records/record_position.h
================================================
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef PYTHON_RIEGELI_RECORDS_RECORD_POSITION_H_
#define PYTHON_RIEGELI_RECORDS_RECORD_POSITION_H_
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#include <Python.h>
// clang-format: do not reorder the above include.
#include <optional>
#include "python/riegeli/base/utils.h"
#include "riegeli/records/record_position.h"
namespace riegeli::python {
// Access the API thus:
// ```
// static constexpr ImportedCapsule<RecordPositionApi> kRecordPositionApi(
// kRecordPositionCapsuleName);
// ```
struct RecordPositionApi {
PythonPtr (*RecordPositionToPython)(FutureRecordPosition value);
std::optional<RecordPosition> (*RecordPositionFromPython)(PyObject* object);
};
inline constexpr const char* kRecordPositionCapsuleName =
"riegeli.records.record_position._CPPAPI";
} // namespace riegeli::python
#endif // PYTHON_RIEGELI_RECORDS_RECORD_POSITION_H_
================================================
FILE: python/riegeli/records/record_reader.cc
================================================
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// From https://docs.python.org/3/c-api/intro.html:
// Since Python may define some pre-processor definitions which affect the
// standard headers on some systems, you must include Python.h before any
// standard headers are included.
#define PY_SSIZE_T_CLEAN
#include <Python.h>
// clang-format: do not reorder the above include.
#include <stddef.h>
#include <functional>
#include <memory>
#include <optional>
#include <utility>
#include "absl/base/optimization.h"
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
#include "python/riegeli/base/utils.h"
#include "python/riegeli/bytes/python_reader.h"
#include "python/riegeli/records/record_position.h"
#include "riegeli/base/arithmetic.h"
#include "riegeli/base/assert.h"
#include "riegeli/base/chain.h"
#include "riegeli/base/compare.h"
#include "riegeli/base/types.h"
#include "riegeli/chunk_encoding/field_projection.h"
#include "riegeli/records/record_position.h"
#include "riegeli/records/record_reader.h"
#include "riegeli/records/skipped_region.h"
namespace riegeli::python {
namespace {
constexpr ImportedCapsule<RecordPositionApi> kRecordPositionApi(
kRecordPositionCapsuleName);
// `extern "C"` sets the C calling convention for compatibility with the Python
// API. `static` avoids making symbols public, as `extern "C"` trumps anonymous
// namespace.
extern "C" {
static PyObject* GetRecordType(PyObject* self, PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"metadata", nullptr};
PyObject* metadata_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:get_record_type", const_cast<char**>(keywords),
&metadata_arg))) {
return nullptr;
}
// record_type_name = metadata.record_type_name
static constexpr Identifier id_record_type_name("record_type_name");
const PythonPtr record_type_name(
PyObject_GetAttr(metadata_arg, id_record_type_name.get()));
if (ABSL_PREDICT_FALSE(record_type_name == nullptr)) return nullptr;
// if not record_type_name: return None
const int record_type_name_is_true = PyObject_IsTrue(record_type_name.get());
if (ABSL_PREDICT_FALSE(record_type_name_is_true < 0)) return nullptr;
if (record_type_name_is_true == 0) Py_RETURN_NONE;
// file_descriptors = metadata.file_descriptor
static constexpr Identifier id_file_descriptor("file_descriptor");
const PythonPtr file_descriptors(
PyObject_GetAttr(metadata_arg, id_file_descriptor.get()));
if (ABSL_PREDICT_FALSE(file_descriptors == nullptr)) return nullptr;
// if not file_descriptors: return None
const int file_descriptors_is_true = PyObject_IsTrue(file_descriptors.get());
if (ABSL_PREDICT_FALSE(file_descriptors_is_true < 0)) return nullptr;
if (file_descriptors_is_true == 0) Py_RETURN_NONE;
// pool = DescriptorPool()
static constexpr ImportedConstant kDescriptorPool(
"google.protobuf.descriptor_pool", "DescriptorPool");
if (ABSL_PREDICT_FALSE(!kDescriptorPool.Verify())) return nullptr;
const PythonPtr pool(
PyObject_CallFunctionObjArgs(kDescriptorPool.get(), nullptr));
if (ABSL_PREDICT_FALSE(pool == nullptr)) return nullptr;
// for file_descriptor in file_descriptors:
// pool.Add(file_descriptor)
const PythonPtr iter(PyObject_GetIter(file_descriptors.get()));
if (ABSL_PREDICT_FALSE(iter == nullptr)) return nullptr;
while (const PythonPtr file_descriptor{PyIter_Next(iter.get())}) {
static constexpr Identifier id_Add("Add");
const PythonPtr add_result(PyObject_CallMethodObjArgs(
pool.get(), id_Add.get(), file_descriptor.get(), nullptr));
if (ABSL_PREDICT_FALSE(add_result == nullptr)) return nullptr;
}
if (ABSL_PREDICT_FALSE(PyErr_Occurred() != nullptr)) return nullptr;
// message_descriptor = pool.FindMessageTypeByName(record_type_name)
static constexpr Identifier id_FindMessageTypeByName("FindMessageTypeByName");
const PythonPtr message_descriptor(
PyObject_CallMethodObjArgs(pool.get(), id_FindMessageTypeByName.get(),
record_type_name.get(), nullptr));
if (ABSL_PREDICT_FALSE(message_descriptor == nullptr)) return nullptr;
// return GetMessageClass(message_descriptor)
const PythonPtr message_factory(
PyImport_ImportModule("google.protobuf.message_factory"));
if (ABSL_PREDICT_FALSE(message_factory == nullptr)) return nullptr;
static constexpr Identifier id_GetMessageClass("GetMessageClass");
return PyObject_CallMethodObjArgs(message_factory.get(),
id_GetMessageClass.get(),
message_descriptor.get(), nullptr);
}
} // extern "C"
struct PyRecordReaderObject {
// clang-format off
PyObject_HEAD
static_assert(true, ""); // clang-format workaround.
// clang-format on
PythonWrapped<RecordReader<PythonReader>> record_reader;
PyObject* recovery;
PythonWrapped<Exception> recovery_exception;
};
extern PyTypeObject PyRecordReader_Type;
struct PyRecordIterObject {
// clang-format off
PyObject_HEAD
static_assert(true, ""); // clang-format workaround.
// clang-format on
PyObject* (*read_record)(PyRecordReaderObject* self, PyObject* args);
PyRecordReaderObject* record_reader;
PyObject* args;
};
extern PyTypeObject PyRecordIter_Type;
bool RecordReaderHasException(PyRecordReaderObject* self) {
return self->recovery_exception.has_value() || !self->record_reader->ok();
}
void SetExceptionFromRecordReader(PyRecordReaderObject* self) {
if (self->recovery_exception.has_value()) {
self->recovery_exception->Restore();
return;
}
RIEGELI_ASSERT(!self->record_reader->ok())
<< "Failed precondition of SetExceptionFromRecordReader(): "
"RecordReader OK";
if (!self->record_reader->src().exception().ok()) {
self->record_reader->src().exception().Restore();
return;
}
SetRiegeliError(self->record_reader->status());
}
std::optional<int> VerifyFieldNumber(long field_number_value) {
static_assert(Field::kExistenceOnly == 0,
"VerifyFieldNumber() assumes that Field::kExistenceOnly == 0");
if (ABSL_PREDICT_FALSE(field_number_value < Field::kExistenceOnly ||
field_number_value > (1 << 29) - 1)) {
PyErr_Format(PyExc_OverflowError, "Field number out of range: %ld",
field_number_value);
return std::nullopt;
}
return IntCast<int>(field_number_value);
}
std::optional<int> FieldNumberFromPython(PyObject* object) {
if (ABSL_PREDICT_FALSE(!PyLong_Check(object))) {
PyErr_Format(PyExc_TypeError, "Expected int, not %s",
Py_TYPE(object)->tp_name);
return std::nullopt;
}
const long field_number_value = PyLong_AsLong(object);
if (ABSL_PREDICT_FALSE(field_number_value == -1) && PyErr_Occurred()) {
return std::nullopt;
}
return VerifyFieldNumber(field_number_value);
}
std::optional<FieldProjection> FieldProjectionFromPython(PyObject* object) {
FieldProjection field_projection;
const PythonPtr field_iter(PyObject_GetIter(object));
if (ABSL_PREDICT_FALSE(field_iter == nullptr)) return std::nullopt;
while (const PythonPtr field_object{PyIter_Next(field_iter.get())}) {
Field field;
const PythonPtr field_number_iter(PyObject_GetIter(field_object.get()));
if (ABSL_PREDICT_FALSE(field_number_iter == nullptr)) return std::nullopt;
while (const PythonPtr field_number_object{
PyIter_Next(field_number_iter.get())}) {
const std::optional<int> field_number =
FieldNumberFromPython(field_number_object.get());
if (ABSL_PREDICT_FALSE(field_number == std::nullopt)) return std::nullopt;
field.AddFieldNumber(*field_number);
}
if (ABSL_PREDICT_FALSE(PyErr_Occurred() != nullptr)) return std::nullopt;
field_projection.AddField(std::move(field));
}
if (ABSL_PREDICT_FALSE(PyErr_Occurred() != nullptr)) return std::nullopt;
return field_projection;
}
// `extern "C"` sets the C calling convention for compatibility with the Python
// API. `static` avoids making symbols public, as `extern "C"` trumps anonymous
// namespace.
extern "C" {
static void RecordReaderDestructor(PyRecordReaderObject* self) {
PyObject_GC_UnTrack(self);
#if PY_VERSION_HEX < 0x030D0000 // < 3.13
Py_TRASHCAN_BEGIN(self, RecordReaderDestructor);
#endif
PythonUnlocked([&] { self->record_reader.reset(); });
Py_XDECREF(self->recovery);
self->recovery_exception.reset();
Py_TYPE(self)->tp_free(self);
#if PY_VERSION_HEX < 0x030D0000 // < 3.13
Py_TRASHCAN_END;
#endif
}
static int RecordReaderTraverse(PyRecordReaderObject* self, visitproc visit,
void* arg) {
Py_VISIT(self->recovery);
if (self->recovery_exception.has_value()) {
const int recovery_exception_result =
self->recovery_exception->Traverse(visit, arg);
if (ABSL_PREDICT_FALSE(recovery_exception_result != 0)) {
return recovery_exception_result;
}
}
if (self->record_reader.has_value()) {
return self->record_reader->src().Traverse(visit, arg);
}
return 0;
}
static int RecordReaderClear(PyRecordReaderObject* self) {
PythonUnlocked([&] { self->record_reader.reset(); });
Py_CLEAR(self->recovery);
self->recovery_exception.reset();
return 0;
}
static int RecordReaderInit(PyRecordReaderObject* self, PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"src",
"owns_src",
"assumed_pos",
"min_buffer_size",
"max_buffer_size",
"buffer_size",
"field_projection",
"recovery",
nullptr};
PyObject* src_arg;
PyObject* owns_src_arg = nullptr;
PyObject* assumed_pos_arg = nullptr;
PyObject* min_buffer_size_arg = nullptr;
PyObject* max_buffer_size_arg = nullptr;
PyObject* buffer_size_arg = nullptr;
PyObject* field_projection_arg = nullptr;
PyObject* recovery_arg = nullptr;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O|$OOOOOOO:RecordReader", const_cast<char**>(keywords),
&src_arg, &owns_src_arg, &assumed_pos_arg, &min_buffer_size_arg,
&max_buffer_size_arg, &buffer_size_arg, &field_projection_arg,
&recovery_arg))) {
return -1;
}
PythonReader::Options python_reader_options;
python_reader_options.set_owns_src(true);
if (owns_src_arg != nullptr) {
const int owns_src_is_true = PyObject_IsTrue(owns_src_arg);
if (ABSL_PREDICT_FALSE(owns_src_is_true < 0)) return -1;
python_reader_options.set_owns_src(owns_src_is_true != 0);
}
if (assumed_pos_arg != nullptr && assumed_pos_arg != Py_None) {
const std::optional<Position> assumed_pos =
PositionFromPython(assumed_pos_arg);
if (ABSL_PREDICT_FALSE(assumed_pos == std::nullopt)) return -1;
python_reader_options.set_assumed_pos(*assumed_pos);
}
if (buffer_size_arg != nullptr && buffer_size_arg != Py_None) {
min_buffer_size_arg = buffer_size_arg;
max_buffer_size_arg = buffer_size_arg;
}
if (min_buffer_size_arg != nullptr) {
const std::optional<size_t> min_buffer_size =
SizeFromPython(min_buffer_size_arg);
if (ABSL_PREDICT_FALSE(min_buffer_size == std::nullopt)) return -1;
python_reader_options.set_min_buffer_size(*min_buffer_size);
}
if (max_buffer_size_arg != nullptr) {
const std::optional<size_t> max_buffer_size =
SizeFromPython(max_buffer_size_arg);
if (ABSL_PREDICT_FALSE(max_buffer_size == std::nullopt)) return -1;
python_reader_options.set_max_buffer_size(*max_buffer_size);
}
RecordReaderBase::Options record_reader_options;
if (field_projection_arg != nullptr && field_projection_arg != Py_None) {
std::optional<FieldProjection> field_projection =
FieldProjectionFromPython(field_projection_arg);
if (ABSL_PREDICT_FALSE(field_projection == std::nullopt)) return -1;
record_reader_options.set_field_projection(*std::move(field_projection));
}
if (recovery_arg != nullptr && recovery_arg != Py_None) {
Py_INCREF(recovery_arg);
Py_XDECREF(self->recovery);
self->recovery = recovery_arg;
record_reader_options.set_recovery([self](
const SkippedRegion& skipped_region,
RecordReaderBase& record_reader) {
PythonLock lock;
const PythonPtr begin_object = PositionToPython(skipped_region.begin());
if (ABSL_PREDICT_FALSE(begin_object == nullptr)) {
self->recovery_exception.emplace(Exception::Fetch());
return false;
}
const PythonPtr end_object = PositionToPython(skipped_region.end());
if (ABSL_PREDICT_FALSE(end_object == nullptr)) {
self->recovery_exception.emplace(Exception::Fetch());
return false;
}
const PythonPtr message_object = StringToPython(skipped_region.message());
if (ABSL_PREDICT_FALSE(message_object == nullptr)) {
self->recovery_exception.emplace(Exception::Fetch());
return false;
}
static constexpr ImportedConstant kSkippedRegion(
"riegeli.records.skipped_region", "SkippedRegion");
if (ABSL_PREDICT_FALSE(!kSkippedRegion.Verify())) {
self->recovery_exception.emplace(Exception::Fetch());
return false;
}
const PythonPtr skipped_region_object(PyObject_CallFunctionObjArgs(
kSkippedRegion.get(), begin_object.get(), end_object.get(),
message_object.get(), nullptr));
if (ABSL_PREDICT_FALSE(skipped_region_object == nullptr)) {
self->recovery_exception.emplace(Exception::Fetch());
return false;
}
const PythonPtr recovery_result(PyObject_CallFunctionObjArgs(
self->recovery, skipped_region_object.get(), nullptr));
if (ABSL_PREDICT_FALSE(recovery_result == nullptr)) {
if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
PyErr_Clear();
} else {
self->recovery_exception.emplace(Exception::Fetch());
}
return false;
}
return true;
});
}
PythonReader python_reader(src_arg, std::move(python_reader_options));
PythonUnlocked([&] {
self->record_reader.emplace(std::move(python_reader),
std::move(record_reader_options));
});
if (ABSL_PREDICT_FALSE(!self->record_reader->ok())) {
self->record_reader->src().Close();
SetExceptionFromRecordReader(self);
return -1;
}
return 0;
}
static PyObject* RecordReaderSrc(PyRecordReaderObject* self, void* closure) {
PyObject* const src = ABSL_PREDICT_FALSE(!self->record_reader.has_value())
? Py_None
: self->record_reader->src().src();
Py_INCREF(src);
return src;
}
static PyObject* RecordReaderRepr(PyRecordReaderObject* self) {
const PythonPtr format = StringToPython("<RecordReader src={!r}>");
if (ABSL_PREDICT_FALSE(format == nullptr)) return nullptr;
// return format.format(self.src)
PyObject* const src = ABSL_PREDICT_FALSE(!self->record_reader.has_value())
? Py_None
: self->record_reader->src().src();
static constexpr Identifier id_format("format");
return PyObject_CallMethodObjArgs(format.get(), id_format.get(), src,
nullptr);
}
static PyObject* RecordReaderEnter(PyObject* self, PyObject* args) {
// return self
Py_INCREF(self);
return self;
}
static PyObject* RecordReaderExit(PyRecordReaderObject* self, PyObject* args) {
PyObject* exc_type;
PyObject* exc_value;
PyObject* traceback;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTuple(args, "OOO:__exit__", &exc_type,
&exc_value, &traceback))) {
return nullptr;
}
// self.close(), suppressing exceptions if exc_type != None.
if (ABSL_PREDICT_TRUE(self->record_reader.has_value())) {
const bool close_ok =
PythonUnlocked([&] { return self->record_reader->Close(); });
if (ABSL_PREDICT_FALSE(!close_ok) && exc_type == Py_None) {
SetExceptionFromRecordReader(self);
return nullptr;
}
}
Py_RETURN_FALSE;
}
static PyObject* RecordReaderClose(PyRecordReaderObject* self, PyObject* args) {
if (ABSL_PREDICT_TRUE(self->record_reader.has_value())) {
const bool close_ok =
PythonUnlocked([&] { return self->record_reader->Close(); });
if (ABSL_PREDICT_FALSE(!close_ok)) {
SetExceptionFromRecordReader(self);
return nullptr;
}
}
Py_RETURN_NONE;
}
static PyObject* RecordReaderCheckFileFormat(PyRecordReaderObject* self,
PyObject* args) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
const bool check_file_format_ok =
PythonUnlocked([&] { return self->record_reader->CheckFileFormat(); });
if (ABSL_PREDICT_FALSE(!check_file_format_ok)) {
if (ABSL_PREDICT_FALSE(RecordReaderHasException(self))) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
static PyObject* RecordReaderReadMetadata(PyRecordReaderObject* self,
PyObject* args) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
Chain metadata;
const bool read_serialized_metadata_ok = PythonUnlocked(
[&] { return self->record_reader->ReadSerializedMetadata(metadata); });
if (ABSL_PREDICT_FALSE(!read_serialized_metadata_ok)) {
if (ABSL_PREDICT_FALSE(RecordReaderHasException(self))) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_NONE;
}
const PythonPtr serialized_metadata = ChainToPython(metadata);
if (ABSL_PREDICT_FALSE(serialized_metadata == nullptr)) return nullptr;
// return RecordsMetadata.FromString(serialized_metadata)
static constexpr ImportedConstant kRecordsMetadata(
"riegeli.records.records_metadata_pb2", "RecordsMetadata");
if (ABSL_PREDICT_FALSE(!kRecordsMetadata.Verify())) return nullptr;
static constexpr ImportedConstant kDecodeError("google.protobuf.message",
"DecodeError");
if (ABSL_PREDICT_FALSE(!kDecodeError.Verify())) return nullptr;
static constexpr Identifier id_FromString("FromString");
PythonPtr metadata_object(
PyObject_CallMethodObjArgs(kRecordsMetadata.get(), id_FromString.get(),
serialized_metadata.get(), nullptr));
if (ABSL_PREDICT_FALSE(metadata_object == nullptr)) {
if (self->record_reader->recovery() != nullptr &&
PyErr_ExceptionMatches(kDecodeError.get())) {
const Exception exception = Exception::Fetch();
if (self->record_reader->recovery()(
SkippedRegion(self->record_reader->last_pos().chunk_begin(),
self->record_reader->pos().numeric(),
exception.message()),
*self->record_reader)) {
// Recovered metadata decoding, assume empty `RecordsMetadata`.
return PyObject_CallFunctionObjArgs(kRecordsMetadata.get(), nullptr);
}
if (ABSL_PREDICT_FALSE(self->recovery_exception.has_value())) {
self->recovery_exception->Restore();
return nullptr;
}
Py_RETURN_NONE;
}
return nullptr;
}
return metadata_object.release();
}
static PyObject* RecordReaderReadSerializedMetadata(PyRecordReaderObject* self,
PyObject* args) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
Chain metadata;
const bool read_serialized_metadata_ok = PythonUnlocked(
[&] { return self->record_reader->ReadSerializedMetadata(metadata); });
if (ABSL_PREDICT_FALSE(!read_serialized_metadata_ok)) {
if (ABSL_PREDICT_FALSE(RecordReaderHasException(self))) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_NONE;
}
return ChainToPython(metadata).release();
}
static PyObject* RecordReaderReadRecord(PyRecordReaderObject* self,
PyObject* args) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
Chain record;
const bool read_record_ok =
PythonUnlocked([&] { return self->record_reader->ReadRecord(record); });
if (ABSL_PREDICT_FALSE(!read_record_ok)) {
if (ABSL_PREDICT_FALSE(RecordReaderHasException(self))) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_NONE;
}
return ChainToPython(record).release();
}
static PyObject* RecordReaderReadMessage(PyRecordReaderObject* self,
PyObject* args, PyObject* kwargs) {
static constexpr const char* keywords[] = {"message_type", nullptr};
PyObject* message_type_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:read_message", const_cast<char**>(keywords),
&message_type_arg))) {
return nullptr;
}
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
absl::string_view record;
for (;;) {
const bool read_record_ok =
PythonUnlocked([&] { return self->record_reader->ReadRecord(record); });
if (ABSL_PREDICT_FALSE(!read_record_ok)) {
if (ABSL_PREDICT_FALSE(RecordReaderHasException(self))) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_NONE;
}
MemoryView memory_view;
PyObject* const record_object = memory_view.ToPython(record);
if (ABSL_PREDICT_FALSE(record_object == nullptr)) return nullptr;
static constexpr ImportedConstant kDecodeError("google.protobuf.message",
"DecodeError");
if (ABSL_PREDICT_FALSE(!kDecodeError.Verify())) return nullptr;
// return message_type.FromString(record)
static constexpr Identifier id_FromString("FromString");
PythonPtr message(PyObject_CallMethodObjArgs(
message_type_arg, id_FromString.get(), record_object, nullptr));
if (ABSL_PREDICT_FALSE(message == nullptr)) {
if (self->record_reader->recovery() != nullptr &&
PyErr_ExceptionMatches(kDecodeError.get())) {
const Exception exception = Exception::Fetch();
if (ABSL_PREDICT_FALSE(!memory_view.Release())) return nullptr;
if (self->record_reader->recovery()(
SkippedRegion(self->record_reader->last_pos().numeric(),
self->record_reader->pos().numeric(),
exception.message()),
*self->record_reader)) {
continue;
}
if (ABSL_PREDICT_FALSE(self->recovery_exception.has_value())) {
self->recovery_exception->Restore();
return nullptr;
}
Py_RETURN_NONE;
}
return nullptr;
}
if (ABSL_PREDICT_FALSE(!memory_view.Release())) return nullptr;
return message.release();
}
}
static PyRecordIterObject* RecordReaderReadRecords(PyRecordReaderObject* self,
PyObject* args) {
std::unique_ptr<PyRecordIterObject, Deleter> iter(
PyObject_GC_New(PyRecordIterObject, &PyRecordIter_Type));
if (ABSL_PREDICT_FALSE(iter == nullptr)) return nullptr;
iter->read_record = [](PyRecordReaderObject* self, PyObject* args) {
return RecordReaderReadRecord(self, args);
};
Py_INCREF(self);
iter->record_reader = self;
iter->args = nullptr;
return iter.release();
}
static PyRecordIterObject* RecordReaderReadMessages(PyRecordReaderObject* self,
PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"message_type", nullptr};
PyObject* message_type_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:read_messages", const_cast<char**>(keywords),
&message_type_arg))) {
return nullptr;
}
std::unique_ptr<PyRecordIterObject, Deleter> iter(
PyObject_GC_New(PyRecordIterObject, &PyRecordIter_Type));
if (ABSL_PREDICT_FALSE(iter == nullptr)) return nullptr;
iter->read_record = [](PyRecordReaderObject* self, PyObject* args) {
return RecordReaderReadMessage(self, args, nullptr);
};
Py_INCREF(self);
iter->record_reader = self;
iter->args = PyTuple_Pack(1, message_type_arg);
if (ABSL_PREDICT_FALSE(iter->args == nullptr)) return nullptr;
return iter.release();
}
static PyObject* RecordReaderSetFieldProjection(PyRecordReaderObject* self,
PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"field_projection", nullptr};
PyObject* field_projection_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:set_field_projection", const_cast<char**>(keywords),
&field_projection_arg))) {
return nullptr;
}
std::optional<FieldProjection> field_projection;
if (field_projection_arg == Py_None) {
field_projection = FieldProjection::All();
} else {
field_projection = FieldProjectionFromPython(field_projection_arg);
if (ABSL_PREDICT_FALSE(field_projection == std::nullopt)) return nullptr;
}
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
const bool set_field_projection_ok = PythonUnlocked([&] {
return self->record_reader->SetFieldProjection(
*std::move(field_projection));
});
if (ABSL_PREDICT_FALSE(!set_field_projection_ok)) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_NONE;
}
static PyObject* RecordReaderLastPos(PyRecordReaderObject* self,
void* closure) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
if (ABSL_PREDICT_FALSE(!kRecordPositionApi.Verify())) return nullptr;
if (ABSL_PREDICT_FALSE(!self->record_reader->last_record_is_valid())) {
SetRiegeliError(absl::FailedPreconditionError("No record was read"));
return nullptr;
}
return kRecordPositionApi
->RecordPositionToPython(self->record_reader->last_pos())
.release();
}
static PyObject* RecordReaderPos(PyRecordReaderObject* self, void* closure) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
if (ABSL_PREDICT_FALSE(!kRecordPositionApi.Verify())) return nullptr;
return kRecordPositionApi->RecordPositionToPython(self->record_reader->pos())
.release();
}
static PyObject* RecordReaderSupportsRandomAccess(PyRecordReaderObject* self,
void* closure) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
return PyBool_FromLong(self->record_reader->SupportsRandomAccess());
}
static PyObject* RecordReaderSeek(PyRecordReaderObject* self, PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"pos", nullptr};
PyObject* pos_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:seek", const_cast<char**>(keywords), &pos_arg))) {
return nullptr;
}
if (ABSL_PREDICT_FALSE(!kRecordPositionApi.Verify())) return nullptr;
const std::optional<RecordPosition> pos =
kRecordPositionApi->RecordPositionFromPython(pos_arg);
if (ABSL_PREDICT_FALSE(pos == std::nullopt)) return nullptr;
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
const bool seek_ok =
PythonUnlocked([&] { return self->record_reader->Seek(*pos); });
if (ABSL_PREDICT_FALSE(!seek_ok)) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_NONE;
}
static PyObject* RecordReaderSeekNumeric(PyRecordReaderObject* self,
PyObject* args, PyObject* kwargs) {
static constexpr const char* keywords[] = {"pos", nullptr};
PyObject* pos_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:seek_numeric", const_cast<char**>(keywords),
&pos_arg))) {
return nullptr;
}
const std::optional<Position> pos = PositionFromPython(pos_arg);
if (ABSL_PREDICT_FALSE(pos == std::nullopt)) return nullptr;
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
const bool seek_ok =
PythonUnlocked([&] { return self->record_reader->Seek(*pos); });
if (ABSL_PREDICT_FALSE(!seek_ok)) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_NONE;
}
static PyObject* RecordReaderSeekBack(PyRecordReaderObject* self,
PyObject* args) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
const bool seek_back_ok =
PythonUnlocked([&] { return self->record_reader->SeekBack(); });
if (ABSL_PREDICT_FALSE(!seek_back_ok)) {
if (ABSL_PREDICT_FALSE(RecordReaderHasException(self))) {
SetExceptionFromRecordReader(self);
return nullptr;
}
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
static PyObject* RecordReaderSize(PyRecordReaderObject* self, PyObject* args) {
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
const std::optional<Position> size =
PythonUnlocked([&] { return self->record_reader->Size(); });
if (ABSL_PREDICT_FALSE(size == std::nullopt)) {
SetExceptionFromRecordReader(self);
return nullptr;
}
return PositionToPython(*size).release();
}
static PyObject* RecordReaderSearch(PyRecordReaderObject* self, PyObject* args,
PyObject* kwargs) {
static constexpr const char* keywords[] = {"test", nullptr};
PyObject* test_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:search", const_cast<char**>(keywords), &test_arg))) {
return nullptr;
}
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
std::optional<Exception> test_exception;
const std::optional<PartialOrdering> result = PythonUnlocked([&] {
return self->record_reader->Search(
[&](RecordReaderBase&) -> std::optional<PartialOrdering> {
PythonLock lock;
const PythonPtr test_result(
PyObject_CallFunctionObjArgs(test_arg, self, nullptr));
if (ABSL_PREDICT_FALSE(test_result == nullptr)) {
test_exception.emplace(Exception::Fetch());
return std::nullopt;
}
const std::optional<PartialOrdering> ordering =
PartialOrderingFromPython(test_result.get());
if (ABSL_PREDICT_FALSE(ordering == std::nullopt)) {
test_exception.emplace(Exception::Fetch());
return std::nullopt;
}
return *ordering;
});
});
if (ABSL_PREDICT_FALSE(result == std::nullopt)) {
if (test_exception != std::nullopt) {
test_exception->Restore();
} else {
SetExceptionFromRecordReader(self);
}
return nullptr;
}
return PartialOrderingToPython(*result).release();
}
static PyObject* RecordReaderSearchForRecord(PyRecordReaderObject* self,
PyObject* args, PyObject* kwargs) {
static constexpr const char* keywords[] = {"test", nullptr};
PyObject* test_arg;
if (ABSL_PREDICT_FALSE(!PyArg_ParseTupleAndKeywords(
args, kwargs, "O:search_for_record", const_cast<char**>(keywords),
&test_arg))) {
return nullptr;
}
if (ABSL_PREDICT_FALSE(!self->record_reader.Verify())) return nullptr;
std::optional<Exception> test_exception;
const std::optional<PartialOrdering> result = PythonUnlocked([&] {
return self->record_reader->Search<Chain>(
[&](const Chain& record) -> std::optional<PartialOrdering> {
PythonLock lock;
const PythonPtr record_object = ChainToPython(record);
if (ABSL_PREDICT_FALSE(record_object == nullptr)) {
test_exception.emplace(Exception::Fetch());
return std::nullopt;
gitextract_1atzokxc/
├── .bazelrc
├── CONTRIBUTING.md
├── LICENSE
├── MANIFEST.in
├── MODULE.bazel
├── README.md
├── configure
├── doc/
│ ├── index.md
│ ├── record_writer_options.md
│ └── riegeli_records_file_format.md
├── python/
│ ├── BUILD
│ ├── MANIFEST.in
│ ├── README.md
│ ├── __init__.py
│ ├── build_pip_package.sh
│ ├── dummy_binary.py
│ ├── riegeli/
│ │ ├── BUILD
│ │ ├── BUILD.tpl
│ │ ├── __init__.py
│ │ ├── base/
│ │ │ ├── BUILD
│ │ │ ├── __init__.py
│ │ │ ├── riegeli_error.py
│ │ │ ├── utils.cc
│ │ │ └── utils.h
│ │ ├── bytes/
│ │ │ ├── BUILD
│ │ │ ├── python_reader.cc
│ │ │ ├── python_reader.h
│ │ │ ├── python_writer.cc
│ │ │ └── python_writer.h
│ │ ├── py_extension.bzl
│ │ ├── python_configure.bzl
│ │ ├── records/
│ │ │ ├── BUILD
│ │ │ ├── __init__.py
│ │ │ ├── examples/
│ │ │ │ ├── BUILD
│ │ │ │ ├── __init__.py
│ │ │ │ └── write_read_records.py
│ │ │ ├── record_position.cc
│ │ │ ├── record_position.h
│ │ │ ├── record_reader.cc
│ │ │ ├── record_writer.cc
│ │ │ ├── records_metadata.proto
│ │ │ ├── skipped_region.py
│ │ │ └── tests/
│ │ │ ├── BUILD
│ │ │ ├── __init__.py
│ │ │ ├── records_test.proto
│ │ │ └── records_test.py
│ │ └── tensorflow/
│ │ ├── BUILD
│ │ ├── __init__.py
│ │ ├── kernel_tests/
│ │ │ ├── __init__.py
│ │ │ └── riegeli_dataset_test.py
│ │ └── ops/
│ │ ├── __init__.py
│ │ └── riegeli_dataset_ops.py
│ └── setup.py
├── riegeli/
│ ├── .gitignore
│ ├── BUILD
│ ├── base/
│ │ ├── BUILD
│ │ ├── any.h
│ │ ├── any_initializer.h
│ │ ├── any_internal.h
│ │ ├── arithmetic.h
│ │ ├── assert.cc
│ │ ├── assert.h
│ │ ├── background_cleaning.cc
│ │ ├── background_cleaning.h
│ │ ├── binary_search.h
│ │ ├── buffer.cc
│ │ ├── buffer.h
│ │ ├── buffering.h
│ │ ├── byte_fill.cc
│ │ ├── byte_fill.h
│ │ ├── bytes_ref.h
│ │ ├── c_string_ref.h
│ │ ├── chain.cc
│ │ ├── chain.h
│ │ ├── chain_base.h
│ │ ├── chain_details.h
│ │ ├── closing_ptr.h
│ │ ├── compact_string.cc
│ │ ├── compact_string.h
│ │ ├── compare.h
│ │ ├── constexpr.h
│ │ ├── cord_iterator_span.cc
│ │ ├── cord_iterator_span.h
│ │ ├── cord_utils.cc
│ │ ├── cord_utils.h
│ │ ├── debug.cc
│ │ ├── debug.h
│ │ ├── dependency.h
│ │ ├── dependency_base.h
│ │ ├── dependency_manager.h
│ │ ├── errno_mapping.cc
│ │ ├── errno_mapping.h
│ │ ├── estimated_allocated_size.h
│ │ ├── external_data.cc
│ │ ├── external_data.h
│ │ ├── external_ref.h
│ │ ├── external_ref_base.h
│ │ ├── external_ref_support.h
│ │ ├── global.h
│ │ ├── hybrid_direct_common.h
│ │ ├── hybrid_direct_internal.h
│ │ ├── hybrid_direct_map.h
│ │ ├── hybrid_direct_set.h
│ │ ├── initializer.h
│ │ ├── initializer_internal.h
│ │ ├── intrusive_shared_ptr.h
│ │ ├── invoker.h
│ │ ├── iterable.h
│ │ ├── maker.h
│ │ ├── memory_estimator.cc
│ │ ├── memory_estimator.h
│ │ ├── moving_dependency.h
│ │ ├── new_aligned.h
│ │ ├── null_safe_memcpy.h
│ │ ├── object.cc
│ │ ├── object.h
│ │ ├── optional_compact_string.h
│ │ ├── options_parser.cc
│ │ ├── options_parser.h
│ │ ├── ownership.h
│ │ ├── parallelism.cc
│ │ ├── parallelism.h
│ │ ├── port.h
│ │ ├── recycling_pool.h
│ │ ├── ref_count.h
│ │ ├── reset.h
│ │ ├── shared_buffer.cc
│ │ ├── shared_buffer.h
│ │ ├── shared_ptr.h
│ │ ├── sized_shared_buffer.cc
│ │ ├── sized_shared_buffer.h
│ │ ├── stable_dependency.h
│ │ ├── status.cc
│ │ ├── status.h
│ │ ├── stream_utils.cc
│ │ ├── stream_utils.h
│ │ ├── string_ref.h
│ │ ├── string_utils.cc
│ │ ├── string_utils.h
│ │ ├── temporary_storage.h
│ │ ├── type_erased_ref.h
│ │ ├── type_id.h
│ │ ├── type_traits.h
│ │ ├── types.h
│ │ ├── unicode.cc
│ │ ├── unicode.h
│ │ └── uninitialized_vector.h
│ ├── brotli/
│ │ ├── BUILD
│ │ ├── brotli_allocator.cc
│ │ ├── brotli_allocator.h
│ │ ├── brotli_dictionary.cc
│ │ ├── brotli_dictionary.h
│ │ ├── brotli_reader.cc
│ │ ├── brotli_reader.h
│ │ ├── brotli_writer.cc
│ │ └── brotli_writer.h
│ ├── bytes/
│ │ ├── BUILD
│ │ ├── array_backward_writer.cc
│ │ ├── array_backward_writer.h
│ │ ├── array_writer.cc
│ │ ├── array_writer.h
│ │ ├── backward_writer.cc
│ │ ├── backward_writer.h
│ │ ├── buffer_options.cc
│ │ ├── buffer_options.h
│ │ ├── buffered_reader.cc
│ │ ├── buffered_reader.h
│ │ ├── buffered_writer.cc
│ │ ├── buffered_writer.h
│ │ ├── cfile_handle.cc
│ │ ├── cfile_handle.h
│ │ ├── cfile_internal.cc
│ │ ├── cfile_internal.h
│ │ ├── cfile_internal_for_cc.h
│ │ ├── cfile_reader.cc
│ │ ├── cfile_reader.h
│ │ ├── cfile_writer.cc
│ │ ├── cfile_writer.h
│ │ ├── chain_backward_writer.cc
│ │ ├── chain_backward_writer.h
│ │ ├── chain_reader.cc
│ │ ├── chain_reader.h
│ │ ├── chain_writer.cc
│ │ ├── chain_writer.h
│ │ ├── compact_string_writer.h
│ │ ├── copy_all.cc
│ │ ├── copy_all.h
│ │ ├── cord_backward_writer.cc
│ │ ├── cord_backward_writer.h
│ │ ├── cord_reader.cc
│ │ ├── cord_reader.h
│ │ ├── cord_writer.cc
│ │ ├── cord_writer.h
│ │ ├── fd_handle.cc
│ │ ├── fd_handle.h
│ │ ├── fd_internal.cc
│ │ ├── fd_internal.h
│ │ ├── fd_internal_for_cc.h
│ │ ├── fd_mmap_reader.cc
│ │ ├── fd_mmap_reader.h
│ │ ├── fd_reader.cc
│ │ ├── fd_reader.h
│ │ ├── fd_writer.cc
│ │ ├── fd_writer.h
│ │ ├── file_mode_string.cc
│ │ ├── file_mode_string.h
│ │ ├── iostream_internal.h
│ │ ├── istream_reader.cc
│ │ ├── istream_reader.h
│ │ ├── joining_reader.cc
│ │ ├── joining_reader.h
│ │ ├── limiting_backward_writer.cc
│ │ ├── limiting_backward_writer.h
│ │ ├── limiting_reader.cc
│ │ ├── limiting_reader.h
│ │ ├── limiting_writer.cc
│ │ ├── limiting_writer.h
│ │ ├── null_backward_writer.cc
│ │ ├── null_backward_writer.h
│ │ ├── null_writer.cc
│ │ ├── null_writer.h
│ │ ├── ostream_writer.cc
│ │ ├── ostream_writer.h
│ │ ├── path_ref.h
│ │ ├── position_shifting_backward_writer.cc
│ │ ├── position_shifting_backward_writer.h
│ │ ├── position_shifting_reader.cc
│ │ ├── position_shifting_reader.h
│ │ ├── position_shifting_writer.cc
│ │ ├── position_shifting_writer.h
│ │ ├── prefix_limiting_backward_writer.cc
│ │ ├── prefix_limiting_backward_writer.h
│ │ ├── prefix_limiting_reader.cc
│ │ ├── prefix_limiting_reader.h
│ │ ├── prefix_limiting_writer.cc
│ │ ├── prefix_limiting_writer.h
│ │ ├── pullable_reader.cc
│ │ ├── pullable_reader.h
│ │ ├── pushable_backward_writer.cc
│ │ ├── pushable_backward_writer.h
│ │ ├── pushable_writer.cc
│ │ ├── pushable_writer.h
│ │ ├── read_all.cc
│ │ ├── read_all.h
│ │ ├── reader.cc
│ │ ├── reader.h
│ │ ├── reader_cfile.cc
│ │ ├── reader_cfile.h
│ │ ├── reader_factory.cc
│ │ ├── reader_factory.h
│ │ ├── reader_istream.cc
│ │ ├── reader_istream.h
│ │ ├── resizable_writer.cc
│ │ ├── resizable_writer.h
│ │ ├── restricted_chain_writer.cc
│ │ ├── restricted_chain_writer.h
│ │ ├── splitting_writer.cc
│ │ ├── splitting_writer.h
│ │ ├── std_io.cc
│ │ ├── std_io.h
│ │ ├── string_reader.cc
│ │ ├── string_reader.h
│ │ ├── string_writer.cc
│ │ ├── string_writer.h
│ │ ├── stringify.h
│ │ ├── stringify_writer.h
│ │ ├── vector_writer.h
│ │ ├── wrapping_backward_writer.cc
│ │ ├── wrapping_backward_writer.h
│ │ ├── wrapping_reader.cc
│ │ ├── wrapping_reader.h
│ │ ├── wrapping_writer.cc
│ │ ├── wrapping_writer.h
│ │ ├── write.h
│ │ ├── write_int_internal.cc
│ │ ├── write_int_internal.h
│ │ ├── writer.cc
│ │ ├── writer.h
│ │ ├── writer_cfile.cc
│ │ ├── writer_cfile.h
│ │ ├── writer_ostream.cc
│ │ └── writer_ostream.h
│ ├── bzip2/
│ │ ├── BUILD
│ │ ├── bzip2_error.cc
│ │ ├── bzip2_error.h
│ │ ├── bzip2_reader.cc
│ │ ├── bzip2_reader.h
│ │ ├── bzip2_writer.cc
│ │ └── bzip2_writer.h
│ ├── chunk_encoding/
│ │ ├── BUILD
│ │ ├── README.md
│ │ ├── brotli_encoder_selection.cc
│ │ ├── brotli_encoder_selection.h
│ │ ├── chunk.cc
│ │ ├── chunk.h
│ │ ├── chunk_decoder.cc
│ │ ├── chunk_decoder.h
│ │ ├── chunk_encoder.cc
│ │ ├── chunk_encoder.h
│ │ ├── compressor.cc
│ │ ├── compressor.h
│ │ ├── compressor_options.cc
│ │ ├── compressor_options.h
│ │ ├── constants.h
│ │ ├── decompressor.cc
│ │ ├── decompressor.h
│ │ ├── deferred_encoder.cc
│ │ ├── deferred_encoder.h
│ │ ├── field_projection.h
│ │ ├── hash.cc
│ │ ├── hash.h
│ │ ├── simple_decoder.cc
│ │ ├── simple_decoder.h
│ │ ├── simple_encoder.cc
│ │ ├── simple_encoder.h
│ │ ├── transpose_decoder.cc
│ │ ├── transpose_decoder.h
│ │ ├── transpose_encoder.cc
│ │ ├── transpose_encoder.h
│ │ └── transpose_internal.h
│ ├── containers/
│ │ ├── BUILD
│ │ ├── chunked_sorted_string_set.cc
│ │ ├── chunked_sorted_string_set.h
│ │ ├── linear_sorted_string_set.cc
│ │ └── linear_sorted_string_set.h
│ ├── csv/
│ │ ├── BUILD
│ │ ├── csv_reader.cc
│ │ ├── csv_reader.h
│ │ ├── csv_record.cc
│ │ ├── csv_record.h
│ │ ├── csv_writer.cc
│ │ └── csv_writer.h
│ ├── digests/
│ │ ├── BUILD
│ │ ├── adler32_digester.cc
│ │ ├── adler32_digester.h
│ │ ├── crc32_digester.cc
│ │ ├── crc32_digester.h
│ │ ├── crc32c_digester.h
│ │ ├── digest_converter.h
│ │ ├── digester_handle.cc
│ │ ├── digester_handle.h
│ │ ├── digesting_reader.cc
│ │ ├── digesting_reader.h
│ │ ├── digesting_writer.cc
│ │ ├── digesting_writer.h
│ │ ├── highwayhash_digester.cc
│ │ ├── highwayhash_digester.h
│ │ ├── md5_digester.h
│ │ ├── openssl_digester.h
│ │ ├── sha1_digester.h
│ │ ├── sha256_digester.h
│ │ ├── sha512_256_digester.h
│ │ ├── sha512_digester.h
│ │ └── wrapping_digester.h
│ ├── endian/
│ │ ├── BUILD
│ │ ├── endian_reading.h
│ │ └── endian_writing.h
│ ├── gcs/
│ │ ├── BUILD
│ │ ├── gcs_internal.h
│ │ ├── gcs_object.cc
│ │ ├── gcs_object.h
│ │ ├── gcs_reader.cc
│ │ ├── gcs_reader.h
│ │ ├── gcs_writer.cc
│ │ └── gcs_writer.h
│ ├── lines/
│ │ ├── BUILD
│ │ ├── line_reading.cc
│ │ ├── line_reading.h
│ │ ├── line_writing.h
│ │ ├── newline.h
│ │ ├── text_reader.cc
│ │ ├── text_reader.h
│ │ ├── text_writer.cc
│ │ └── text_writer.h
│ ├── lz4/
│ │ ├── BUILD
│ │ ├── lz4_dictionary.cc
│ │ ├── lz4_dictionary.h
│ │ ├── lz4_reader.cc
│ │ ├── lz4_reader.h
│ │ ├── lz4_writer.cc
│ │ └── lz4_writer.h
│ ├── messages/
│ │ ├── BUILD
│ │ ├── context_projection.h
│ │ ├── dynamic_field_handler.h
│ │ ├── field_copier.h
│ │ ├── field_handler_map.h
│ │ ├── field_handlers.cc
│ │ ├── field_handlers.h
│ │ ├── map_entry_field.h
│ │ ├── message_wire_format.h
│ │ ├── parse_message.cc
│ │ ├── parse_message.h
│ │ ├── serialize_message.cc
│ │ ├── serialize_message.h
│ │ ├── serialized_message_assembler.cc
│ │ ├── serialized_message_assembler.h
│ │ ├── serialized_message_backward_writer.cc
│ │ ├── serialized_message_backward_writer.h
│ │ ├── serialized_message_internal.h
│ │ ├── serialized_message_reader.cc
│ │ ├── serialized_message_reader.h
│ │ ├── serialized_message_reader_internal.h
│ │ ├── serialized_message_writer.cc
│ │ ├── serialized_message_writer.h
│ │ ├── text_parse_message.cc
│ │ ├── text_parse_message.h
│ │ ├── text_print_message.cc
│ │ └── text_print_message.h
│ ├── ordered_varint/
│ │ ├── BUILD
│ │ ├── ordered_varint_internal.h
│ │ ├── ordered_varint_reading.cc
│ │ ├── ordered_varint_reading.h
│ │ ├── ordered_varint_writing.cc
│ │ └── ordered_varint_writing.h
│ ├── records/
│ │ ├── BUILD
│ │ ├── README.md
│ │ ├── block.h
│ │ ├── chunk_reader.cc
│ │ ├── chunk_reader.h
│ │ ├── chunk_writer.cc
│ │ ├── chunk_writer.h
│ │ ├── record_position.cc
│ │ ├── record_position.h
│ │ ├── record_reader.cc
│ │ ├── record_reader.h
│ │ ├── record_writer.cc
│ │ ├── record_writer.h
│ │ ├── records_metadata.proto
│ │ ├── skipped_region.cc
│ │ ├── skipped_region.h
│ │ └── tools/
│ │ ├── BUILD
│ │ ├── describe_riegeli_file.cc
│ │ ├── records_benchmark.cc
│ │ ├── riegeli_summary.proto
│ │ ├── tfrecord_recognizer.cc
│ │ └── tfrecord_recognizer.h
│ ├── snappy/
│ │ ├── BUILD
│ │ ├── framed/
│ │ │ ├── BUILD
│ │ │ ├── framed_snappy_reader.cc
│ │ │ ├── framed_snappy_reader.h
│ │ │ ├── framed_snappy_writer.cc
│ │ │ └── framed_snappy_writer.h
│ │ ├── hadoop/
│ │ │ ├── BUILD
│ │ │ ├── hadoop_snappy_reader.cc
│ │ │ ├── hadoop_snappy_reader.h
│ │ │ ├── hadoop_snappy_writer.cc
│ │ │ └── hadoop_snappy_writer.h
│ │ ├── snappy_reader.cc
│ │ ├── snappy_reader.h
│ │ ├── snappy_streams.cc
│ │ ├── snappy_streams.h
│ │ ├── snappy_writer.cc
│ │ └── snappy_writer.h
│ ├── tensorflow/
│ │ ├── BUILD
│ │ ├── io/
│ │ │ ├── BUILD
│ │ │ ├── file_reader.cc
│ │ │ ├── file_reader.h
│ │ │ ├── file_writer.cc
│ │ │ ├── file_writer.h
│ │ │ └── tstring_writer.h
│ │ ├── kernels/
│ │ │ └── riegeli_dataset_ops.cc
│ │ └── ops/
│ │ └── riegeli_dataset_ops.cc
│ ├── text/
│ │ ├── BUILD
│ │ ├── ascii_align.h
│ │ ├── concat.h
│ │ ├── join.h
│ │ ├── write_int.cc
│ │ └── write_int.h
│ ├── varint/
│ │ ├── BUILD
│ │ ├── varint_internal.h
│ │ ├── varint_reading.cc
│ │ ├── varint_reading.h
│ │ └── varint_writing.h
│ ├── xz/
│ │ ├── BUILD
│ │ ├── xz_error.cc
│ │ ├── xz_error.h
│ │ ├── xz_reader.cc
│ │ ├── xz_reader.h
│ │ ├── xz_writer.cc
│ │ └── xz_writer.h
│ ├── zlib/
│ │ ├── BUILD
│ │ ├── zlib_dictionary.h
│ │ ├── zlib_error.cc
│ │ ├── zlib_error.h
│ │ ├── zlib_reader.cc
│ │ ├── zlib_reader.h
│ │ ├── zlib_writer.cc
│ │ └── zlib_writer.h
│ └── zstd/
│ ├── BUILD
│ ├── zstd_dictionary.cc
│ ├── zstd_dictionary.h
│ ├── zstd_reader.cc
│ ├── zstd_reader.h
│ ├── zstd_writer.cc
│ └── zstd_writer.h
└── tf_dependency/
├── BUILD
├── BUILD.tpl
└── tf_configure.bzl
SYMBOL INDEX (1901 symbols across 421 files)
FILE: python/riegeli/base/riegeli_error.py
class RiegeliError (line 36) | class RiegeliError(Exception):
class CancelledError (line 49) | class CancelledError(RiegeliError):
class UnknownError (line 55) | class UnknownError(RiegeliError):
class InvalidArgumentError (line 67) | class InvalidArgumentError(RiegeliError):
class DeadlineExceededError (line 78) | class DeadlineExceededError(RiegeliError):
class NotFoundError (line 90) | class NotFoundError(RiegeliError):
class AlreadyExistsError (line 103) | class AlreadyExistsError(RiegeliError):
class PermissionDeniedError (line 109) | class PermissionDeniedError(RiegeliError):
class UnauthenticatedError (line 123) | class UnauthenticatedError(RiegeliError):
class ResourceExhaustedError (line 129) | class ResourceExhaustedError(RiegeliError):
class FailedPreconditionError (line 139) | class FailedPreconditionError(RiegeliError):
class AbortedError (line 162) | class AbortedError(RiegeliError):
class OutOfRangeError (line 175) | class OutOfRangeError(RiegeliError):
class UnimplementedError (line 195) | class UnimplementedError(RiegeliError):
class InternalError (line 204) | class InternalError(RiegeliError):
class UnavailableError (line 214) | class UnavailableError(RiegeliError):
class DataLossError (line 227) | class DataLossError(RiegeliError):
FILE: python/riegeli/base/utils.cc
type riegeli::python (line 46) | namespace riegeli::python {
function Exception (line 48) | Exception& Exception::operator=(const Exception& that) noexcept {
function Exception (line 59) | Exception Exception::Fetch() {
function PyObject (line 69) | PyObject* Exception::Restore() const& {
function PyObject (line 78) | PyObject* Exception::Restore() && {
function SetRiegeliError (line 109) | void SetRiegeliError(const absl::Status& status) {
type py_internal (line 152) | namespace py_internal {
function FreeStaticObjectsImpl (line 163) | void FreeStaticObjectsImpl() {
function FreeStaticObjects (line 176) | static void FreeStaticObjects() { FreeStaticObjectsImpl(); }
function ExportCapsule (line 224) | bool ExportCapsule(PyObject* module, const char* capsule_name,
function PyObject (line 255) | PyObject* MemoryView::ToPython(absl::string_view value) {
function PyObject (line 265) | PyObject* MemoryView::MutableToPython(absl::Span<char> value) {
function PythonPtr (line 310) | PythonPtr ChainToPython(const Chain& value) {
function ChainFromPython (line 318) | std::optional<Chain> ChainFromPython(PyObject* object) {
function PythonPtr (line 330) | PythonPtr SizeToPython(size_t value) {
function SizeFromPython (line 340) | std::optional<size_t> SizeFromPython(PyObject* object) {
function PythonPtr (line 358) | PythonPtr PositionToPython(Position value) {
function PositionFromPython (line 369) | std::optional<Position> PositionFromPython(PyObject* object) {
function PythonPtr (line 388) | PythonPtr PartialOrderingToPython(PartialOrdering ordering) {
function PartialOrderingFromPython (line 395) | std::optional<PartialOrdering> PartialOrderingFromPython(PyObject* obj...
FILE: python/riegeli/base/utils.h
function class (line 50) | class PythonLock {
function class (line 70) | class PythonUnlock {
type Deleter (line 98) | struct Deleter {
type LockingDeleter (line 110) | struct LockingDeleter {
function ABSL_ATTRIBUTE_REINITIALIZES (line 146) | ABSL_ATTRIBUTE_REINITIALIZES void reset() {
function T (line 155) | T* get() ABSL_ATTRIBUTE_LIFETIME_BOUND {
function T (line 159) | const T* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function namespace (line 222) | namespace py_internal {
function class (line 278) | class Identifier : public py_internal::StaticObject {
function class (line 318) | class ImportedConstant : public py_internal::StaticObject {
function T (line 379) | const T* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function PythonPtr (line 394) | inline PythonPtr IntToPython(long value) {
function PythonPtr (line 401) | inline PythonPtr BytesToPython(absl::string_view value) {
function class (line 410) | class MemoryView {
function class (line 451) | class BytesLike {
FILE: python/riegeli/bytes/python_reader.cc
type riegeli::python (line 45) | namespace riegeli::python {
FILE: python/riegeli/bytes/python_reader.h
function namespace (line 39) | namespace riegeli::python {
function Traverse (line 174) | inline int PythonReader::Traverse(visitproc visit, void* arg) {
FILE: python/riegeli/bytes/python_writer.cc
type riegeli::python (line 44) | namespace riegeli::python {
FILE: python/riegeli/bytes/python_writer.h
function namespace (line 37) | namespace riegeli::python {
function Traverse (line 164) | inline int PythonWriter::Traverse(visitproc visit, void* arg) {
FILE: python/riegeli/records/examples/write_read_records.py
function sample_string (line 22) | def sample_string(i, size):
function sample_message (line 28) | def sample_message(i, size):
function write_records (line 32) | def write_records(filename):
function read_records (line 42) | def read_records(filename):
function main (line 58) | def main():
FILE: python/riegeli/records/record_position.cc
type riegeli::python (line 42) | namespace riegeli::python {
type PyRecordPositionObject (line 46) | struct PyRecordPositionObject {
function RecordPositionDestructor (line 62) | static void RecordPositionDestructor(PyRecordPositionObject* self) {
function PyRecordPositionObject (line 67) | static PyRecordPositionObject* RecordPositionNew(PyTypeObject* cls,
function PyObject (line 101) | static PyObject* RecordPositionChunkBegin(PyRecordPositionObject* self,
function PyObject (line 108) | static PyObject* RecordPositionRecordIndex(PyRecordPositionObject* self,
function PyObject (line 115) | static PyObject* RecordPositionNumeric(PyRecordPositionObject* self,
function PyObject (line 122) | static PyObject* RecordPositionCompare(PyObject* a, PyObject* b, int o...
function Py_hash_t (line 155) | static Py_hash_t RecordPositionHash(PyRecordPositionObject* self) {
function PyObject (line 163) | static PyObject* RecordPositionStr(PyRecordPositionObject* self) {
function PyRecordPositionObject (line 169) | static PyRecordPositionObject* RecordPositionFromStr(PyTypeObject* cls,
function PyObject (line 195) | static PyObject* RecordPositionRepr(PyRecordPositionObject* self) {
function PyObject (line 203) | static PyObject* RecordPositionToBytes(PyRecordPositionObject* self,
function PyRecordPositionObject (line 210) | static PyRecordPositionObject* RecordPositionFromBytes(PyTypeObject* cls,
function PythonPtr (line 380) | PythonPtr RecordPositionToPython(FutureRecordPosition value) {
function RecordPositionFromPython (line 388) | std::optional<RecordPosition> RecordPositionFromPython(PyObject* objec...
function PyObject (line 416) | PyObject* InitModule() {
function PyMODINIT_FUNC (line 441) | PyMODINIT_FUNC PyInit_record_position() { return InitModule(); }
FILE: python/riegeli/records/record_position.h
function namespace (line 30) | namespace riegeli::python {
FILE: python/riegeli/records/record_reader.cc
type riegeli::python (line 46) | namespace riegeli::python {
function PyObject (line 58) | static PyObject* GetRecordType(PyObject* self, PyObject* args,
type PyRecordReaderObject (line 121) | struct PyRecordReaderObject {
type PyRecordIterObject (line 134) | struct PyRecordIterObject {
function RecordReaderHasException (line 147) | bool RecordReaderHasException(PyRecordReaderObject* self) {
function SetExceptionFromRecordReader (line 151) | void SetExceptionFromRecordReader(PyRecordReaderObject* self) {
function VerifyFieldNumber (line 166) | std::optional<int> VerifyFieldNumber(long field_number_value) {
function FieldNumberFromPython (line 178) | std::optional<int> FieldNumberFromPython(PyObject* object) {
function FieldProjectionFromPython (line 191) | std::optional<FieldProjection> FieldProjectionFromPython(PyObject* obj...
function RecordReaderDestructor (line 218) | static void RecordReaderDestructor(PyRecordReaderObject* self) {
function RecordReaderTraverse (line 232) | static int RecordReaderTraverse(PyRecordReaderObject* self, visitproc ...
function RecordReaderClear (line 248) | static int RecordReaderClear(PyRecordReaderObject* self) {
function RecordReaderInit (line 255) | static int RecordReaderInit(PyRecordReaderObject* self, PyObject* args,
function PyObject (line 382) | static PyObject* RecordReaderSrc(PyRecordReaderObject* self, void* clo...
function PyObject (line 390) | static PyObject* RecordReaderRepr(PyRecordReaderObject* self) {
function PyObject (line 402) | static PyObject* RecordReaderEnter(PyObject* self, PyObject* args) {
function PyObject (line 408) | static PyObject* RecordReaderExit(PyRecordReaderObject* self, PyObject...
function PyObject (line 428) | static PyObject* RecordReaderClose(PyRecordReaderObject* self, PyObjec...
function PyObject (line 440) | static PyObject* RecordReaderCheckFileFormat(PyRecordReaderObject* self,
function PyObject (line 455) | static PyObject* RecordReaderReadMetadata(PyRecordReaderObject* self,
function PyObject (line 504) | static PyObject* RecordReaderReadSerializedMetadata(PyRecordReaderObje...
function PyObject (line 520) | static PyObject* RecordReaderReadRecord(PyRecordReaderObject* self,
function PyObject (line 536) | static PyObject* RecordReaderReadMessage(PyRecordReaderObject* self,
function PyRecordIterObject (line 592) | static PyRecordIterObject* RecordReaderReadRecords(PyRecordReaderObjec...
function PyRecordIterObject (line 606) | static PyRecordIterObject* RecordReaderReadMessages(PyRecordReaderObje...
function PyObject (line 629) | static PyObject* RecordReaderSetFieldProjection(PyRecordReaderObject* ...
function PyObject (line 658) | static PyObject* RecordReaderLastPos(PyRecordReaderObject* self,
function PyObject (line 671) | static PyObject* RecordReaderPos(PyRecordReaderObject* self, void* clo...
function PyObject (line 678) | static PyObject* RecordReaderSupportsRandomAccess(PyRecordReaderObject...
function PyObject (line 684) | static PyObject* RecordReaderSeek(PyRecordReaderObject* self, PyObject...
function PyObject (line 706) | static PyObject* RecordReaderSeekNumeric(PyRecordReaderObject* self,
function PyObject (line 727) | static PyObject* RecordReaderSeekBack(PyRecordReaderObject* self,
function PyObject (line 742) | static PyObject* RecordReaderSize(PyRecordReaderObject* self, PyObject...
function PyObject (line 753) | static PyObject* RecordReaderSearch(PyRecordReaderObject* self, PyObje...
function PyObject (line 793) | static PyObject* RecordReaderSearchForRecord(PyRecordReaderObject* self,
function PyObject (line 843) | static PyObject* RecordReaderSearchForMessage(PyRecordReaderObject* self,
function RecordIterDestructor (line 1385) | static void RecordIterDestructor(PyRecordIterObject* self) {
function RecordIterTraverse (line 1398) | static int RecordIterTraverse(PyRecordIterObject* self, visitproc visit,
function RecordIterClear (line 1405) | static int RecordIterClear(PyRecordIterObject* self) {
function PyObject (line 1411) | static PyObject* RecordIterNext(PyRecordIterObject* self) {
function PyObject (line 1511) | PyObject* InitModule() {
function PyMODINIT_FUNC (line 1537) | PyMODINIT_FUNC PyInit_record_reader() { return InitModule(); }
FILE: python/riegeli/records/record_writer.cc
type riegeli::python (line 39) | namespace riegeli::python {
function PythonPtr (line 48) | PythonPtr DefineFlushType() {
function FlushTypeFromPython (line 61) | bool FlushTypeFromPython(PyObject* object, FlushType* value) {
class FileDescriptorCollector (line 78) | class FileDescriptorCollector {
method Init (line 80) | bool Init(PyObject* file_descriptors) {
method AddFile (line 86) | bool AddFile(PyObject* file_descriptor) {
function PyObject (line 133) | static PyObject* SetRecordType(PyObject* self, PyObject* args,
type PyRecordWriterObject (line 190) | struct PyRecordWriterObject {
function SetExceptionFromRecordWriter (line 201) | void SetExceptionFromRecordWriter(PyRecordWriterObject* self) {
function RecordWriterDestructor (line 217) | static void RecordWriterDestructor(PyRecordWriterObject* self) {
function RecordWriterTraverse (line 225) | static int RecordWriterTraverse(PyRecordWriterObject* self, visitproc ...
function RecordWriterClear (line 233) | static int RecordWriterClear(PyRecordWriterObject* self) {
function RecordWriterInit (line 238) | static int RecordWriterInit(PyRecordWriterObject* self, PyObject* args,
function PyObject (line 347) | static PyObject* RecordWriterDest(PyRecordWriterObject* self, void* cl...
function PyObject (line 355) | static PyObject* RecordWriterRepr(PyRecordWriterObject* self) {
function PyObject (line 367) | static PyObject* RecordWriterEnter(PyObject* self, PyObject* args) {
function PyObject (line 373) | static PyObject* RecordWriterExit(PyRecordWriterObject* self, PyObject...
function PyObject (line 393) | static PyObject* RecordWriterClose(PyRecordWriterObject* self, PyObjec...
function PyObject (line 405) | static PyObject* RecordWriterWriteRecord(PyRecordWriterObject* self,
function PyObject (line 426) | static PyObject* RecordWriterWriteMessage(PyRecordWriterObject* self,
function PyObject (line 454) | static PyObject* RecordWriterWriteRecords(PyRecordWriterObject* self,
function PyObject (line 484) | static PyObject* RecordWriterWriteMessages(PyRecordWriterObject* self,
function PyObject (line 518) | static PyObject* RecordWriterFlush(PyRecordWriterObject* self, PyObjec...
function PyObject (line 543) | static PyObject* RecordWriterLastPos(PyRecordWriterObject* self,
function PyObject (line 556) | static PyObject* RecordWriterPos(PyRecordWriterObject* self, void* clo...
function PyObject (line 563) | static PyObject* RecordWriterEstimatedSize(PyRecordWriterObject* self,
function PyObject (line 878) | PyObject* InitModule() {
function PyMODINIT_FUNC (line 901) | PyMODINIT_FUNC PyInit_record_writer() { return InitModule(); }
FILE: python/riegeli/records/skipped_region.py
class SkippedRegion (line 19) | class SkippedRegion:
method __init__ (line 31) | def __init__(self, begin, end, message):
method length (line 39) | def length(self):
method __str__ (line 42) | def __str__(self):
method __repr__ (line 45) | def __repr__(self):
FILE: python/riegeli/records/tests/records_test.py
function combine_named_parameters (line 29) | def combine_named_parameters(*testcase_sets):
class RandomAccess (line 45) | class RandomAccess(Enum):
class FakeFile (line 51) | class FakeFile:
method __init__ (line 54) | def __init__(self, random_access):
method seekable (line 57) | def seekable(self):
method tell (line 60) | def tell(self):
method __getattr__ (line 65) | def __getattr__(self, name):
class UnseekableWrapper (line 69) | class UnseekableWrapper:
method __init__ (line 72) | def __init__(self, wrapped):
method seekable (line 75) | def seekable(self):
method tell (line 78) | def tell(self, *args):
method seek (line 81) | def seek(self, *args):
method __getattr__ (line 84) | def __getattr__(self, name):
class FileSpecBase (line 88) | class FileSpecBase(metaclass=abc.ABCMeta):
method __init__ (line 91) | def __init__(self, create_tempfile, random_access):
method _open_for_writing (line 96) | def _open_for_writing(self):
method writing_open (line 99) | def writing_open(self):
method writing_should_close (line 108) | def writing_should_close(self):
method writing_assumed_pos (line 112) | def writing_assumed_pos(self):
method _open_for_reading (line 118) | def _open_for_reading(self):
method reading_open (line 121) | def reading_open(self):
method reading_should_close (line 130) | def reading_should_close(self):
method reading_assumed_pos (line 134) | def reading_assumed_pos(self):
method close (line 139) | def close(self):
class BytesIOSpec (line 143) | class BytesIOSpec(FileSpecBase):
method _open_for_writing (line 146) | def _open_for_writing(self):
method writing_should_close (line 150) | def writing_should_close(self):
method _open_for_reading (line 154) | def _open_for_reading(self):
class LocalFileSpecBase (line 160) | class LocalFileSpecBase(FileSpecBase):
method __init__ (line 163) | def __init__(self, create_tempfile, random_access):
class FileIOSpec (line 168) | class FileIOSpec(LocalFileSpecBase):
method _open_for_writing (line 171) | def _open_for_writing(self):
method _open_for_reading (line 174) | def _open_for_reading(self):
class BufferedIOSpec (line 178) | class BufferedIOSpec(LocalFileSpecBase):
method _open_for_writing (line 181) | def _open_for_writing(self):
method _open_for_reading (line 184) | def _open_for_reading(self):
class BuiltinFileSpec (line 188) | class BuiltinFileSpec(LocalFileSpecBase):
method _open_for_writing (line 191) | def _open_for_writing(self):
method _open_for_reading (line 194) | def _open_for_reading(self):
function sample_string (line 198) | def sample_string(i, size):
function sample_message (line 204) | def sample_message(i, size):
function sample_message_id_only (line 208) | def sample_message_id_only(i):
function sample_invalid_message (line 212) | def sample_invalid_message(size):
function record_writer_options (line 216) | def record_writer_options(parallelism, transpose=False, chunk_size=35000):
class RecordsTest (line 261) | class RecordsTest(parameterized.TestCase):
method corrupt_at (line 263) | def corrupt_at(self, files, index):
method test_record_writer_exception_from_file (line 279) | def test_record_writer_exception_from_file(self, random_access, parall...
method test_record_reader_exception_from_file (line 294) | def test_record_reader_exception_from_file(self, random_access):
method test_write_read_record (line 309) | def test_write_read_record(self, file_spec, random_access, parallelism):
method test_write_read_message (line 348) | def test_write_read_message(self, file_spec, random_access, parallelism):
method test_write_read_records (line 390) | def test_write_read_records(self, file_spec, random_access, parallelism):
method test_write_read_messages (line 412) | def test_write_read_messages(self, file_spec, random_access, paralleli...
method test_write_read_messages_with_field_projection (line 434) | def test_write_read_messages_with_field_projection(
method test_write_read_messages_with_field_projection_later (line 463) | def test_write_read_messages_with_field_projection_later(
method test_write_read_metadata (line 507) | def test_write_read_metadata(self, file_spec, random_access, paralleli...
method test_invalid_metadata_exception (line 547) | def test_invalid_metadata_exception(
method test_invalid_metadata_recovery (line 570) | def test_invalid_metadata_recovery(
method test_invalid_metadata_recovery_stop_iteration (line 597) | def test_invalid_metadata_recovery_stop_iteration(
method test_field_projection (line 624) | def test_field_projection(self, file_spec, random_access, parallelism):
method test_field_projection_existence_only (line 654) | def test_field_projection_existence_only(
method test_seek (line 694) | def test_seek(self, file_spec, parallelism):
method test_seek_numeric (line 758) | def test_seek_numeric(self, file_spec, parallelism):
method test_seek_back (line 822) | def test_seek_back(self, file_spec):
method test_search (line 849) | def test_search(self, file_spec):
method test_search_for_record (line 890) | def test_search_for_record(self, file_spec):
method test_search_for_record_stop_iteration (line 931) | def test_search_for_record_stop_iteration(self, file_spec):
method test_search_for_message (line 957) | def test_search_for_message(self, file_spec):
method test_search_for_message_stop_iteration (line 1018) | def test_search_for_message_stop_iteration(self, file_spec):
method test_search_for_invalid_message_exception (line 1046) | def test_search_for_invalid_message_exception(self, file_spec):
method test_search_for_invalid_message_recovery (line 1081) | def test_search_for_invalid_message_recovery(self, file_spec):
method test_search_for_invalid_message_recovery_stop_iteration (line 1129) | def test_search_for_invalid_message_recovery_stop_iteration(self, file...
method test_corruption_exception (line 1175) | def test_corruption_exception(self, file_spec, random_access, parallel...
method test_corruption_recovery (line 1205) | def test_corruption_recovery(self, file_spec, random_access, paralleli...
method test_corruption_recovery_stop_iteration (line 1241) | def test_corruption_recovery_stop_iteration(
method test_corruption_recovery_exception (line 1281) | def test_corruption_recovery_exception(
method test_invalid_message_exception (line 1316) | def test_invalid_message_exception(
method test_invalid_message_recovery (line 1366) | def test_invalid_message_recovery(
method test_invalid_message_recovery_stop_iteration (line 1428) | def test_invalid_message_recovery_stop_iteration(
FILE: python/riegeli/tensorflow/kernel_tests/riegeli_dataset_test.py
class DatasetTestBase (line 33) | class DatasetTestBase(test.TestCase):
method assertValuesEqual (line 36) | def assertValuesEqual(self, expected, actual):
method getNext (line 45) | def getNext(self, dataset, requires_initialization=False, shared_name=...
method _compareOutputToExpected (line 92) | def _compareOutputToExpected(
method assertDatasetProduces (line 105) | def assertDatasetProduces(
class RiegeliDatasetTest (line 181) | class RiegeliDatasetTest(DatasetTestBase):
method setUp (line 183) | def setUp(self):
method dataset_fn (line 190) | def dataset_fn(self, filenames, num_epochs=1, batch_size=None):
method _record (line 198) | def _record(self, f, r):
method _create_files (line 201) | def _create_files(self):
method test_read_one_epoch (line 214) | def test_read_one_epoch(self):
method test_read_ten_epochs (line 238) | def test_read_ten_epochs(self):
method test_read_ten_epochs_of_batches (line 247) | def test_read_ten_epochs_of_batches(self):
FILE: python/riegeli/tensorflow/ops/riegeli_dataset_ops.py
class RiegeliDataset (line 31) | class RiegeliDataset(dataset_ops.DatasetSource):
method __init__ (line 36) | def __init__(
method element_spec (line 78) | def element_spec(self):
FILE: python/setup.py
class BinaryDistribution (line 23) | class BinaryDistribution(dist.Distribution):
method has_ext_modules (line 26) | def has_ext_modules(self):
FILE: riegeli/base/any.h
function namespace (line 48) | namespace riegeli {
function friend (line 125) | friend void RiegeliRegisterSubobjects(const AnyBase* self,
function namespace (line 554) | namespace any_internal {
FILE: riegeli/base/any_initializer.h
function namespace (line 34) | namespace riegeli {
FILE: riegeli/base/any_internal.h
function namespace (line 37) | namespace riegeli {
function Handle (line 229) | Handle SentinelHandle() {
function Move (line 240) | static void Move(ABSL_ATTRIBUTE_UNUSED Storage src,
function IsOwning (line 246) | static bool IsOwning(ABSL_ATTRIBUTE_UNUSED const Storage self) {
function RegisterSubobjects (line 249) | static void RegisterSubobjects(
function Destroy (line 267) | static void Destroy(ABSL_ATTRIBUTE_UNUSED Storage self) {}
function Move (line 268) | static void Move(Storage src, Storage dest,
function IsOwning (line 274) | static bool IsOwning(const Storage self) { return dep_ptr(self)->IsOwnin...
function TypeErasedRef (line 275) | static TypeErasedRef GetRawManager(const Storage self) {
function RegisterSubobjects (line 278) | static void RegisterSubobjects(
function Destroy (line 311) | static void Destroy(Storage self) { delete dep_ptr(self); }
function Move (line 312) | static void Move(Storage src, Storage dest,
function MakeReference (line 318) | static void MakeReference(Storage src, Storage dest,
function IsOwning (line 325) | static bool IsOwning(const Storage self) { return dep_ptr(self)->IsOwnin...
function TypeErasedRef (line 326) | static TypeErasedRef GetRawManager(const Storage self) {
function RegisterSubobjects (line 329) | static void RegisterSubobjects(const Storage self,
function Destroy (line 369) | static void Destroy(Storage self) {
function Move (line 372) | static void Move(Storage src, Storage dest,
function MoveToHeap (line 379) | static void MoveToHeap(Storage src, Storage dest,
function MakeReference (line 388) | static void MakeReference(Storage src, Storage dest,
function IsOwning (line 395) | static bool IsOwning(const Storage self) { return dep(self).IsOwning(); }
function TypeErasedRef (line 396) | static TypeErasedRef GetRawManager(const Storage self) {
function RegisterSubobjects (line 399) | static void RegisterSubobjects(const Storage self,
FILE: riegeli/base/arithmetic.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 30) | ABSL_POINTERS_DEFAULT_NONNULL
function A (line 148) | A SignedMin(A a) {
function A (line 173) | A SignedMax(A a) {
function A (line 198) | A UnsignedMin(A a) {
function A (line 223) | A UnsignedMax(A a) {
function A (line 266) | A SaturatingIntCast(B value) {
function A (line 276) | A SaturatingIntCast(B value) {
function A (line 288) | A SaturatingIntCast(B value) {
function A (line 299) | A SaturatingIntCast(B value) {
function T (line 313) | T SaturatingAdd() {
function T (line 318) | T SaturatingAdd(T a) {
function T (line 323) | T SaturatingAdd(T a, T b) {
function T (line 335) | T SaturatingAdd(T a, T b, Rest... rest) {
function T (line 344) | T SaturatingSub(T a, U b) {
function T (line 357) | T RoundDown(T value) {
FILE: riegeli/base/assert.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 23) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/assert.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 32) | ABSL_POINTERS_DEFAULT_NONNULL
function AssertInitialized (line 506) | inline void AssertInitialized(ABSL_ATTRIBUTE_UNUSED const char* data,
function MarkPoisoned (line 515) | inline void MarkPoisoned(ABSL_ATTRIBUTE_UNUSED const char* data,
FILE: riegeli/base/background_cleaning.cc
type riegeli (line 25) | namespace riegeli {
type Args (line 58) | struct Args {
function ABSL_EXCLUSIVE_LOCKS_REQUIRED (line 114) | ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
FILE: riegeli/base/background_cleaning.h
function namespace (line 26) | namespace riegeli {
FILE: riegeli/base/binary_search.h
function namespace (line 28) | namespace riegeli {
function T (line 309) | T Next(T current) const { return current + 1; }
function Empty (line 312) | bool Empty(T low, T high) const { return low >= high; }
function namespace (line 325) | namespace binary_search_internal {
function PartialOrdering (line 350) | static PartialOrdering DoCancel(ABSL_ATTRIBUTE_UNUSED const Pos& pos) {
function PartialOrdering (line 353) | static PartialOrdering DoNotCancel(Ordering ordering) {
function SearchGuide (line 363) | static SearchGuide<Pos> DoNotCancel(SearchGuide<OtherPos>&& guide) {
FILE: riegeli/base/buffer.cc
type riegeli (line 22) | namespace riegeli {
FILE: riegeli/base/buffer.h
function namespace (line 31) | namespace riegeli {
function Reset (line 109) | inline void Buffer::Reset(size_t min_capacity) {
function AllocateInternal (line 119) | inline void Buffer::AllocateInternal(size_t min_capacity) {
FILE: riegeli/base/buffering.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 27) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/byte_fill.cc
type riegeli (line 41) | namespace riegeli {
FILE: riegeli/base/byte_fill.h
function namespace (line 41) | namespace riegeli {
function ExternalDelegate (line 563) | void ByteFill::Blocks::ExternalDelegate(absl::string_view substr,
FILE: riegeli/base/bytes_ref.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 35) | ABSL_POINTERS_DEFAULT_NONNULL
function class (line 140) | class BytesInitializer : public Initializer<std::string> {
FILE: riegeli/base/c_string_ref.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 31) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/chain.cc
type riegeli (line 53) | namespace riegeli {
class FlatCordBlock (line 62) | class FlatCordBlock {
method FlatCordBlock (line 66) | FlatCordBlock(FlatCordBlock&& that) = default;
method FlatCordBlock (line 67) | FlatCordBlock& operator=(FlatCordBlock&& that) = default;
method RiegeliDumpStructure (line 74) | void RiegeliDumpStructure(
method RiegeliRegisterSubobjects (line 81) | void RiegeliRegisterSubobjects(const FlatCordBlock* self,
type chain_internal (line 109) | namespace chain_internal {
function DumpStructureDefault (line 111) | void DumpStructureDefault(std::ostream& dest) { dest << "[external] ...
function RiegeliDumpStructure (line 115) | void RiegeliDumpStructure(const std::string* self, std::ostream& dest) {
function Chain (line 355) | Chain& Chain::operator=(const Chain& that) {
function swap (line 1942) | void swap(Chain& a, Chain& b) noexcept {
function StrongOrdering (line 1958) | StrongOrdering Chain::Compare(const Chain& a, const Chain& b) {
function StrongOrdering (line 1996) | StrongOrdering Chain::Compare(const Chain& a, absl::string_view b) {
FILE: riegeli/base/chain_base.h
function namespace (line 51) | namespace riegeli {
FILE: riegeli/base/chain_details.h
function namespace (line 54) | namespace riegeli {
function DynamicSizeOf (line 506) | size_t Chain::ExternalMethodsFor<T>::DynamicSizeOf() {
function kInternalAllocatedOffset (line 537) | constexpr size_t Chain::RawBlock::kInternalAllocatedOffset() {
function kExternalObjectOffset (line 542) | size_t Chain::RawBlock::kExternalObjectOffset() {
function kExternalAllocatedSize (line 548) | size_t Chain::RawBlock::kExternalAllocatedSize() {
function Unref (line 559) | void Chain::RawBlock::Unref() {
function T (line 599) | const T* Chain::RawBlock::checked_external_object() const {
function TryClear (line 614) | inline bool Chain::RawBlock::TryClear() {
function TryRemoveSuffix (line 622) | inline bool Chain::RawBlock::TryRemoveSuffix(size_t length) {
function TryRemovePrefix (line 633) | inline bool Chain::RawBlock::TryRemovePrefix(size_t length) {
function Chain (line 652) | inline const Chain::BlockPtr* Chain::BlockPtrPtr::as_ptr() const {
function ExternalDelegate (line 719) | void Chain::BlockRef::ExternalDelegate(absl::string_view substr,
function T (line 729) | const T* Chain::BlockRef::external_object() const {
function Chain (line 746) | inline Chain::BlockIterator::BlockIterator(const Chain* chain, BlockPtrP...
function else (line 752) | else if (ptr_ == kEndShortData) {
function CharIndexInChain (line 759) | inline size_t Chain::BlockIterator::CharIndexInChain(
FILE: riegeli/base/closing_ptr.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 23) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/compact_string.cc
type riegeli (line 30) | namespace riegeli {
FILE: riegeli/base/compact_string.h
function namespace (line 41) | namespace riegeli {
function StrongOrdering (line 302) | StrongOrdering RIEGELI_COMPARE(const CompactString& a, T&& b) {
function HashState (line 307) | HashState AbslHashValue(HashState hash_state,
function friend (line 324) | friend void AbslFormatFlush(CompactString* dest, absl::string_view src) {
function friend (line 331) | friend void RiegeliSupportsExternalRef(CompactString*) {}
function friend (line 334) | friend bool RiegeliExternalCopy(const CompactString* self) {
function friend (line 339) | friend ExternalStorage RiegeliToExternalStorage(CompactString* self) {
function friend (line 353) | friend void RiegeliDumpStructure(const CompactString* self,
function RiegeliRegisterSubobjects (line 361) | void RiegeliRegisterSubobjects(const CompactString* self,
type FromReprTag (line 367) | struct FromReprTag {
function explicit (line 371) | explicit CompactString(FromReprTag, uintptr_t raw) : repr_(raw) {}
function inline_size (line 406) | static size_t inline_size(uintptr_t repr) {
function allocated_size_for_tag (line 428) | size_t allocated_size_for_tag(uintptr_t tag) const {
function allocated_size_for_tag (line 432) | static size_t allocated_size_for_tag(uintptr_t tag, uintptr_t repr) {
function allocated_size (line 445) | size_t allocated_size(uintptr_t repr) {
function set_allocated_size (line 470) | void set_allocated_size(size_t size, uintptr_t repr) {
function allocated_capacity_for_tag (line 481) | size_t allocated_capacity_for_tag(uintptr_t tag) const {
function allocated_capacity_for_tag (line 485) | static size_t allocated_capacity_for_tag(uintptr_t tag, uintptr_t repr) {
function allocated_capacity (line 498) | size_t allocated_capacity(uintptr_t repr) {
function const (line 555) | struct CompactStringHash {
function const (line 564) | struct CompactStringEq {
function const (line 572) | bool operator()(absl::string_view a, const CompactString& b) const {
function const (line 575) | bool operator()(absl::string_view a, absl::string_view b) const {
function MakeRepr (line 582) | inline uintptr_t CompactString::MakeRepr(size_t size, size_t capacity) {
function CompactString (line 628) | inline CompactString::CompactString(const CompactString& that) {
function size (line 678) | inline size_t CompactString::size() const {
function ABSL_ATTRIBUTE_LIFETIME_BOUND (line 695) | inline char& CompactString::operator[](size_t index)
function const (line 702) | inline const char& CompactString::operator[](size_t index) const
function at (line 709) | inline char& CompactString::at(size_t index) ABSL_ATTRIBUTE_LIFETIME_BOU...
function at (line 715) | inline const char& CompactString::at(size_t index) const
function front (line 722) | inline char& CompactString::front() ABSL_ATTRIBUTE_LIFETIME_BOUND {
function set_size (line 746) | inline void CompactString::set_size(size_t new_size) {
function set_allocated_size_for_tag (line 757) | inline void CompactString::set_allocated_size_for_tag(uintptr_t tag,
function resize (line 778) | inline void CompactString::resize(size_t new_size) {
function reserve (line 811) | inline void CompactString::reserve(size_t min_capacity) {
function shrink_to_fit (line 817) | inline void CompactString::shrink_to_fit() {
function append (line 834) | inline void CompactString::append(absl::string_view src) {
function RegisterSubobjects (line 889) | void CompactString::RegisterSubobjects(
FILE: riegeli/base/compare.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 26) | ABSL_POINTERS_DEFAULT_NONNULL
function namespace (line 93) | namespace compare_internal {
function PartialOrdering (line 126) | PartialOrdering AsPartialOrdering(T ordering) {
function PartialOrdering (line 135) | PartialOrdering AsPartialOrdering(T ordering) {
function StrongOrdering (line 149) | StrongOrdering AsStrongOrdering(T ordering) {
function StrongOrdering (line 159) | StrongOrdering AsStrongOrdering(T ordering) {
function StrongOrdering (line 173) | StrongOrdering RIEGELI_COMPARE(A a, B b) {
function PartialOrdering (line 185) | PartialOrdering RIEGELI_COMPARE(A a, B b) {
function StrongOrdering (line 196) | StrongOrdering RIEGELI_COMPARE(T a, T b) {
function StrongOrdering (line 203) | StrongOrdering RIEGELI_COMPARE(T* a, T* b) {
function StrongOrdering (line 209) | inline StrongOrdering RIEGELI_COMPARE(absl::string_view a,
function Ordering (line 291) | Ordering NegateOrdering(Ordering ordering) {
function Ordering (line 306) | Ordering NegateOrdering(Ordering ordering) {
FILE: riegeli/base/constexpr.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 21) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/cord_iterator_span.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 30) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/cord_iterator_span.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 34) | ABSL_POINTERS_DEFAULT_NONNULL
function CordIteratorSpan (line 111) | const CordIteratorSpan& manager() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function Read (line 134) | inline void CordIteratorSpan::Read(absl::Cord::CharIterator& src, size_t...
FILE: riegeli/base/cord_utils.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 30) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/cord_utils.h
function MaxBytesToCopyToCord (line 65) | inline size_t MaxBytesToCopyToCord(absl::Cord& dest) {
function CordBufferSizeForCapacity (line 92) | inline size_t CordBufferSizeForCapacity(size_t capacity) {
FILE: riegeli/base/debug.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 29) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/debug.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 36) | ABSL_POINTERS_DEFAULT_NONNULL
function Write (line 106) | void Write(char src) { dest_->write(&src, 1); }
function Write (line 109) | void Write(absl::string_view src) {
function else (line 139) | else if constexpr (debug_internal::HasOperatorOutput<T>::value) {
function DebugStringQuote (line 161) | void DebugStringQuote() { Write('"'); }
function RiegeliDebug (line 176) | inline void RiegeliDebug(signed char src, DebugStream& dest) {
function src_ (line 269) | DebugType(const T& src) : src_(src) {}
FILE: riegeli/base/dependency.h
function namespace (line 33) | namespace riegeli {
function ABSL_ATTRIBUTE_LIFETIME_BOUND (line 503) | const
function namespace (line 649) | namespace dependency_internal {
function namespace (line 681) | namespace dependency_internal {
function ABSL_ATTRIBUTE_LIFETIME_BOUND (line 721) | const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function AssertNotNull (line 763) | void AssertNotNull(Subhandle handle, absl::string_view message) {
function AssertNotNull (line 770) | void AssertNotNull(ABSL_ATTRIBUTE_UNUSED Subhandle handle,
function namespace (line 795) | namespace dependency_internal {
FILE: riegeli/base/dependency_base.h
function namespace (line 28) | namespace riegeli {
function ABSL_ATTRIBUTE_LIFETIME_BOUND (line 154) | const ABSL_ATTRIBUTE_LIFETIME_BOUND { return manager_; }
function ABSL_ATTRIBUTE_LIFETIME_BOUND (line 164) | const ABSL_ATTRIBUTE_LIFETIME_BOUND {
FILE: riegeli/base/dependency_manager.h
function namespace (line 28) | namespace riegeli {
function T (line 134) | T* ptr() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->manager().ge...
function T (line 159) | T* ptr() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function namespace (line 165) | namespace dependency_manager_internal {
function Manager (line 245) | Manager* ptr() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function Manager (line 307) | Manager* ptr() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function Manager (line 369) | Manager* ptr() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function namespace (line 374) | namespace dependency_manager_internal {
FILE: riegeli/base/errno_mapping.cc
type riegeli (line 37) | namespace riegeli {
function WindowsErrorToStatusCode (line 43) | absl::StatusCode WindowsErrorToStatusCode(DWORD error_number) {
function StatusCodeToErrno (line 115) | int StatusCodeToErrno(absl::StatusCode status_code) {
function WindowsErrorToStatus (line 168) | absl::Status WindowsErrorToStatus(uint32_t error_number,
FILE: riegeli/base/errno_mapping.h
function namespace (line 27) | namespace riegeli {
FILE: riegeli/base/estimated_allocated_size.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 24) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/external_data.cc
type riegeli (line 21) | namespace riegeli {
function ExternalData (line 23) | ExternalData ExternalDataCopy(absl::string_view data) {
FILE: riegeli/base/external_data.h
function namespace (line 23) | namespace riegeli {
FILE: riegeli/base/external_ref_base.h
function namespace (line 41) | namespace riegeli {
function CallOperatorWhole (line 293) | void CallOperatorWhole(ABSL_ATTRIBUTE_UNUSED T&& object) {}
function CallOperatorSubstr (line 336) | void CallOperatorSubstr(
function AssertSubstr (line 348) | void AssertSubstr(const T& object, absl::string_view substr) {
function AssertSubstr (line 366) | static void AssertSubstr(ABSL_ATTRIBUTE_UNUSED const T& object,
function string_view (line 516) | string_view data) {
function string_view (line 526) | string_view substr) {
function substr_ (line 1119) | object_(std::move(object)), substr_(substr) {
function class (line 1468) | class StorageBase {
function StorageWholeWithoutCallOperator (line 1509) | StorageWholeWithoutCallOperator final : public StorageBase {
function StorageWholeWithCallOperator (line 1567) | StorageWholeWithCallOperator final : public StorageBase {
function StorageSubstrWithoutCallOperator (line 1634) | StorageSubstrWithoutCallOperator final : public StorageBase {
function StorageSubstrWithCallOperator (line 1686) | StorageSubstrWithCallOperator final : public StorageBase {
function friend (line 1911) | friend void RiegeliReset(absl::Cord& dest, ExternalRef src) {
function ExternalData (line 1943) | ExternalData result{ExternalStorage(nullptr, nullptr), absl::string_view()}
function explicit (line 1958) | explicit ExternalRef(StorageBase* storage) : storage_(storage) {}
function InitializeTo (line 1962) | void InitializeTo(Chain& dest) && {
function Options (line 2012) | Options options) && {
function Options (line 2038) | Options options) && {
type ChainWithOptions (line 2053) | struct ChainWithOptions {
FILE: riegeli/base/external_ref_support.h
function namespace (line 30) | namespace riegeli {
FILE: riegeli/base/global.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 24) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/hybrid_direct_common.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 27) | ABSL_POINTERS_DEFAULT_NONNULL
function Key (line 100) | Key FromRawKey(RawKey raw_key) {
function Key (line 125) | static Key FromRawKey(RawKey raw_key) {
function Key (line 149) | static Key FromRawKey(RawKey raw_key) {
FILE: riegeli/base/hybrid_direct_internal.h
function Abandon (line 68) | void Abandon() {}
function explicit (line 94) | explicit SizedDeleter(size_t size) : size_(size) {}
function const (line 104) | void operator()(T* ptr) const {
function Index (line 249) | Index operator*() const { return index_; }
FILE: riegeli/base/hybrid_direct_map.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 43) | ABSL_POINTERS_DEFAULT_NONNULL
function Key (line 273) | Key operator()(ElementTypeT<const Src&> entry) const { return entry.firs...
function ABSL_ATTRIBUTE_REINITIALIZES (line 399) | ABSL_ATTRIBUTE_REINITIALIZES void Reset(
function Reset (line 448) | void Reset(
function namespace (line 462) | namespace hybrid_direct_internal {
FILE: riegeli/base/hybrid_direct_set.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 41) | ABSL_POINTERS_DEFAULT_NONNULL
function Key (line 104) | Key operator()(ElementTypeT<const Src&> key) const { return key; }
function Reset (line 196) | void Reset(
function ABSL_ATTRIBUTE_REINITIALIZES (line 202) | ABSL_ATTRIBUTE_REINITIALIZES void Reset(
function Reset (line 215) | void Reset(
function Reset (line 228) | void Reset(
function const_iterator (line 242) | const_iterator cbegin() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function absl_nullable (line 274) | absl_nullable DirectSet CopyDirectSet() const;
function explicit (line 358) | explicit iterator(const HybridDirectSet* set ABSL_ATTRIBUTE_LIFETIME_BOUND,
FILE: riegeli/base/initializer.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 35) | ABSL_POINTERS_DEFAULT_NONNULL
type Methods (line 155) | struct Methods {
function Methods (line 190) | const Methods* methods() const { return methods_; }
type Methods (line 225) | struct Methods
function explicit (line 258) | explicit InitializerAssignableBase(const Methods* methods)
function Methods (line 269) | const Methods* methods() const {
function T (line 287) | T Construct() && { return methods()->construct(context()); }
type Methods (line 361) | struct Methods {
function Methods (line 389) | const Methods* methods() const { return methods_; }
function ABSL_NULLABILITY_COMPATIBLE (line 442) | ABSL_NULLABILITY_COMPATIBLE Initializer
function namespace (line 608) | namespace initializer_internal {
function namespace (line 654) | namespace initializer_internal {
function namespace (line 694) | namespace initializer_internal {
FILE: riegeli/base/intrusive_shared_ptr.h
function namespace (line 33) | namespace riegeli {
function explicit (line 109) | explicit IntrusiveSharedPtr(T* ptr ABSL_ATTRIBUTE_LIFETIME_BOUND,
function explicit (line 112) | explicit IntrusiveSharedPtr(T* ptr ABSL_ATTRIBUTE_LIFETIME_BOUND,
function explicit (line 120) | explicit IntrusiveSharedPtr(Initializer<T> value) : ptr_(std::move(value...
function ptr_ (line 160) | IntrusiveSharedPtr(const IntrusiveSharedPtr& that) noexcept
function ABSL_ATTRIBUTE_REINITIALIZES (line 180) | ABSL_ATTRIBUTE_REINITIALIZES
function T (line 246) | T* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return ptr_.get(); }
type Unrefer (line 308) | struct Unrefer {
function SubT (line 313) | SubT* Ref(SubT* ptr) {
function ResetImpl (line 327) | void ResetImpl(Initializer<T> value) {
FILE: riegeli/base/invoker.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 29) | ABSL_POINTERS_DEFAULT_NONNULL
function operator (line 136) | operator Result() && { return std::move(*this).Invoke(); }
function operator (line 166) | operator Result() && { return std::move(*this).Invoke(); }
function namespace (line 377) | namespace invoker_internal {
FILE: riegeli/base/iterable.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 28) | ABSL_POINTERS_DEFAULT_NONNULL
function namespace (line 217) | namespace iterable_internal {
function Reference (line 283) | const Reference* operator->() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
FILE: riegeli/base/maker.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 31) | ABSL_POINTERS_DEFAULT_NONNULL
function namespace (line 428) | namespace maker_internal {
FILE: riegeli/base/memory_estimator.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 32) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/memory_estimator.h
function namespace (line 51) | namespace riegeli {
function class (line 69) | class MemoryEstimator {
function RegisterDynamicMemoryImpl (line 245) | void RegisterDynamicMemoryImpl(size_t memory) override {
function RegisterUnknownTypeImpl (line 249) | void RegisterUnknownTypeImpl() override {}
function RegisterUnknownTypeImpl (line 250) | void RegisterUnknownTypeImpl(
function RegisterDynamicMemoryImpl (line 276) | void RegisterDynamicMemoryImpl(size_t memory) override {
function RegisterNodeImpl (line 279) | bool RegisterNodeImpl(const void* absl_nullable ptr) override {
function RegisterUnknownTypeImpl (line 282) | void RegisterUnknownTypeImpl() override {}
function RegisterUnknownTypeImpl (line 283) | void RegisterUnknownTypeImpl(
function class (line 295) | class MemoryEstimatorReportingUnknownTypes : public MemoryEstimatorDefau...
function EstimateMemory (line 324) | size_t EstimateMemory(const T& object) {
function EstimateMemorySimplified (line 334) | size_t EstimateMemorySimplified(const T& object) {
type TotalMemoryWithUnknownTypes (line 345) | struct TotalMemoryWithUnknownTypes {
function TotalMemoryWithUnknownTypes (line 351) | TotalMemoryWithUnknownTypes EstimateMemoryReportingUnknownTypes(
FILE: riegeli/base/moving_dependency.h
function namespace (line 27) | namespace riegeli {
function Done (line 160) | void Done(ABSL_ATTRIBUTE_UNUSED Mover& mover,
FILE: riegeli/base/new_aligned.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 31) | ABSL_POINTERS_DEFAULT_NONNULL
function T (line 86) | T* NewAligned(size_t num_bytes, Args&&... args) {
function DeleteAligned (line 101) | void DeleteAligned(T* ptr, size_t num_bytes) {
function T (line 122) | T* SizeReturningNewAligned(size_t min_num_bytes,
FILE: riegeli/base/null_safe_memcpy.h
function null_safe_memcpy (line 65) | __attribute__((no_sanitize("nonnull-attribute")))
function null_safe_memmove (line 76) | __attribute__((no_sanitize("nonnull-attribute")))
function null_safe_memset (line 87) | __attribute__((no_sanitize("nonnull-attribute")))
FILE: riegeli/base/object.cc
type riegeli (line 26) | namespace riegeli {
function ObjectState (line 28) | ObjectState& ObjectState::operator=(ObjectState&& that) noexcept {
function TypeId (line 135) | TypeId Object::GetTypeId() const { return TypeId(); }
FILE: riegeli/base/object.h
function namespace (line 29) | namespace riegeli {
function Target (line 335) | const Target* Object::GetIf() const {
FILE: riegeli/base/optional_compact_string.h
function namespace (line 33) | namespace riegeli {
function explicit (line 66) | explicit OptionalCompactString(const CompactString& src)
function explicit (line 77) | explicit OptionalCompactString(CompactString&& src)
function repr_ (line 85) | OptionalCompactString(const OptionalCompactString& that) noexcept
function CompactString (line 113) | CompactString Release() ABSL_ATTRIBUTE_LIFETIME_BOUND {
function ABSL_ATTRIBUTE_LIFETIME_BOUND (line 130) | const
function StrongOrdering (line 187) | StrongOrdering RIEGELI_COMPARE(const OptionalCompactString& a, T&& b) {
FILE: riegeli/base/options_parser.cc
type riegeli (line 36) | namespace riegeli {
FILE: riegeli/base/options_parser.h
function namespace (line 34) | namespace riegeli {
function Function (line 236) | Function ValueParser::And(Initializer<Function> function1,
function Function (line 243) | Function ValueParser::FailIfSeen(StringInitializer key,
function AddOption (line 249) | inline void OptionsParser::AddOption(
FILE: riegeli/base/ownership.h
function namespace (line 20) | namespace riegeli {
FILE: riegeli/base/parallelism.cc
type riegeli::internal (line 30) | namespace riegeli::internal {
function ThreadPool (line 74) | ThreadPool& ThreadPool::global() {
FILE: riegeli/base/parallelism.h
function namespace (line 27) | namespace riegeli::internal {
FILE: riegeli/base/recycling_pool.h
function namespace (line 42) | namespace riegeli {
FILE: riegeli/base/ref_count.h
function namespace (line 25) | namespace riegeli {
FILE: riegeli/base/reset.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 29) | ABSL_POINTERS_DEFAULT_NONNULL
function else (line 135) | else if constexpr (reset_internal::HasAssignment<T, Args...>::value) {
function else (line 138) | else if constexpr (std::conjunction_v<std::is_constructible<T, Args...>,
FILE: riegeli/base/shared_buffer.cc
type riegeli (line 24) | namespace riegeli {
FILE: riegeli/base/shared_buffer.h
function namespace (line 30) | namespace riegeli {
FILE: riegeli/base/shared_ptr.h
function namespace (line 36) | namespace riegeli {
function T (line 180) | T* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return ptr_.get(); }
type Control (line 258) | struct Control {
type Unrefer (line 265) | struct Unrefer {
function DestroyMethod (line 272) | void DestroyMethod(void* ptr) {
function T (line 281) | T* UpCast(SubT* ptr) {
function ResetImpl (line 372) | void ResetImpl(Initializer<T> value) {
FILE: riegeli/base/sized_shared_buffer.cc
type riegeli (line 30) | namespace riegeli {
FILE: riegeli/base/sized_shared_buffer.h
function class (line 36) | class ABSL_ATTRIBUTE_TRIVIAL_ABI SizedSharedBuffer {
FILE: riegeli/base/stable_dependency.h
function namespace (line 30) | namespace riegeli {
function namespace (line 244) | namespace dependency_internal {
FILE: riegeli/base/status.cc
type riegeli (line 22) | namespace riegeli {
function SetMessage (line 24) | absl::Status SetMessage(const absl::Status& status, absl::string_view ...
function Annotate (line 33) | absl::Status Annotate(const absl::Status& status, absl::string_view de...
FILE: riegeli/base/status.h
function namespace (line 24) | namespace riegeli {
function Status (line 106) | struct StatusOrMaker<absl::Status> {
function type (line 126) | static type FromStatus(const absl::Status& status) { return status; }
function type (line 127) | static type FromStatus(absl::Status&& status) { return std::move(status); }
function Update (line 134) | static void Update(type& result, const absl::Status& status) {
FILE: riegeli/base/stream_utils.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 26) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/stream_utils.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 33) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/string_ref.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 36) | ABSL_POINTERS_DEFAULT_NONNULL
function class (line 152) | class StringInitializer : public Initializer<std::string> {
function at (line 201) | inline const char& StringRef::at(size_t index) const {
function remove_prefix (line 219) | inline void StringRef::remove_prefix(size_t length) {
function remove_suffix (line 226) | inline void StringRef::remove_suffix(size_t length) {
FILE: riegeli/base/string_utils.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 24) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/string_utils.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 27) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/temporary_storage.h
type class (line 99) | enum class
FILE: riegeli/base/type_erased_ref.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 27) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/type_id.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 25) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/type_traits.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 31) | ABSL_POINTERS_DEFAULT_NONNULL
function namespace (line 326) | namespace type_traits_internal {
function false_type (line 404) | struct IsComparableAgainstNullptr<absl::string_view> : std::false_type {}
FILE: riegeli/base/types.h
type class (line 39) | enum class
function SyncType (line 53) | enum class SyncType {
FILE: riegeli/base/unicode.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 36) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/unicode.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 26) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/base/uninitialized_vector.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 27) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/brotli/brotli_allocator.cc
type riegeli (line 19) | namespace riegeli {
type brotli_internal (line 21) | namespace brotli_internal {
function RiegeliBrotliFreeFunc (line 27) | void RiegeliBrotliFreeFunc(void* opaque, void* ptr) {
FILE: riegeli/brotli/brotli_allocator.h
function namespace (line 31) | namespace brotli_internal {
function class (line 46) | class BrotliAllocator {
FILE: riegeli/brotli/brotli_dictionary.cc
type riegeli (line 29) | namespace riegeli {
FILE: riegeli/brotli/brotli_dictionary.h
function class (line 60) | class BrotliDictionary {
FILE: riegeli/brotli/brotli_reader.cc
type riegeli (line 38) | namespace riegeli {
FILE: riegeli/brotli/brotli_reader.h
function namespace (line 36) | namespace riegeli {
function explicit (line 200) | explicit BrotliReader(Closed) -> BrotliReader<DeleteCtad<Closed>>;
function Reset (line 242) | inline void BrotliReaderBase::Reset(BrotliDictionary&& dictionary,
FILE: riegeli/brotli/brotli_writer.cc
type riegeli (line 40) | namespace riegeli {
type BrotliEncoderDictionaryDeleter (line 44) | struct BrotliEncoderDictionaryDeleter {
function Reader (line 215) | Reader* BrotliWriterBase::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/brotli/brotli_writer.h
function namespace (line 38) | namespace riegeli {
function Done (line 172) | void Done() override;
function Writer (line 237) | Writer* DestWriter() const ABSL_ATTRIBUTE_LIFETIME_BOUND override {
function explicit (line 250) | explicit BrotliWriter(Closed) -> BrotliWriter<DeleteCtad<Closed>>;
function Reset (line 295) | inline void BrotliWriterBase::Reset(BufferOptions buffer_options,
FILE: riegeli/bytes/array_backward_writer.cc
type riegeli (line 26) | namespace riegeli {
FILE: riegeli/bytes/array_backward_writer.h
function namespace (line 35) | namespace riegeli {
function Done (line 208) | void Done(ArrayBackwardWriter& self) {
FILE: riegeli/bytes/array_writer.cc
type riegeli (line 32) | namespace riegeli {
function Reader (line 112) | Reader* ArrayWriterBase::ReadModeBehindScratch(Position initial_pos) {
FILE: riegeli/bytes/array_writer.h
function namespace (line 36) | namespace riegeli {
function Done (line 224) | void Done(ArrayWriter& self) {
FILE: riegeli/bytes/backward_writer.cc
type riegeli (line 45) | namespace riegeli {
function WriteUnsigned (line 50) | inline bool WriteUnsigned(T src, BackwardWriter& dest) {
function WriteSigned (line 60) | inline bool WriteSigned(T src, BackwardWriter& dest) {
FILE: riegeli/bytes/backward_writer.h
function WriteByte (line 149) | bool WriteByte(uint8_t src) { return Write(static_cast<char>(src)); }
function Write (line 160) | bool Write(char8_t src) { return Write(static_cast<char>(src)); }
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 163) | ABSL_ATTRIBUTE_ALWAYS_INLINE
function Position (line 279) | Position pos() const;
function Done (line 324) | void Done() override;
function WriteInternal (line 413) | bool WriteInternal(std::tuple<Srcs...>&& srcs) {
function Reset (line 454) | inline void BackwardWriter::Reset(Closed) {
function Reset (line 462) | inline void BackwardWriter::Reset() {
function Close (line 470) | inline bool BackwardWriter::Close() {
function Done (line 475) | inline void BackwardWriter::Done() {
function SetWriteSizeHint (line 480) | inline void BackwardWriter::SetWriteSizeHint(
function Push (line 486) | inline bool BackwardWriter::Push(size_t min_length, size_t recommended_l...
function move_cursor (line 498) | inline void BackwardWriter::move_cursor(size_t length) {
function set_cursor (line 505) | inline void BackwardWriter::set_cursor(char* cursor) {
function set_buffer (line 515) | inline void BackwardWriter::set_buffer(char* limit, size_t start_to_limit,
function Write (line 525) | inline bool BackwardWriter::Write(char src) {
function Write (line 532) | inline bool BackwardWriter::Write(BytesRef src) {
function Write (line 543) | inline bool BackwardWriter::Write(ExternalRef src) {
function Write (line 556) | bool BackwardWriter::Write(Src&& src) {
function Write (line 560) | inline bool BackwardWriter::Write(ByteFill src) {
function Flush (line 591) | inline bool BackwardWriter::Flush(FlushType flush_type) {
function move_start_pos (line 612) | inline void BackwardWriter::move_start_pos(Position length) {
function set_start_pos (line 619) | inline void BackwardWriter::set_start_pos(Position start_pos) {
function Truncate (line 623) | inline bool BackwardWriter::Truncate(Position new_size) {
FILE: riegeli/bytes/buffer_options.cc
type riegeli (line 27) | namespace riegeli {
function ApplySizeHintAndRoundPos (line 40) | inline size_t ApplySizeHintAndRoundPos(Position base_length, size_t mi...
FILE: riegeli/bytes/buffer_options.h
function class (line 192) | class ReadBufferSizer {
function class (line 282) | class WriteBufferSizer {
function ReadBufferSizer (line 351) | inline ReadBufferSizer::ReadBufferSizer(BufferOptions buffer_options)
function Reset (line 354) | inline void ReadBufferSizer::Reset() {
function Reset (line 362) | inline void ReadBufferSizer::Reset(BufferOptions buffer_options) {
function EndRun (line 370) | inline void ReadBufferSizer::EndRun(Position pos) {
function WriteBufferSizer (line 382) | inline WriteBufferSizer::WriteBufferSizer(BufferOptions buffer_options)
function Reset (line 385) | inline void WriteBufferSizer::Reset() {
function Reset (line 392) | inline void WriteBufferSizer::Reset(BufferOptions buffer_options) {
function EndRun (line 399) | inline void WriteBufferSizer::EndRun(Position pos) {
FILE: riegeli/bytes/buffered_reader.cc
type riegeli (line 41) | namespace riegeli {
function SizedSharedBuffer (line 610) | SizedSharedBuffer BufferedReader::SaveBuffer() {
FILE: riegeli/bytes/buffered_reader.h
function namespace (line 31) | namespace riegeli {
function Reset (line 235) | inline void BufferedReader::Reset(Closed) {
function Reset (line 241) | inline void BufferedReader::Reset(BufferOptions buffer_options) {
FILE: riegeli/bytes/buffered_writer.cc
type riegeli (line 36) | namespace riegeli {
function Reader (line 132) | Reader* BufferedWriter::ReadModeBehindBuffer(Position initial_pos) {
function Reader (line 211) | Reader* BufferedWriter::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/buffered_writer.h
function namespace (line 31) | namespace riegeli {
function Reset (line 162) | inline void BufferedWriter::Reset(Closed) {
function Reset (line 169) | inline void BufferedWriter::Reset(BufferOptions buffer_options) {
FILE: riegeli/bytes/cfile_handle.cc
type riegeli (line 47) | namespace riegeli {
type cfile_internal (line 49) | namespace cfile_internal {
class CFileBase<UnownedCFileDeleter> (line 51) | class CFileBase<UnownedCFileDeleter>
class CFileBase<OwnedCFileDeleter> (line 52) | class CFileBase<OwnedCFileDeleter>
function FILE (line 56) | FILE* CFileHandle::GetMethodDefault(
FILE: riegeli/bytes/cfile_handle.h
function namespace (line 40) | namespace riegeli {
function class (line 112) | class CFileHandle : public WithEqual<CFileHandle> {
type Methods (line 164) | struct Methods {
function FILE (line 204) | FILE* GetMethod(TypeErasedRef target) {
function IsOwningMethod (line 209) | bool IsOwningMethod(TypeErasedRef target) {
function namespace (line 245) | namespace cfile_internal {
function FILE (line 450) | FILE* get() const { return file_; }
function FILE (line 486) | FILE* Release() { return std::exchange(file_, nullptr); }
function Destroy (line 492) | void Destroy() {
function CFileBase (line 516) | CFileBase(file) {}
function explicit (line 519) | explicit UnownedCFile(CFileHandle file)
function ABSL_ATTRIBUTE_REINITIALIZES (line 530) | ABSL_ATTRIBUTE_REINITIALIZES void Reset(CFileHandle file) {
function FILE (line 556) | FILE* get() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return CFileBase::get(...
FILE: riegeli/bytes/cfile_internal.cc
type riegeli::cfile_internal (line 24) | namespace riegeli::cfile_internal {
function FilenameForCFile (line 26) | std::string FilenameForCFile(FILE* file) {
FILE: riegeli/bytes/cfile_internal.h
function namespace (line 22) | namespace riegeli::cfile_internal {
FILE: riegeli/bytes/cfile_internal_for_cc.h
function namespace (line 32) | namespace riegeli::cfile_internal {
FILE: riegeli/bytes/cfile_reader.cc
type riegeli (line 57) | namespace riegeli {
type HasAvailableLength (line 70) | struct HasAvailableLength : std::false_type {}
function AvailableLength (line 80) | inline size_t AvailableLength(DependentFILE* src) {
type HasAvailableLength<
DependentFILE,
std::void_t<decltype(std::declval<DependentFILE>()._IO_read_end -
std::declval<DependentFILE>()._IO_read_ptr)>> (line 73) | struct HasAvailableLength<
FILE: riegeli/bytes/cfile_reader.h
function namespace (line 43) | namespace riegeli {
function virtual (line 171) | virtual CFileHandle SrcCFileHandle() const ABSL_ATTRIBUTE_LIFETIME_BOUND...
function FILE (line 313) | FILE* SrcFile() const ABSL_ATTRIBUTE_LIFETIME_BOUND override {
function explicit (line 334) | explicit CFileReader(Closed) -> CFileReader<DeleteCtad<Closed>>;
FILE: riegeli/bytes/cfile_writer.cc
type riegeli (line 63) | namespace riegeli {
function Reader (line 509) | Reader* CFileWriterBase::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/bytes/cfile_writer.h
function namespace (line 45) | namespace riegeli {
function read (line 120) | bool read() const { return file_internal::GetRead(mode_); }
function append (line 145) | bool append() const { return file_internal::GetAppend(mode_); }
function exclusive (line 169) | bool exclusive() const { return file_internal::GetExclusive(mode_); }
function inheritable (line 194) | bool inheritable() const { return file_internal::GetInheritable(mode_); }
function virtual (line 254) | virtual CFileHandle DestCFileHandle() const ABSL_ATTRIBUTE_LIFETIME_BOUN...
function FILE (line 416) | FILE* DestFile() const ABSL_ATTRIBUTE_LIFETIME_BOUND override {
function explicit (line 438) | explicit CFileWriter(Closed) -> CFileWriter<DeleteCtad<Closed>>;
function Reset (line 484) | inline void CFileWriterBase::Reset(Closed) {
function Reset (line 497) | inline void CFileWriterBase::Reset(BufferOptions buffer_options) {
FILE: riegeli/bytes/chain_backward_writer.cc
type riegeli (line 35) | namespace riegeli {
FILE: riegeli/bytes/chain_backward_writer.h
function namespace (line 42) | namespace riegeli {
FILE: riegeli/bytes/chain_reader.cc
type riegeli (line 38) | namespace riegeli {
FILE: riegeli/bytes/chain_reader.h
function namespace (line 36) | namespace riegeli {
function Done (line 191) | void Done(ChainReader& self) {
FILE: riegeli/bytes/chain_writer.cc
type riegeli (line 39) | namespace riegeli {
function Reader (line 414) | Reader* ChainWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/chain_writer.h
function namespace (line 43) | namespace riegeli {
FILE: riegeli/bytes/compact_string_writer.h
function namespace (line 29) | namespace riegeli {
FILE: riegeli/bytes/copy_all.cc
type riegeli::copy_all_internal (line 34) | namespace riegeli::copy_all_internal {
function ABSL_ATTRIBUTE_COLD (line 38) | ABSL_ATTRIBUTE_COLD absl::Status MaxLengthExceeded(Reader& src,
function CopyAllImpl (line 44) | absl::Status CopyAllImpl(Reader& src, Writer& dest, Position max_length,
function CopyAllImpl (line 85) | absl::Status CopyAllImpl(Reader& src, Writer& dest, Position max_length,
function CopyAllImpl (line 101) | absl::Status CopyAllImpl(Reader& src, BackwardWriter& dest, size_t max...
FILE: riegeli/bytes/copy_all.h
function namespace (line 32) | namespace riegeli {
FILE: riegeli/bytes/cord_backward_writer.cc
type riegeli (line 39) | namespace riegeli {
FILE: riegeli/bytes/cord_backward_writer.h
function SupportsTruncate (line 125) | bool SupportsTruncate() override { return true; }
function Done (line 139) | void Done() override;
function explicit (line 246) | explicit CordBackwardWriter(Closed) -> CordBackwardWriter<DeleteCtad<Clo...
FILE: riegeli/bytes/cord_reader.cc
type riegeli (line 37) | namespace riegeli {
FILE: riegeli/bytes/cord_reader.h
function namespace (line 39) | namespace riegeli {
function Done (line 254) | void Done(CordReader& self) {
FILE: riegeli/bytes/cord_writer.cc
type riegeli (line 42) | namespace riegeli {
function Reader (line 488) | Reader* CordWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/cord_writer.h
function SupportsRandomAccess (line 134) | bool SupportsRandomAccess() override { return true; }
function SupportsReadMode (line 135) | bool SupportsReadMode() override { return true; }
function Done (line 149) | void Done() override;
function explicit (line 304) | explicit CordWriter(Closed) -> CordWriter<DeleteCtad<Closed>>;
FILE: riegeli/bytes/fd_handle.cc
type riegeli (line 49) | namespace riegeli {
type fd_internal (line 51) | namespace fd_internal {
class FdBase<UnownedFdDeleter> (line 53) | class FdBase<UnownedFdDeleter>
class FdBase<OwnedFdDeleter> (line 54) | class FdBase<OwnedFdDeleter>
FILE: riegeli/bytes/fd_handle.h
function namespace (line 49) | namespace riegeli {
function class (line 147) | class FdHandle : public WithEqual<FdHandle> {
type Methods (line 197) | struct Methods {
function GetMethod (line 237) | int GetMethod(TypeErasedRef target) {
function IsOwningMethod (line 242) | bool IsOwningMethod(TypeErasedRef target) {
function namespace (line 278) | namespace fd_internal {
function Release (line 521) | int Release() { return std::exchange(fd_, -1); }
function Destroy (line 527) | void Destroy() {
function FdBase (line 550) | UnownedFd(int fd ABSL_ATTRIBUTE_LIFETIME_BOUND) : FdBase(fd) {}
function explicit (line 553) | explicit UnownedFd(FdHandle fd) : FdBase(fd.get(), fd.filename()) {}
function ABSL_ATTRIBUTE_REINITIALIZES (line 563) | ABSL_ATTRIBUTE_REINITIALIZES void Reset(FdHandle fd) {
function get (line 596) | int get() const ABSL_ATTRIBUTE_LIFETIME_BOUND { return FdBase::get(); }
FILE: riegeli/bytes/fd_internal.cc
type riegeli::fd_internal (line 46) | namespace riegeli::fd_internal {
function FilenameForFd (line 48) | std::string FilenameForFd(int fd) {
FILE: riegeli/bytes/fd_internal.h
function namespace (line 24) | namespace riegeli::fd_internal {
FILE: riegeli/bytes/fd_internal_for_cc.h
function namespace (line 33) | namespace riegeli::fd_internal {
FILE: riegeli/bytes/fd_mmap_reader.cc
type riegeli (line 91) | namespace riegeli {
type HandleDeleter (line 97) | struct HandleDeleter {
function GetPageSize (line 112) | inline absl::StatusOr<Position> GetPageSize() {
function Position (line 122) | inline Position GetPageSize() {
type HavePosixFadvise (line 135) | struct HavePosixFadvise : std::false_type {}
type HavePosixFadvise<
DependentInt,
std::void_t<decltype(posix_fadvise(
std::declval<DependentInt>(), std::declval<fd_internal::Offset>(),
std::declval<fd_internal::Offset>(), std::declval<int>()))>> (line 138) | struct HavePosixFadvise<
function FdSetReadAllHint (line 146) | inline void FdSetReadAllHint(ABSL_ATTRIBUTE_UNUSED DependentInt src,
class MMapBlock (line 158) | class MMapBlock {
method MMapBlock (line 160) | explicit MMapBlock(const char* addr) : addr_(addr) {}
method MMapBlock (line 162) | MMapBlock(MMapBlock&& that) = default;
method MMapBlock (line 163) | MMapBlock& operator=(MMapBlock&& that) = default;
method RiegeliSupportsExternalRef (line 166) | void RiegeliSupportsExternalRef(MMapBlock*) {}
method RiegeliDumpStructure (line 172) | void RiegeliDumpStructure(ABSL_ATTRIBUTE_UNUSED const MMapBlock* self,
FILE: riegeli/bytes/fd_mmap_reader.h
function namespace (line 44) | namespace riegeli {
FILE: riegeli/bytes/fd_reader.cc
type riegeli (line 87) | namespace riegeli {
type HaveCopyFileRange (line 111) | struct HaveCopyFileRange : std::false_type {}
type HaveCopyFileRange<
DependentInt,
std::void_t<decltype(copy_file_range(
std::declval<DependentInt>(), std::declval<fd_internal::Offset*>(),
std::declval<int>(), std::declval<fd_internal::Offset*>(),
std::declval<size_t>(), std::declval<unsigned>()))>> (line 114) | struct HaveCopyFileRange<
function CopyFileRange (line 123) | inline ssize_t CopyFileRange(DependentInt src, fd_internal::Offset* sr...
type HavePosixFadvise (line 139) | struct HavePosixFadvise : std::false_type {}
type HavePosixFadvise<
DependentInt,
std::void_t<decltype(posix_fadvise(
std::declval<DependentInt>(), std::declval<fd_internal::Offset>(),
std::declval<fd_internal::Offset>(), std::declval<int>()))>> (line 142) | struct HavePosixFadvise<
function FdSetReadAllHint (line 150) | inline void FdSetReadAllHint(ABSL_ATTRIBUTE_UNUSED DependentInt src,
FILE: riegeli/bytes/fd_reader.h
function namespace (line 47) | namespace riegeli {
function explicit (line 434) | explicit FdReader(Closed) -> FdReader<DeleteCtad<Closed>>;
FILE: riegeli/bytes/fd_writer.cc
type riegeli (line 76) | namespace riegeli {
function TypeId (line 78) | TypeId FdWriterBase::GetTypeId() const { return TypeId::For<FdWriterBa...
function Reader (line 694) | Reader* FdWriterBase::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/bytes/fd_writer.h
function namespace (line 48) | namespace riegeli {
function explicit (line 609) | explicit FdWriter(Closed) -> FdWriter<DeleteCtad<Closed>>;
function Reset (line 659) | inline void FdWriterBase::Reset(Closed) {
function Reset (line 673) | inline void FdWriterBase::Reset(BufferOptions buffer_options) {
FILE: riegeli/bytes/file_mode_string.cc
type riegeli::file_internal (line 35) | namespace riegeli::file_internal {
function SetExisting (line 37) | void SetExisting(bool existing, std::string& mode) {
function SetRead (line 61) | void SetRead(bool read, std::string& mode) {
function GetRead (line 84) | bool GetRead(absl::string_view mode) {
function SetExclusive (line 94) | void SetExclusive(bool exclusive, std::string& mode) {
function GetExclusive (line 122) | bool GetExclusive(absl::string_view mode) {
function SetInheritableImpl (line 133) | void SetInheritableImpl(bool inheritable, std::string& mode) {
function SetInheritableReading (line 168) | void SetInheritableReading(bool inheritable, std::string& mode) {
function SetInheritableWriting (line 173) | void SetInheritableWriting(bool inheritable, std::string& mode) {
function GetInheritable (line 178) | bool GetInheritable(absl::string_view mode) {
function SetTextImpl (line 194) | inline void SetTextImpl(bool text, std::string& mode) {
function SetTextReading (line 226) | void SetTextReading(bool text, std::string& mode) {
function SetTextWriting (line 231) | void SetTextWriting(bool text, std::string& mode) {
function GetTextAsFlags (line 238) | int GetTextAsFlags(absl::string_view mode) {
FILE: riegeli/bytes/file_mode_string.h
function namespace (line 23) | namespace riegeli::file_internal {
FILE: riegeli/bytes/iostream_internal.h
function namespace (line 22) | namespace riegeli::iostream_internal {
FILE: riegeli/bytes/istream_reader.cc
type riegeli (line 36) | namespace riegeli {
FILE: riegeli/bytes/istream_reader.h
function namespace (line 37) | namespace riegeli {
function explicit (line 185) | explicit IStreamReader(Closed) -> IStreamReader<DeleteCtad<Closed>>;
function Reset (line 224) | inline void IStreamReaderBase::Reset(BufferOptions buffer_options,
FILE: riegeli/bytes/joining_reader.cc
type riegeli (line 36) | namespace riegeli {
FILE: riegeli/bytes/joining_reader.h
function namespace (line 35) | namespace riegeli {
function Reset (line 223) | inline void JoiningReaderBase::Reset(Closed) {
function Reset (line 228) | inline void JoiningReaderBase::Reset() {
function shard_is_open (line 233) | inline bool JoiningReaderBase::shard_is_open() const {
function SyncBuffer (line 241) | inline void JoiningReaderBase::SyncBuffer(Reader& shard) {
function MakeBuffer (line 250) | inline void JoiningReaderBase::MakeBuffer(Reader& shard) {
function Done (line 275) | void Done(JoiningReader& self) {
FILE: riegeli/bytes/limiting_backward_writer.cc
type riegeli (line 37) | namespace riegeli {
FILE: riegeli/bytes/limiting_backward_writer.h
function namespace (line 40) | namespace riegeli {
function Reset (line 297) | inline void LimitingBackwardWriterBase::Reset(Closed) {
function Reset (line 303) | inline void LimitingBackwardWriterBase::Reset(bool exact) {
function SyncBuffer (line 309) | inline bool LimitingBackwardWriterBase::SyncBuffer(BackwardWriter& dest) {
function MakeBuffer (line 318) | inline void LimitingBackwardWriterBase::MakeBuffer(BackwardWriter& dest) {
function Done (line 345) | void Done(LimitingBackwardWriter& self) {
FILE: riegeli/bytes/limiting_reader.cc
type riegeli (line 37) | namespace riegeli {
FILE: riegeli/bytes/limiting_reader.h
function namespace (line 40) | namespace riegeli {
function explicit (line 353) | explicit LimitingReader(Closed) -> LimitingReader<DeleteCtad<Closed>>;
function class (line 402) | class ScopedLimiter {
function explicit (line 454) | explicit ScopedLimiterOrLimitingReader(Src* src ABSL_ATTRIBUTE_LIFETIME_...
function explicit (line 458) | explicit ScopedLimiterOrLimitingReader(const ReaderSpan<Src>& src)
function Close (line 475) | bool Close() { return reader_.Close(); }
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 492) | ABSL_ATTRIBUTE_ALWAYS_INLINE
function explicit (line 497) | explicit ScopedLimiterOrLimitingReader(const ReaderSpan<Src>& src)
function Close (line 518) | bool Close() { return reader().ok(); }
function ReaderSpan (line 546) | const ReaderSpan<ReaderType>& manager() const ABSL_ATTRIBUTE_LIFETIME_BO...
function LimitingReaderBase (line 571) | inline LimitingReaderBase::LimitingReaderBase(bool exact, bool fail_if_l...
function Reset (line 590) | inline void LimitingReaderBase::Reset(Closed) {
function Reset (line 597) | inline void LimitingReaderBase::Reset(bool exact, bool fail_if_longer) {
function restore_max_pos (line 604) | inline void LimitingReaderBase::restore_max_pos(Position max_pos) {
function set_max_pos (line 618) | inline void LimitingReaderBase::set_max_pos(Position max_pos) {
function set_max_length (line 623) | inline void LimitingReaderBase::set_max_length(Position max_length) {
function SyncBuffer (line 638) | inline void LimitingReaderBase::SyncBuffer(Reader& src) {
function MakeBuffer (line 642) | inline void LimitingReaderBase::MakeBuffer(Reader& src) {
function Done (line 661) | void Done(LimitingReader& self) {
FILE: riegeli/bytes/limiting_writer.cc
type riegeli (line 38) | namespace riegeli {
function Reader (line 309) | Reader* LimitingWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/limiting_writer.h
function namespace (line 40) | namespace riegeli {
function Reset (line 294) | inline void LimitingWriterBase::Reset(Closed) {
function Reset (line 300) | inline void LimitingWriterBase::Reset(bool exact) {
function SyncBuffer (line 306) | inline bool LimitingWriterBase::SyncBuffer(Writer& dest) {
function MakeBuffer (line 315) | inline void LimitingWriterBase::MakeBuffer(Writer& dest) {
function Done (line 342) | void Done(LimitingWriter& self) {
FILE: riegeli/bytes/null_backward_writer.cc
type riegeli (line 36) | namespace riegeli {
FILE: riegeli/bytes/null_backward_writer.h
function namespace (line 36) | namespace riegeli {
function NullBackwardWriter (line 94) | inline NullBackwardWriter::NullBackwardWriter(Options options) noexcept
function Reset (line 111) | inline void NullBackwardWriter::Reset(Closed) {
function Reset (line 117) | inline void NullBackwardWriter::Reset(Options options) {
FILE: riegeli/bytes/null_writer.cc
type riegeli (line 36) | namespace riegeli {
FILE: riegeli/bytes/null_writer.h
function namespace (line 36) | namespace riegeli {
function NullWriter (line 100) | inline NullWriter::NullWriter(Options options) noexcept
function Reset (line 117) | inline void NullWriter::Reset(Closed) {
function Reset (line 124) | inline void NullWriter::Reset(Options options) {
FILE: riegeli/bytes/ostream_writer.cc
type riegeli (line 40) | namespace riegeli {
function Reader (line 305) | Reader* OStreamWriterBase::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/bytes/ostream_writer.h
function namespace (line 40) | namespace riegeli {
function explicit (line 213) | explicit OStreamWriter(Closed) -> OStreamWriter<DeleteCtad<Closed>>;
function Reset (line 263) | inline void OStreamWriterBase::Reset(BufferOptions buffer_options) {
FILE: riegeli/bytes/path_ref.h
function namespace (line 38) | namespace riegeli {
function friend (line 163) | friend riegeli::StrongOrdering RIEGELI_COMPARE(
function class (line 175) | class PathInitializer : public Initializer<std::string> {
FILE: riegeli/bytes/position_shifting_backward_writer.cc
type riegeli (line 37) | namespace riegeli {
FILE: riegeli/bytes/position_shifting_backward_writer.h
function namespace (line 41) | namespace riegeli {
FILE: riegeli/bytes/position_shifting_reader.cc
type riegeli (line 39) | namespace riegeli {
FILE: riegeli/bytes/position_shifting_reader.h
function namespace (line 39) | namespace riegeli {
function Reset (line 221) | inline void PositionShiftingReaderBase::Reset(Closed) {
function Reset (line 226) | inline void PositionShiftingReaderBase::Reset(Position base_pos) {
function Initialize (line 231) | inline void PositionShiftingReaderBase::Initialize(Reader* src) {
function SyncBuffer (line 237) | inline void PositionShiftingReaderBase::SyncBuffer(Reader& src) {
function MakeBuffer (line 241) | inline bool PositionShiftingReaderBase::MakeBuffer(Reader& src,
function Done (line 297) | void Done(PositionShiftingReader& self) {
FILE: riegeli/bytes/position_shifting_writer.cc
type riegeli (line 41) | namespace riegeli {
function Reader (line 208) | Reader* PositionShiftingWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/position_shifting_writer.h
function namespace (line 41) | namespace riegeli {
function Reset (line 226) | inline void PositionShiftingWriterBase::Reset(Closed) {
function Reset (line 232) | inline void PositionShiftingWriterBase::Reset(Position base_pos) {
function Initialize (line 238) | inline void PositionShiftingWriterBase::Initialize(Writer* dest) {
function SyncBuffer (line 244) | inline void PositionShiftingWriterBase::SyncBuffer(Writer& dest) {
function MakeBuffer (line 248) | inline bool PositionShiftingWriterBase::MakeBuffer(Writer& dest,
function Done (line 282) | void Done(PositionShiftingWriter& self) {
FILE: riegeli/bytes/prefix_limiting_backward_writer.cc
type riegeli (line 37) | namespace riegeli {
FILE: riegeli/bytes/prefix_limiting_backward_writer.h
function namespace (line 40) | namespace riegeli {
FILE: riegeli/bytes/prefix_limiting_reader.cc
type riegeli (line 38) | namespace riegeli {
FILE: riegeli/bytes/prefix_limiting_reader.h
function namespace (line 37) | namespace riegeli {
FILE: riegeli/bytes/prefix_limiting_writer.cc
type riegeli (line 40) | namespace riegeli {
function Reader (line 204) | Reader* PrefixLimitingWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/prefix_limiting_writer.h
function namespace (line 40) | namespace riegeli {
FILE: riegeli/bytes/pullable_reader.cc
type riegeli (line 43) | namespace riegeli {
FILE: riegeli/bytes/pullable_reader.h
function namespace (line 32) | namespace riegeli {
function Reset (line 199) | inline void PullableReader::Reset(Closed) {
function Reset (line 204) | inline void PullableReader::Reset() {
function PullableReader (line 213) | inline PullableReader::BehindScratch::BehindScratch(
function PullableReader (line 219) | inline PullableReader::BehindScratch::~BehindScratch() {
FILE: riegeli/bytes/pushable_backward_writer.cc
type riegeli (line 42) | namespace riegeli {
FILE: riegeli/bytes/pushable_backward_writer.h
function namespace (line 35) | namespace riegeli {
function Reset (line 212) | inline void PushableBackwardWriter::Reset(Closed) {
function Reset (line 217) | inline void PushableBackwardWriter::Reset() {
function PushableBackwardWriter (line 226) | inline PushableBackwardWriter::BehindScratch::BehindScratch(
function PushableBackwardWriter (line 232) | inline PushableBackwardWriter::BehindScratch::~BehindScratch() {
FILE: riegeli/bytes/pushable_writer.cc
type riegeli (line 42) | namespace riegeli {
function Reader (line 299) | Reader* PushableWriter::ReadModeBehindScratch(Position initial_pos) {
function Reader (line 444) | Reader* PushableWriter::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/pushable_writer.h
function namespace (line 36) | namespace riegeli {
function Reset (line 215) | inline void PushableWriter::Reset(Closed) {
function Reset (line 220) | inline void PushableWriter::Reset() {
function PushableWriter (line 229) | inline PushableWriter::BehindScratch::BehindScratch(
function PushableWriter (line 235) | inline PushableWriter::BehindScratch::~BehindScratch() {
FILE: riegeli/bytes/read_all.cc
type riegeli::read_all_internal (line 37) | namespace riegeli::read_all_internal {
function ABSL_ATTRIBUTE_COLD (line 41) | ABSL_ATTRIBUTE_COLD absl::Status MaxLengthExceeded(Reader& src,
function ReadAllImpl (line 47) | absl::Status ReadAllImpl(Reader& src, absl::string_view& dest,
function ReadAndAppendAllImpl (line 81) | absl::Status ReadAndAppendAllImpl(Reader& src, std::string& dest,
function ReadAndAppendAllImpl (line 139) | absl::Status ReadAndAppendAllImpl(Reader& src, Chain& dest, size_t max...
function ReadAndAppendAllImpl (line 172) | absl::Status ReadAndAppendAllImpl(Reader& src, absl::Cord& dest,
function ReadAllImpl (line 208) | absl::Status ReadAllImpl(Reader& src, absl::string_view& dest,
function ReadAllImpl (line 215) | absl::Status ReadAllImpl(Reader& src, char* dest, size_t max_length,
function ReadAllImpl (line 225) | absl::Status ReadAllImpl(Reader& src, std::string& dest, size_t max_le...
function ReadAllImpl (line 231) | absl::Status ReadAllImpl(Reader& src, Chain& dest, size_t max_length,
function ReadAllImpl (line 237) | absl::Status ReadAllImpl(Reader& src, absl::Cord& dest, size_t max_len...
function ReadAndAppendAllImpl (line 243) | absl::Status ReadAndAppendAllImpl(Reader& src, std::string& dest,
function ReadAndAppendAllImpl (line 258) | absl::Status ReadAndAppendAllImpl(Reader& src, Chain& dest, size_t max...
function ReadAndAppendAllImpl (line 273) | absl::Status ReadAndAppendAllImpl(Reader& src, absl::Cord& dest,
FILE: riegeli/bytes/read_all.h
function namespace (line 34) | namespace riegeli {
function Status (line 295) | Status ReadAll(Src&& src, char* dest, size_t max_length,
FILE: riegeli/bytes/reader.cc
type riegeli (line 45) | namespace riegeli {
FILE: riegeli/bytes/reader.h
function namespace (line 42) | namespace riegeli {
function Reset (line 664) | inline void Reader::Reset(Closed) {
function Reset (line 672) | inline void Reader::Reset() {
function Done (line 680) | inline void Reader::Done() {
function VerifyEndAndClose (line 685) | inline bool Reader::VerifyEndAndClose() {
function VerifyEnd (line 690) | inline void Reader::VerifyEnd() { VerifyEndImpl(); }
function SetReadAllHint (line 692) | inline void Reader::SetReadAllHint(bool read_all_hint) {
function Pull (line 696) | inline bool Reader::Pull(size_t min_length, size_t recommended_length) {
function move_cursor (line 707) | inline void Reader::move_cursor(size_t length) {
function set_cursor (line 713) | inline void Reader::set_cursor(const char* cursor) {
function set_buffer (line 721) | inline void Reader::set_buffer(const char* start, size_t start_to_limit,
function ReadByte (line 730) | inline bool Reader::ReadByte(uint8_t& dest) {
function Read (line 737) | inline bool Reader::Read(char& dest) {
function Read (line 744) | inline bool Reader::Read(size_t length, absl::string_view& dest,
function Read (line 754) | inline bool Reader::Read(size_t length, char* dest, size_t* length_read) {
function ReadSome (line 765) | inline bool Reader::ReadSome(size_t max_length, absl::string_view& dest,
function ReadSome (line 776) | inline bool Reader::ReadSome(size_t max_length, char* dest,
function ReadHint (line 792) | inline void Reader::ReadHint(size_t min_length, size_t recommended_lengt...
function Sync (line 797) | inline bool Reader::Sync(SyncType sync_type) { return SyncImpl(sync_type...
function move_limit_pos (line 811) | inline void Reader::move_limit_pos(Position length) {
function set_limit_pos (line 818) | inline void Reader::set_limit_pos(Position limit_pos) {
function Seek (line 822) | inline bool Reader::Seek(Position new_pos) {
function Skip (line 830) | inline bool Reader::Skip(Position length, Position* length_skipped) {
function std (line 867) | inline std::optional<Position> Reader::Size() { return SizeImpl(); }
FILE: riegeli/bytes/reader_cfile.cc
type riegeli::cfile_internal (line 43) | namespace riegeli::cfile_internal {
function ReaderCFileRead (line 143) | static ssize_t ReaderCFileRead(void* cookie, char* buf, size_t size) {
function ReaderCFileSeek (line 147) | static int ReaderCFileSeek(void* cookie, off64_t* offset, int whence) {
function ReaderCFileClose (line 159) | static int ReaderCFileClose(void* cookie) {
function OwnedCFile (line 171) | OwnedCFile ReaderCFileImpl(ReaderCFileCookieBase* cookie,
FILE: riegeli/bytes/reader_cfile.h
function namespace (line 39) | namespace riegeli {
function namespace (line 92) | namespace cfile_internal {
FILE: riegeli/bytes/reader_factory.cc
type riegeli (line 44) | namespace riegeli {
class ReaderFactoryBase::ConcurrentReader (line 46) | class ReaderFactoryBase::ConcurrentReader : public PullableReader {
method ConcurrentReader (line 50) | ConcurrentReader(const ConcurrentReader&) = delete;
method ConcurrentReader (line 51) | ConcurrentReader& operator=(const ConcurrentReader&) = delete;
method SupportsRandomAccess (line 54) | bool SupportsRandomAccess() override { return true; }
method SupportsNewReader (line 55) | bool SupportsNewReader() override { return true; }
FILE: riegeli/bytes/reader_factory.h
function namespace (line 34) | namespace riegeli {
FILE: riegeli/bytes/reader_istream.cc
type riegeli (line 32) | namespace riegeli {
type stream_internal (line 34) | namespace stream_internal {
class ReaderStreambuf::BufferSync (line 36) | class ReaderStreambuf::BufferSync {
method BufferSync (line 38) | explicit BufferSync(ReaderStreambuf* streambuf) : streambuf_(strea...
method BufferSync (line 42) | BufferSync(const BufferSync&) = delete;
method BufferSync (line 43) | BufferSync& operator=(const BufferSync&) = delete;
FILE: riegeli/bytes/reader_istream.h
function namespace (line 35) | namespace riegeli {
function Reset (line 296) | inline void ReaderIStreamBase::Reset(Closed) {
function Reset (line 302) | inline void ReaderIStreamBase::Reset() {
function Initialize (line 308) | inline void ReaderIStreamBase::Initialize(Reader* src) {
function Done (line 320) | void Done(ReaderIStream& self) { self.streambuf_.MoveEnd(self.src_.get()...
FILE: riegeli/bytes/resizable_writer.cc
type riegeli (line 40) | namespace riegeli {
function Reader (line 379) | Reader* ResizableWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/resizable_writer.h
function namespace (line 45) | namespace riegeli {
function Size (line 354) | static size_t Size(const Resizable& dest) { return dest.size(); }
function Resize (line 356) | static bool Resize(Resizable& dest, size_t new_size, size_t used_size) {
function Grow (line 367) | static bool Grow(Resizable& dest, size_t new_size, size_t used_size) {
function GrowUnderCapacity (line 381) | static bool GrowUnderCapacity(Resizable& dest, size_t new_size) {
function ResizableWriterBase (line 413) | inline ResizableWriterBase::ResizableWriterBase(BufferOptions buffer_opt...
function Reset (line 437) | inline void ResizableWriterBase::Reset(Closed) {
function Reset (line 445) | inline void ResizableWriterBase::Reset(BufferOptions buffer_options) {
function MoveSecondaryBuffer (line 455) | inline void ResizableWriterBase::MoveSecondaryBuffer(
FILE: riegeli/bytes/restricted_chain_writer.cc
type riegeli (line 33) | namespace riegeli {
FILE: riegeli/bytes/restricted_chain_writer.h
function namespace (line 33) | namespace riegeli {
function RestrictedChainWriter (line 97) | inline RestrictedChainWriter::RestrictedChainWriter(
function Reset (line 110) | inline void RestrictedChainWriter::Reset(Closed) {
function Reset (line 115) | inline void RestrictedChainWriter::Reset() {
function MoveDest (line 120) | inline void RestrictedChainWriter::MoveDest(RestrictedChainWriter&& that) {
FILE: riegeli/bytes/splitting_writer.cc
type riegeli (line 41) | namespace riegeli {
FILE: riegeli/bytes/splitting_writer.h
function namespace (line 37) | namespace riegeli {
function Reset (line 243) | inline void SplittingWriterBase::Reset(Closed) {
function Reset (line 249) | inline void SplittingWriterBase::Reset() {
function shard_is_open (line 255) | inline bool SplittingWriterBase::shard_is_open() const {
function SyncBuffer (line 263) | inline void SplittingWriterBase::SyncBuffer(Writer& shard) {
function MakeBuffer (line 272) | inline void SplittingWriterBase::MakeBuffer(Writer& shard) {
function Done (line 298) | void Done(SplittingWriter& self) {
FILE: riegeli/bytes/std_io.cc
type riegeli (line 25) | namespace riegeli {
function SizedSharedBuffer (line 33) | SizedSharedBuffer& StdInPending() {
FILE: riegeli/bytes/std_io.h
function namespace (line 25) | namespace riegeli {
FILE: riegeli/bytes/string_reader.cc
type riegeli (line 27) | namespace riegeli {
FILE: riegeli/bytes/string_reader.h
function namespace (line 35) | namespace riegeli {
function Initialize (line 171) | inline void StringReaderBase::Initialize(absl::string_view src) {
function Done (line 192) | void Done(StringReader& self) {
FILE: riegeli/bytes/string_writer.cc
type riegeli (line 42) | namespace riegeli {
function Reader (line 541) | Reader* StringWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/string_writer.h
function namespace (line 41) | namespace riegeli {
function explicit (line 270) | explicit StringWriter(Closed) -> StringWriter<DeleteCtad<Closed>>;
function Reset (line 316) | inline void StringWriterBase::Reset(BufferOptions buffer_options) {
function Initialize (line 326) | inline void StringWriterBase::Initialize(std::string* dest, bool append) {
function MoveSecondaryBuffer (line 337) | inline void StringWriterBase::MoveSecondaryBuffer(StringWriterBase& that) {
function Done (line 376) | void Done(StringWriter& self) {
FILE: riegeli/bytes/stringify.h
function namespace (line 32) | namespace riegeli {
function Position (line 62) | inline Position StringifiedSize(ABSL_ATTRIBUTE_UNUSED char src) { return...
function Position (line 64) | inline Position StringifiedSize(ABSL_ATTRIBUTE_UNUSED char8_t src) { ret...
function Position (line 66) | inline Position StringifiedSize(BytesRef src) { return src.size(); }
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 67) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline Position StringifiedSize(const char*...
function Position (line 70) | inline Position StringifiedSize(const Chain& src) { return src.size(); }
function Position (line 71) | inline Position StringifiedSize(const absl::Cord& src) { return src.size...
function Position (line 72) | inline Position StringifiedSize(ByteFill src) { return src.size(); }
function Position (line 73) | inline Position StringifiedSize(signed char src) {
function Position (line 76) | inline Position StringifiedSize(unsigned char src) {
function Position (line 79) | inline Position StringifiedSize(short src) {
function Position (line 82) | inline Position StringifiedSize(unsigned short src) {
function Position (line 85) | inline Position StringifiedSize(int src) {
function Position (line 88) | inline Position StringifiedSize(unsigned src) {
function Position (line 91) | inline Position StringifiedSize(long src) {
function Position (line 94) | inline Position StringifiedSize(unsigned long src) {
function Position (line 97) | inline Position StringifiedSize(long long src) {
function Position (line 100) | inline Position StringifiedSize(unsigned long long src) {
function Position (line 103) | inline Position StringifiedSize(absl::int128 src) {
function Position (line 106) | inline Position StringifiedSize(absl::uint128 src) {
function namespace (line 132) | namespace stringify_internal {
FILE: riegeli/bytes/stringify_writer.h
function namespace (line 33) | namespace riegeli {
function explicit (line 82) | explicit StringifyWriter(
function ABSL_ATTRIBUTE_LIFETIME_BOUND (line 97) | const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function explicit (line 105) | explicit StringifyWriter(Closed) -> StringifyWriter<DeleteCtad<Closed>>;
function Reset (line 139) | inline void StringifyWriter<WriterStringifySink*>::Reset(Closed) {
function Reset (line 144) | inline void StringifyWriter<WriterStringifySink*>::Reset(
FILE: riegeli/bytes/vector_writer.h
function namespace (line 30) | namespace riegeli {
FILE: riegeli/bytes/wrapping_backward_writer.cc
type riegeli (line 34) | namespace riegeli {
FILE: riegeli/bytes/wrapping_backward_writer.h
function namespace (line 39) | namespace riegeli {
FILE: riegeli/bytes/wrapping_reader.cc
type riegeli (line 36) | namespace riegeli {
FILE: riegeli/bytes/wrapping_reader.h
function namespace (line 37) | namespace riegeli {
function Initialize (line 169) | inline void WrappingReaderBase::Initialize(Reader* src) {
function SyncBuffer (line 175) | inline void WrappingReaderBase::SyncBuffer(Reader& src) {
function MakeBuffer (line 179) | inline void WrappingReaderBase::MakeBuffer(Reader& src) {
function Done (line 197) | void Done(WrappingReader& self) {
FILE: riegeli/bytes/wrapping_writer.cc
type riegeli (line 36) | namespace riegeli {
function Reader (line 174) | Reader* WrappingWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/bytes/wrapping_writer.h
function namespace (line 39) | namespace riegeli {
function Initialize (line 165) | inline void WrappingWriterBase::Initialize(Writer* dest) {
function SyncBuffer (line 171) | inline void WrappingWriterBase::SyncBuffer(Writer& dest) {
function MakeBuffer (line 175) | inline void WrappingWriterBase::MakeBuffer(Writer& dest) {
function Done (line 193) | void Done(WrappingWriter& self) {
FILE: riegeli/bytes/write.h
function namespace (line 33) | namespace riegeli {
FILE: riegeli/bytes/write_int_internal.cc
type riegeli::write_int_internal (line 31) | namespace riegeli::write_int_internal {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 72) | ABSL_ATTRIBUTE_ALWAYS_INLINE
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 133) | ABSL_ATTRIBUTE_ALWAYS_INLINE
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 156) | ABSL_ATTRIBUTE_ALWAYS_INLINE
type StringifiedSizeEntry (line 351) | struct StringifiedSizeEntry {
function MakeStringifiedSizeEntries (line 362) | constexpr std::array<StringifiedSizeEntry<T>, kNumStringifiedSizeEntri...
function StringifiedSize (line 388) | size_t StringifiedSize(uint32_t src) {
function StringifiedSize (line 397) | size_t StringifiedSize(uint64_t src) {
function StringifiedSize (line 406) | size_t StringifiedSize(absl::uint128 src) {
function StringifiedSize (line 418) | size_t StringifiedSize(int32_t src) {
function StringifiedSize (line 426) | size_t StringifiedSize(int64_t src) {
function StringifiedSize (line 434) | size_t StringifiedSize(absl::int128 src) {
FILE: riegeli/bytes/write_int_internal.h
function namespace (line 27) | namespace riegeli::write_int_internal {
FILE: riegeli/bytes/writer.cc
type riegeli (line 44) | namespace riegeli {
function WriteUnsigned (line 49) | inline bool WriteUnsigned(T src, Writer& dest) {
function WriteSigned (line 58) | inline bool WriteSigned(T src, Writer& dest) {
function Reader (line 281) | Reader* Writer::ReadModeImpl(Position initial_pos) {
type writer_internal (line 286) | namespace writer_internal {
function DeleteReader (line 288) | void DeleteReader(Reader* reader) { delete reader; }
FILE: riegeli/bytes/writer.h
function class (line 52) | class WriterStringifySink {
function WriteByte (line 189) | bool WriteByte(uint8_t src) { return Write(static_cast<char>(src)); }
function Write (line 199) | bool Write(char8_t src) { return Write(static_cast<char>(src)); }
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 202) | ABSL_ATTRIBUTE_ALWAYS_INLINE
function Position (line 311) | Position pos() const;
function virtual (line 348) | virtual bool SupportsTruncate() { return SupportsRandomAccess(); }
function virtual (line 363) | virtual bool SupportsReadMode() { return false; }
function friend (line 387) | friend void AbslFormatFlush(Writer* dest, absl::string_view src) {
function Done (line 410) | void Done() override;
function Append (line 563) | inline void WriterStringifySink::Append(size_t length, char fill) {
function Append (line 567) | inline void WriterStringifySink::Append(absl::string_view src) {
function Reset (line 587) | inline void Writer::Reset(Closed) {
function Reset (line 595) | inline void Writer::Reset() {
function Close (line 603) | inline bool Writer::Close() {
function Done (line 608) | inline void Writer::Done() {
function SetWriteSizeHint (line 613) | inline void Writer::SetWriteSizeHint(std::optional<Position> write_size_...
function Push (line 618) | inline bool Writer::Push(size_t min_length, size_t recommended_length) {
function move_cursor (line 630) | inline void Writer::move_cursor(size_t length) {
function set_cursor (line 636) | inline void Writer::set_cursor(char* cursor) {
function set_buffer (line 644) | inline void Writer::set_buffer(char* start, size_t start_to_limit,
function Write (line 653) | inline bool Writer::Write(char src) {
function Write (line 660) | inline bool Writer::Write(BytesRef src) {
function Write (line 671) | inline bool Writer::Write(ExternalRef src) {
function Write (line 684) | bool Writer::Write(Src&& src) {
function Write (line 688) | inline bool Writer::Write(ByteFill src) {
function Flush (line 732) | inline bool Writer::Flush(FlushType flush_type) {
function set_start_pos (line 758) | inline void Writer::set_start_pos(Position start_pos) {
function Seek (line 762) | inline bool Writer::Seek(Position new_pos) {
function std (line 768) | inline std::optional<Position> Writer::Size() {
function Truncate (line 773) | inline bool Writer::Truncate(Position new_size) {
function Reader (line 778) | inline Reader* Writer::ReadMode(Position initial_pos)
function namespace (line 784) | namespace writer_internal {
FILE: riegeli/bytes/writer_cfile.cc
type riegeli::cfile_internal (line 45) | namespace riegeli::cfile_internal {
function WriterCFileRead (line 233) | static ssize_t WriterCFileRead(void* cookie, char* buf, size_t size) {
function WriterCFileWrite (line 237) | static ssize_t WriterCFileWrite(void* cookie, const char* buf, size_t ...
function WriterCFileSeek (line 241) | static int WriterCFileSeek(void* cookie, off64_t* offset, int whence) {
function WriterCFileClose (line 253) | static int WriterCFileClose(void* cookie) {
function OwnedCFile (line 265) | OwnedCFile WriterCFileImpl(WriterCFileCookieBase* cookie,
FILE: riegeli/bytes/writer_cfile.h
function namespace (line 39) | namespace riegeli {
function namespace (line 124) | namespace cfile_internal {
FILE: riegeli/bytes/writer_ostream.cc
type riegeli (line 36) | namespace riegeli {
type stream_internal (line 38) | namespace stream_internal {
class WriterStreambuf::BufferSync (line 40) | class WriterStreambuf::BufferSync {
method BufferSync (line 42) | explicit BufferSync(WriterStreambuf* streambuf) : streambuf_(strea...
method BufferSync (line 50) | BufferSync(const BufferSync&) = delete;
method BufferSync (line 51) | BufferSync& operator=(const BufferSync&) = delete;
FILE: riegeli/bytes/writer_ostream.h
function namespace (line 37) | namespace riegeli {
function Reset (line 299) | inline void WriterOStreamBase::Reset(Closed) {
function Reset (line 305) | inline void WriterOStreamBase::Reset() {
function Initialize (line 311) | inline void WriterOStreamBase::Initialize(Writer* dest) {
function Done (line 322) | void Done(WriterOStream& self) {
FILE: riegeli/bzip2/bzip2_error.cc
type riegeli::bzip2_internal (line 25) | namespace riegeli::bzip2_internal {
function Bzip2ErrorToStatus (line 27) | absl::Status Bzip2ErrorToStatus(absl::string_view operation, int bzlib...
FILE: riegeli/bzip2/bzip2_error.h
function namespace (line 21) | namespace riegeli::bzip2_internal {
FILE: riegeli/bzip2/bzip2_reader.cc
type riegeli (line 38) | namespace riegeli {
function GetByte (line 266) | inline bool GetByte(Reader& src, size_t offset, uint8_t& value) {
function RecognizeBzip2 (line 274) | bool RecognizeBzip2(Reader& src) {
FILE: riegeli/bzip2/bzip2_reader.h
function namespace (line 37) | namespace riegeli {
function explicit (line 189) | explicit Bzip2Reader(Closed) -> Bzip2Reader<DeleteCtad<Closed>>;
function Reset (line 234) | inline void Bzip2ReaderBase::Reset(BufferOptions buffer_options,
FILE: riegeli/bzip2/bzip2_writer.cc
type riegeli (line 37) | namespace riegeli {
FILE: riegeli/bzip2/bzip2_writer.h
function namespace (line 35) | namespace riegeli {
function explicit (line 170) | explicit Bzip2Writer(Closed) -> Bzip2Writer<DeleteCtad<Closed>>;
function Reset (line 194) | inline void Bzip2WriterBase::Reset(Closed) {
function Reset (line 200) | inline void Bzip2WriterBase::Reset(BufferOptions buffer_options) {
FILE: riegeli/chunk_encoding/brotli_encoder_selection.cc
type riegeli::chunk_encoding_internal (line 31) | namespace riegeli::chunk_encoding_internal {
function ABSL_ATTRIBUTE_WEAK (line 33) | ABSL_ATTRIBUTE_WEAK std::unique_ptr<Writer> NewBrotliWriter(
function NewCBrotliWriter (line 51) | std::unique_ptr<Writer> NewCBrotliWriter(
FILE: riegeli/chunk_encoding/brotli_encoder_selection.h
function namespace (line 25) | namespace riegeli::chunk_encoding_internal {
FILE: riegeli/chunk_encoding/chunk.cc
type riegeli (line 28) | namespace riegeli {
FILE: riegeli/chunk_encoding/chunk.h
function namespace (line 30) | namespace riegeli {
FILE: riegeli/chunk_encoding/chunk_decoder.cc
type riegeli (line 43) | namespace riegeli {
FILE: riegeli/chunk_encoding/chunk_decoder.h
function namespace (line 45) | namespace riegeli {
function Reset (line 241) | inline void ChunkDecoder::Reset(Options options) {
function Clear (line 247) | inline void ChunkDecoder::Clear() {
function ReadRecord (line 255) | inline bool ChunkDecoder::ReadRecord(absl::string_view& record) {
function ReadRecord (line 270) | inline bool ChunkDecoder::ReadRecord(std::string& record) {
function ReadRecord (line 285) | inline bool ChunkDecoder::ReadRecord(Chain& record) {
function ReadRecord (line 300) | inline bool ChunkDecoder::ReadRecord(absl::Cord& record) {
function SetIndex (line 315) | inline void ChunkDecoder::SetIndex(uint64_t index) {
FILE: riegeli/chunk_encoding/chunk_encoder.cc
type riegeli (line 26) | namespace riegeli {
FILE: riegeli/chunk_encoding/chunk_encoder.h
function namespace (line 37) | namespace riegeli {
FILE: riegeli/chunk_encoding/compressor.cc
type riegeli::chunk_encoding_internal (line 39) | namespace riegeli::chunk_encoding_internal {
FILE: riegeli/chunk_encoding/compressor.h
function class (line 33) | class Compressor : public Object {
FILE: riegeli/chunk_encoding/compressor_options.cc
type riegeli (line 29) | namespace riegeli {
FILE: riegeli/chunk_encoding/compressor_options.h
function BrotliEncoder (line 37) | enum class BrotliEncoder {
FILE: riegeli/chunk_encoding/constants.h
type class (line 25) | enum class
function CompressionType (line 34) | enum class CompressionType : uint8_t {
FILE: riegeli/chunk_encoding/decompressor.cc
type riegeli::chunk_encoding_internal (line 28) | namespace riegeli::chunk_encoding_internal {
function UncompressedSize (line 30) | std::optional<uint64_t> UncompressedSize(const Chain& compressed_data,
FILE: riegeli/chunk_encoding/decompressor.h
function namespace (line 42) | namespace riegeli::chunk_encoding_internal {
FILE: riegeli/chunk_encoding/deferred_encoder.cc
type riegeli (line 41) | namespace riegeli {
FILE: riegeli/chunk_encoding/deferred_encoder.h
function class (line 41) | class DeferredEncoder : public ChunkEncoder {
FILE: riegeli/chunk_encoding/field_projection.h
function namespace (line 26) | namespace riegeli {
function Field (line 118) | inline Field::Field(std::initializer_list<int> path) : path_(path) {
FILE: riegeli/chunk_encoding/hash.cc
type riegeli::chunk_encoding_internal (line 29) | namespace riegeli::chunk_encoding_internal {
function Hash (line 42) | uint64_t Hash(absl::string_view data) {
function Hash (line 49) | uint64_t Hash(const Chain& data) {
FILE: riegeli/chunk_encoding/hash.h
function namespace (line 23) | namespace riegeli::chunk_encoding_internal {
FILE: riegeli/chunk_encoding/simple_decoder.cc
type riegeli (line 34) | namespace riegeli {
FILE: riegeli/chunk_encoding/simple_decoder.h
function class (line 33) | class SimpleDecoder : public Object {
FILE: riegeli/chunk_encoding/simple_encoder.cc
type riegeli (line 42) | namespace riegeli {
FILE: riegeli/chunk_encoding/simple_encoder.h
function class (line 52) | class SimpleEncoder : public ChunkEncoder {
FILE: riegeli/chunk_encoding/transpose_decoder.cc
type riegeli (line 56) | namespace riegeli {
function Reader (line 59) | Reader* kEmptyReader() {
type DataBucket (line 68) | struct DataBucket {
type FieldIncluded (line 83) | enum class FieldIncluded {
function ValidTag (line 90) | bool ValidTag(uint32_t tag) {
type CallbackType (line 108) | enum class CallbackType : uint8_t {
function CallbackType (line 147) | constexpr CallbackType operator+(CallbackType a, uint8_t b) {
type StateMachineNodeTemplate (line 157) | struct StateMachineNodeTemplate {
function CallbackType (line 171) | inline CallbackType GetCopyTagCallbackType(size_t tag_length) {
function CallbackType (line 181) | inline CallbackType GetVarintCallbackType(
function CallbackType (line 198) | inline CallbackType GetFixed32CallbackType(size_t tag_length) {
function CallbackType (line 207) | inline CallbackType GetFixed64CallbackType(size_t tag_length) {
function CallbackType (line 216) | inline CallbackType GetFixed32ExistenceCallbackType(size_t tag_length) {
function CallbackType (line 225) | inline CallbackType GetFixed64ExistenceCallbackType(size_t tag_length) {
function CallbackType (line 234) | inline CallbackType GetStringCallbackType(
function CallbackType (line 254) | inline CallbackType GetStringExcludeCallbackType(
function CallbackType (line 269) | inline CallbackType GetStringExistenceCallbackType(
function CallbackType (line 285) | inline CallbackType GetStartProjectionGroupCallbackType(size_t tag_len...
function CallbackType (line 293) | inline CallbackType GetEndProjectionGroupCallbackType(size_t tag_lengt...
function CallbackType (line 302) | inline CallbackType GetCallbackType(FieldIncluded field_included, uint...
type TransposeDecoder::StateMachineNode (line 370) | struct TransposeDecoder::StateMachineNode {
type TransposeDecoder::Context (line 392) | struct TransposeDecoder::Context {
type IncludeType (line 409) | enum class IncludeType : uint8_t {
type IncludedField (line 420) | struct IncludedField {
function Reader (line 972) | inline Reader* TransposeDecoder::GetBuffer(Context& context,
function ABSL_ATTRIBUTE_NOINLINE (line 1040) | ABSL_ATTRIBUTE_NOINLINE absl::Status InvalidArgumentError(
type TransposeDecoder::DecodingState (line 1045) | struct TransposeDecoder::DecodingState {
class SubmessageStack (line 1052) | class SubmessageStack {
method ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1054) | ABSL_ATTRIBUTE_ALWAYS_INLINE void Push(size_t position,
type SubmessageStackElement (line 1062) | struct SubmessageStackElement {
method Pop (line 1067) | Pop() {
method ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1073) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool Empty() const { return size_ == ...
function DecodingState (line 1110) | explicit DecodingState(TransposeDecoder* decoder, Context* context,
function DecodingState (line 1127) | DecodingState(const DecodingState&) = delete;
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1130) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool SetCallbackType() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1137) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool CopyTagCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1157) | ABSL_ATTRIBUTE_ALWAYS_INLINE void MaskBuffer(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1168) | ABSL_ATTRIBUTE_ALWAYS_INLINE void MaskBuffer(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1179) | ABSL_ATTRIBUTE_ALWAYS_INLINE void MaskBuffer(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1190) | ABSL_ATTRIBUTE_ALWAYS_INLINE void MaskBuffer(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1198) | ABSL_ATTRIBUTE_ALWAYS_INLINE void MaskBuffer(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1206) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool VarintCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1225) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool FixedCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1242) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool FixedExistenceCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1255) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool StringCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1283) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool StartProjectionGroupCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1292) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool EndProjectionGroupCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1297) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool SubmessageStartCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1321) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool MessageStartCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1332) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool NonprotoCallback() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1347) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool HandleNode() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1441) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool TransitionNode() {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 1456) | ABSL_ATTRIBUTE_ALWAYS_INLINE bool Finish() {
function ABSL_ATTRIBUTE_NOINLINE (line 1527) | ABSL_ATTRIBUTE_NOINLINE inline bool TransposeDecoder::SetCallbackType(
FILE: riegeli/chunk_encoding/transpose_decoder.h
type StateMachineNode (line 92) | struct StateMachineNode
type Context (line 93) | struct Context
type DecodingState (line 94) | struct DecodingState
FILE: riegeli/chunk_encoding/transpose_encoder.cc
type riegeli (line 63) | namespace riegeli {
function IsProtoMessage (line 90) | bool IsProtoMessage(Reader& record) {
type PriorityQueueEntry (line 137) | struct PriorityQueueEntry : WithCompare<PriorityQueueEntry> {
method PriorityQueueEntry (line 138) | PriorityQueueEntry() = default;
method PriorityQueueEntry (line 140) | explicit PriorityQueueEntry(uint32_t dest_index, size_t num_transiti...
method StrongOrdering (line 148) | StrongOrdering RIEGELI_COMPARE(const PriorityQueueEntry& a,
function BackwardWriter (line 308) | inline BackwardWriter* TransposeEncoder::GetBuffer(Node* node,
FILE: riegeli/chunk_encoding/transpose_encoder.h
function RecyclingPoolOptions (line 110) | const RecyclingPoolOptions& recycling_pool_options() const
FILE: riegeli/chunk_encoding/transpose_internal.h
function MessageId (line 26) | enum class MessageId : uint32_t {
function Subtype (line 50) | enum class Subtype : uint8_t {
FILE: riegeli/containers/chunked_sorted_string_set.cc
type riegeli (line 42) | namespace riegeli {
function ChunkedSortedStringSet (line 44) | ChunkedSortedStringSet ChunkedSortedStringSet::FromSorted(
function ChunkedSortedStringSet (line 49) | ChunkedSortedStringSet ChunkedSortedStringSet::FromUnsorted(
function StrongOrdering (line 99) | StrongOrdering ChunkedSortedStringSet::Compare(
function ChunkedSortedStringSet (line 295) | ChunkedSortedStringSet ChunkedSortedStringSet::Builder::Build() {
FILE: riegeli/containers/chunked_sorted_string_set.h
function namespace (line 45) | namespace riegeli {
function reference (line 367) | reference operator*() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function pointer (line 375) | pointer operator->() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function explicit (line 402) | explicit IteratorImpl(const ChunkedSortedStringSet* set)
function class (line 421) | class ChunkedSortedStringSet::SplitElements {
function class (line 452) | class ChunkedSortedStringSet::Builder {
function class (line 550) | class ChunkedSortedStringSet::NextInsertIterator {
function Builder (line 601) | Builder* builder() const { return builder_; }
function explicit (line 606) | explicit NextInsertIterator(Builder* builder) : builder_(builder) {}
function ChunkedSortedStringSet (line 633) | ChunkedSortedStringSet ChunkedSortedStringSet::FromUnsorted(
FILE: riegeli/containers/linear_sorted_string_set.cc
type riegeli (line 50) | namespace riegeli {
function CommonPrefix (line 54) | inline size_t CommonPrefix(absl::string_view a, absl::string_view b) {
function LinearSortedStringSet (line 88) | LinearSortedStringSet LinearSortedStringSet::FromSorted(
function LinearSortedStringSet (line 93) | LinearSortedStringSet LinearSortedStringSet::FromUnsorted(
function StrongOrdering (line 250) | StrongOrdering LinearSortedStringSet::Compare(const LinearSortedString...
function StrongOrdering (line 601) | StrongOrdering LinearSortedStringSet::SplitElement::Compare(
function StrongOrdering (line 642) | StrongOrdering LinearSortedStringSet::SplitElement::Compare(
function LinearSortedStringSet (line 772) | LinearSortedStringSet LinearSortedStringSet::Builder::Build() {
FILE: riegeli/containers/linear_sorted_string_set.h
function namespace (line 46) | namespace riegeli {
function class (line 386) | class LinearSortedStringSet::SplitElement : public WithCompare<SplitElem...
function RiegeliDebug (line 440) | void RiegeliDebug(const SplitElement& src, DebugStream& dest) {
function class (line 462) | class LinearSortedStringSet::SplitElementIterator
function class (line 553) | class LinearSortedStringSet::SplitElements {
function class (line 586) | class LinearSortedStringSet::Builder {
function class (line 678) | class LinearSortedStringSet::NextInsertIterator {
function Builder (line 729) | Builder* builder() const { return builder_; }
function explicit (line 734) | explicit NextInsertIterator(Builder* builder) : builder_(builder) {}
function LinearSortedStringSet (line 757) | LinearSortedStringSet LinearSortedStringSet::FromUnsorted(Src&& src) {
function remove_prefix (line 906) | inline void LinearSortedStringSet::SplitElement::remove_prefix(size_t le...
function remove_suffix (line 919) | inline void LinearSortedStringSet::SplitElement::remove_suffix(size_t le...
FILE: riegeli/csv/csv_reader.cc
type riegeli (line 43) | namespace riegeli {
type csv_internal (line 550) | namespace csv_internal {
function ReadStandaloneRecord (line 552) | inline bool ReadStandaloneRecord(CsvReaderBase& csv_reader,
function ReadCsvRecordFromString (line 650) | absl::Status ReadCsvRecordFromString(absl::string_view src,
FILE: riegeli/csv/csv_reader.h
function namespace (line 42) | namespace riegeli {
function set_recovery (line 354) | void set_recovery(
function CsvHeader (line 372) | const CsvHeader& header() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function CharClass (line 471) | enum class CharClass : uint8_t {
function Reader (line 582) | Reader* SrcReader() const ABSL_ATTRIBUTE_LIFETIME_BOUND override {
function explicit (line 594) | explicit CsvReader(Closed) -> CsvReader<DeleteCtad<Closed>>;
function Reset (line 657) | inline void CsvReaderBase::Reset() {
FILE: riegeli/csv/csv_record.cc
type riegeli (line 47) | namespace riegeli {
function WriteDebugQuoted (line 51) | inline void WriteDebugQuoted(absl::string_view src, Writer& dest,
function DebugQuotedIfNeeded (line 68) | inline std::string DebugQuotedIfNeeded(absl::string_view src) {
type csv_internal (line 78) | namespace csv_internal {
function WriteDebugQuotedIfNeeded (line 80) | void WriteDebugQuotedIfNeeded(absl::string_view src, Writer& dest) {
function AsciiCaseInsensitive (line 101) | std::string AsciiCaseInsensitive(absl::string_view name) {
function CsvHeader (line 118) | CsvHeader& CsvHeader::operator=(
FILE: riegeli/csv/csv_record.h
function namespace (line 52) | namespace riegeli::csv_internal {
function namespace (line 69) | namespace riegeli {
function RiegeliRegisterSubobjects (line 354) | void RiegeliRegisterSubobjects(const CsvHeader* self,
type Payload (line 360) | struct Payload {
function CsvHeaderConstant (line 485) | constexpr CsvHeaderConstant(
function class (line 550) | class CsvRecord : public WithEqual<CsvRecord> {
function RiegeliRegisterSubobjects (line 859) | void RiegeliRegisterSubobjects(const CsvRecord* self,
function namespace (line 886) | namespace csv_internal {
function typename (line 892) | inline typename CsvHeader::iterator::reference CsvHeader::iterator::oper...
function typename (line 897) | inline typename CsvHeader::iterator::pointer CsvHeader::iterator::operat...
function Status (line 994) | Status CsvHeader::TryReset(Names&& names) {
function Status (line 1000) | Status CsvHeader::TryReset(
function MaybeResetToCachedPayload (line 1037) | inline bool CsvHeader::MaybeResetToCachedPayload(
function Add (line 1062) | void CsvHeader::Add(StringInitializer name, Names&&... names) {
function std (line 1082) | inline const std::function<std::string(absl::string_view)>&
function CsvRecord (line 1213) | inline CsvRecord::CsvRecord(CsvHeader header)
function Status (line 1234) | Status CsvRecord::TryReset(CsvHeader header, Fields&& fields) {
function Status (line 1278) | Status CsvRecord::TryMerge(Src&& src) {
function Status (line 1332) | Status CsvRecord::TrySplit(Dest& dest) const {
FILE: riegeli/csv/csv_writer.cc
type riegeli (line 37) | namespace riegeli {
FILE: riegeli/csv/csv_writer.h
function namespace (line 46) | namespace riegeli {
function CsvHeader (line 263) | const CsvHeader& header() const ABSL_ATTRIBUTE_LIFETIME_BOUND {
function Writer (line 436) | Writer* DestWriter() const ABSL_ATTRIBUTE_LIFETIME_BOUND override {
function explicit (line 448) | explicit CsvWriter(Closed) -> CsvWriter<DeleteCtad<Closed>>;
function Reset (line 510) | inline void CsvWriterBase::Reset() {
function namespace (line 519) | namespace csv_internal {
function WriteRecord (line 532) | bool CsvWriterBase::WriteRecord(const Record& record) {
function WriteRecord (line 536) | inline bool CsvWriterBase::WriteRecord(
function WriteRecordInternal (line 542) | bool CsvWriterBase::WriteRecordInternal(const Record& record) {
function Options (line 620) | Options options) {
function std (line 633) | inline std::string WriteCsvRecordToString(
FILE: riegeli/digests/adler32_digester.cc
type riegeli (line 25) | namespace riegeli {
FILE: riegeli/digests/adler32_digester.h
function namespace (line 22) | namespace riegeli {
FILE: riegeli/digests/crc32_digester.cc
type riegeli (line 25) | namespace riegeli {
FILE: riegeli/digests/crc32_digester.h
function namespace (line 22) | namespace riegeli {
FILE: riegeli/digests/crc32c_digester.h
function class (line 43) | class Crc32cDigester {
function MaskCrc32c (line 65) | uint32_t MaskCrc32c(uint32_t unmasked) {
function UnmaskCrc32c (line 72) | uint32_t UnmaskCrc32c(uint32_t masked) {
FILE: riegeli/digests/digest_converter.h
function namespace (line 31) | namespace riegeli {
function To (line 71) | To Convert(const From& digest) {
function To (line 89) | static To Convert(const std::array<char, size>& digest) {
function To (line 104) | static To Convert(uint32_t digest) {
function To (line 118) | static To Convert(uint64_t digest) {
function To (line 132) | static To Convert(absl::uint128 digest) {
function To (line 153) | static To Convert(std::array<char, sizeof(uint32_t)> digest) {
function To (line 164) | static To Convert(std::array<char, sizeof(uint64_t)> digest) {
function To (line 176) | static To Convert(std::array<char, sizeof(absl::uint128)> digest) {
function To (line 190) | static To Convert(const std::array<uint64_t, size>& digest) {
function To (line 213) | static To Convert(const std::array<char, size>& digest) {
function namespace (line 284) | namespace digest_converter_internal {
FILE: riegeli/digests/digester_handle.cc
type riegeli (line 32) | namespace riegeli {
FILE: riegeli/digests/digester_handle.h
function namespace (line 44) | namespace riegeli {
function class (line 127) | class DigesterBaseHandle : public WithEqual<DigesterBaseHandle> {
function SetWriteSizeHint (line 155) | void SetWriteSizeHint(std::optional<Position> write_size_hint) {
function Write (line 162) | bool Write(char src) { return Write(absl::string_view(&src, 1)); }
function Write (line 164) | bool Write(char8_t src) { return Write(static_cast<char>(src)); }
function Write (line 166) | bool Write(BytesRef src) { return methods()->write(target(), src); }
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 167) | ABSL_ATTRIBUTE_ALWAYS_INLINE
function Write (line 169) | bool Write(const Chain& src) { return methods()->write_chain(target(), s...
function Write (line 170) | bool Write(const absl::Cord& src) {
function Write (line 173) | bool Write(ByteFill src) { return methods()->write_byte_fill(target(), s...
function Close (line 213) | bool Close() { return methods()->close(target()); }
function SetWriteSizeHintMethod (line 295) | void SetWriteSizeHintMethod(TypeErasedRef target,
function WriteMethod (line 308) | bool WriteMethod(TypeErasedRef target, absl::string_view src) {
function WriteChainMethod (line 320) | bool WriteChainMethod(TypeErasedRef target, const Chain& src) {
function WriteCordMethod (line 338) | bool WriteCordMethod(TypeErasedRef target, const absl::Cord& src) {
function WriteByteFillMethod (line 356) | bool WriteByteFillMethod(TypeErasedRef target, ByteFill src) {
function CloseMethod (line 367) | bool CloseMethod(TypeErasedRef target) {
type Methods (line 385) | struct Methods {
function explicit (line 413) | explicit DigesterBaseHandle(const Methods* methods, TypeErasedRef target)
function Methods (line 416) | const Methods* methods() const { return methods_; }
function namespace (line 426) | namespace digester_handle_internal {
function DigestType (line 521) | static DigestType DigestMethodDefault(
function DigestType (line 527) | DigestType DigestMethod(TypeErasedRef target) {
function Methods (line 537) | struct Methods : DigesterBaseHandle::Methods {
function Methods (line 549) | const Methods* methods() const {
function class (line 605) | class DigesterBaseHandle::DigesterStringifySink {
FILE: riegeli/digests/digesting_reader.cc
type riegeli (line 37) | namespace riegeli {
function ABSL_ATTRIBUTE_COLD (line 41) | ABSL_ATTRIBUTE_COLD absl::Status FailedStatus(DigesterBaseHandle diges...
FILE: riegeli/digests/digesting_reader.h
function namespace (line 43) | namespace riegeli {
function MakeBuffer (line 285) | inline void DigestingReaderBase::MakeBuffer(Reader& src) {
function Done (line 307) | void Done(DigestingReader& self) {
FILE: riegeli/digests/digesting_writer.cc
type riegeli (line 36) | namespace riegeli {
type digesting_writer_internal (line 38) | namespace digesting_writer_internal {
function FailedStatus (line 40) | absl::Status FailedStatus(DigesterBaseHandle digester) {
function Reader (line 157) | Reader* DigestingWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/digests/digesting_writer.h
function namespace (line 48) | namespace riegeli {
function MakeBuffer (line 322) | inline void DigestingWriterBase::MakeBuffer(Writer& dest) {
function Done (line 344) | void Done(DigestingWriter& self) {
function namespace (line 445) | namespace digesting_writer_internal {
function DesiredDigestType (line 459) | DesiredDigestType DigestFromImpl(std::tuple<Srcs...> srcs,
function DesiredDigestType (line 484) | DesiredDigestType DigestFromImpl(std::tuple<Srcs...> srcs,
FILE: riegeli/digests/highwayhash_digester.cc
type riegeli (line 19) | namespace riegeli {
FILE: riegeli/digests/highwayhash_digester.h
function namespace (line 30) | namespace riegeli {
FILE: riegeli/digests/md5_digester.h
function namespace (line 22) | namespace riegeli {
FILE: riegeli/digests/openssl_digester.h
function namespace (line 26) | namespace riegeli {
FILE: riegeli/digests/sha1_digester.h
function namespace (line 22) | namespace riegeli {
FILE: riegeli/digests/sha256_digester.h
function namespace (line 22) | namespace riegeli {
FILE: riegeli/digests/sha512_256_digester.h
function namespace (line 22) | namespace riegeli {
FILE: riegeli/digests/sha512_digester.h
function namespace (line 22) | namespace riegeli {
FILE: riegeli/digests/wrapping_digester.h
function namespace (line 37) | namespace riegeli {
function SetWriteSizeHint (line 107) | void SetWriteSizeHint(std::optional<Position> write_size_hint) {
function Write (line 111) | bool Write(absl::string_view src) { return base_.get().Write(src); }
function Write (line 112) | bool Write(const Chain& src) { return base_.get().Write(src); }
function Write (line 113) | bool Write(const absl::Cord& src) { return base_.get().Write(src); }
function Write (line 114) | bool Write(ByteFill src) { return base_.get().Write(src); }
function Close (line 115) | bool Close() { return !base_.IsOwning() || base_.get().Close(); }
FILE: riegeli/gcs/gcs_internal.h
function namespace (line 22) | namespace riegeli::gcs_internal {
FILE: riegeli/gcs/gcs_object.cc
type riegeli (line 38) | namespace riegeli {
function IsUtf8Tail (line 44) | inline bool IsUtf8Tail(char ch, unsigned char start = 0x80,
FILE: riegeli/gcs/gcs_object.h
function namespace (line 34) | namespace riegeli {
function GcsObject (line 181) | inline GcsObject::GcsObject(absl::string_view uri) { ParseUri(uri); }
function Reset (line 183) | inline void GcsObject::Reset() {
function Reset (line 190) | inline void GcsObject::Reset(StringInitializer bucket_name,
function Reset (line 200) | inline void GcsObject::Reset(absl::string_view uri) {
FILE: riegeli/gcs/gcs_reader.cc
type riegeli (line 43) | namespace riegeli {
FILE: riegeli/gcs/gcs_reader.h
function namespace (line 49) | namespace riegeli {
function Initialize (line 241) | void GcsReader::Initialize(const Options& options,
function SetReadObject (line 289) | void GcsReader::SetReadObject(
FILE: riegeli/gcs/gcs_writer.cc
type riegeli (line 44) | namespace riegeli {
function Reader (line 226) | Reader* GcsWriter::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/gcs/gcs_writer.h
function namespace (line 59) | namespace riegeli {
function google (line 199) | const google::cloud::storage::ObjectMetadata& metadata() const&;
function Initialize (line 450) | void GcsWriter::Initialize(
function SetFunctions (line 480) | void GcsWriter::SetFunctions(
function google (line 539) | inline google::cloud::storage::ObjectMetadata&& GcsWriter::metadata() && {
FILE: riegeli/lines/line_reading.cc
type riegeli (line 34) | namespace riegeli {
function ReadFlatAndSkip (line 45) | inline void ReadFlatAndSkip(Reader& src, size_t length_to_read,
function ReadFlatAndSkip (line 57) | inline void ReadFlatAndSkip(Reader& src, size_t length_to_read,
function ReadFlatAndSkip (line 69) | inline void ReadFlatAndSkip(Reader& src, size_t length_to_read,
function ReadFlat (line 82) | inline void ReadFlat(Reader& src, size_t length, Dest& dest) {
function ABSL_ATTRIBUTE_COLD (line 87) | ABSL_ATTRIBUTE_COLD bool FailMaxLineLengthExceeded(Reader& src, Dest& ...
function FoundNewline (line 95) | inline bool FoundNewline(Reader& src, Dest& dest, ReadLineOptions opti...
function ReadLineInternal (line 107) | inline bool ReadLineInternal(Reader& src, Dest& dest, ReadLineOptions ...
function ReadLine (line 179) | bool ReadLine(Reader& src, absl::string_view& dest, ReadLineOptions op...
function ReadLine (line 244) | bool ReadLine(Reader& src, std::string& dest, ReadLineOptions options) {
function ReadLine (line 249) | bool ReadLine(Reader& src, Chain& dest, ReadLineOptions options) {
function ReadLine (line 254) | bool ReadLine(Reader& src, absl::Cord& dest, ReadLineOptions options) {
function SkipUtf8Bom (line 259) | void SkipUtf8Bom(Reader& src) {
FILE: riegeli/lines/line_reading.h
function namespace (line 31) | namespace riegeli {
FILE: riegeli/lines/line_writing.h
function namespace (line 32) | namespace riegeli {
FILE: riegeli/lines/newline.h
type class (line 23) | enum class
function WriteNewline (line 29) | enum class WriteNewline {
FILE: riegeli/lines/text_reader.cc
type riegeli (line 33) | namespace riegeli {
type text_reader_internal (line 95) | namespace text_reader_internal {
FILE: riegeli/lines/text_reader.h
function namespace (line 39) | namespace riegeli {
function explicit (line 165) | explicit TextReader(Closed) noexcept
FILE: riegeli/lines/text_writer.cc
type riegeli (line 33) | namespace riegeli {
type text_writer_internal (line 53) | namespace text_writer_internal {
class TextWriterImpl<WriteNewline::kCrLf> (line 82) | class TextWriterImpl<WriteNewline::kCrLf>
FILE: riegeli/lines/text_writer.h
function namespace (line 38) | namespace riegeli {
function explicit (line 130) | explicit TextWriter(Closed) noexcept
function Initialize (line 195) | inline void TextWriterBase::Initialize(Writer* dest) {
FILE: riegeli/lz4/lz4_dictionary.cc
type riegeli (line 30) | namespace riegeli {
function LZ4F_CDict (line 36) | inline const LZ4F_CDict* Lz4Dictionary::Repr::PrepareCompressionDictio...
function LZ4F_CDict (line 44) | const LZ4F_CDict* Lz4Dictionary::PrepareCompressionDictionary() const
FILE: riegeli/lz4/lz4_dictionary.h
type LZ4F_CDict (line 38) | typedef struct LZ4F_CDict_s LZ4F_CDict;
function namespace (line 40) | namespace riegeli {
FILE: riegeli/lz4/lz4_reader.cc
type riegeli (line 41) | namespace riegeli {
type lz4_internal (line 336) | namespace lz4_internal {
function GetFrameInfo (line 338) | inline bool GetFrameInfo(Reader& src, LZ4F_frameInfo_t& frame_info,
function RecognizeLz4 (line 375) | bool RecognizeLz4(Reader& src,
function Lz4UncompressedSize (line 381) | std::optional<Position> Lz4UncompressedSize(
FILE: riegeli/lz4/lz4_reader.h
function namespace (line 40) | namespace riegeli {
function explicit (line 277) | explicit Lz4Reader(Closed) -> Lz4Reader<DeleteCtad<Closed>>;
function Reset (line 350) | inline void Lz4ReaderBase::Reset(
FILE: riegeli/lz4/lz4_writer.cc
type riegeli (line 43) | namespace riegeli {
function Reader (line 268) | Reader* Lz4WriterBase::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/lz4/lz4_writer.h
function namespace (line 40) | namespace riegeli {
function RecyclingPoolOptions (line 233) | const RecyclingPoolOptions& recycling_pool_options() const
function Writer (line 351) | Writer* DestWriter() const ABSL_ATTRIBUTE_LIFETIME_BOUND override {
function explicit (line 364) | explicit Lz4Writer(Closed) -> Lz4Writer<DeleteCtad<Closed>>;
function Reset (line 425) | inline void Lz4WriterBase::Reset(
FILE: riegeli/messages/context_projection.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 34) | ABSL_POINTERS_DEFAULT_NONNULL
function namespace (line 83) | namespace context_projection_internal {
function Status (line 156) | Status HandleVarint(uint64_t repr, Context&... context) const {
function Status (line 179) | Status DynamicHandleVarint(Accepted&& accepted, uint64_t repr,
function Status (line 195) | Status HandleFixed32(uint32_t repr, Context&... context) const {
function Status (line 218) | Status DynamicHandleFixed32(Accepted&& accepted, uint32_t repr,
function Status (line 234) | Status HandleFixed64(uint64_t repr, Context&... context) const {
function Status (line 257) | Status DynamicHandleFixed64(Accepted&& accepted, uint64_t repr,
function Status (line 274) | Status HandleLengthDelimitedFromReader(ReaderSpan<> repr,
function Status (line 291) | Status HandleLengthDelimitedFromCord(CordIteratorSpan repr,
function Status (line 309) | Status HandleLengthDelimitedFromString(absl::string_view repr,
function Status (line 336) | Status DynamicHandleLengthDelimitedFromReader(
function Status (line 354) | Status DynamicHandleLengthDelimitedFromCord(Accepted&& accepted,
function Status (line 374) | Status DynamicHandleLengthDelimitedFromString(
function Status (line 390) | Status HandleStartGroup(Context&... context) const {
function Status (line 413) | Status DynamicHandleStartGroup(Accepted&& accepted,
function Status (line 429) | Status HandleEndGroup(Context&... context) const {
function Status (line 452) | Status DynamicHandleEndGroup(Accepted&& accepted,
FILE: riegeli/messages/dynamic_field_handler.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 32) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/field_copier.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 36) | ABSL_POINTERS_DEFAULT_NONNULL
function Status (line 133) | Status HandleVarint(uint64_t repr, Context&... context) const {
function Status (line 141) | Status HandleFixed32(uint32_t repr, Context&... context) const {
function Status (line 149) | Status HandleFixed64(uint64_t repr, Context&... context) const {
function Status (line 158) | Status HandleLengthDelimitedFromReader(ReaderSpan<> repr,
function Status (line 169) | Status HandleLengthDelimitedFromCord(
function Status (line 181) | Status HandleLengthDelimitedFromString(absl::string_view repr,
function Status (line 191) | Status HandleStartGroup(Context&... context) const {
function Status (line 199) | Status HandleEndGroup(Context&... context) const {
function Status (line 240) | Status DynamicHandleVarint(int field_number, uint64_t repr,
function Status (line 257) | Status DynamicHandleFixed32(int field_number, uint32_t repr,
function Status (line 274) | Status DynamicHandleFixed64(int field_number, uint64_t repr,
function Status (line 289) | Status DynamicHandleLengthDelimitedFromReader(
function Status (line 300) | Status DynamicHandleLengthDelimitedFromCord(
function Status (line 312) | Status DynamicHandleLengthDelimitedFromString(
function Status (line 331) | Status DynamicHandleStartGroup(int field_number,
function Status (line 348) | Status DynamicHandleEndGroup(int field_number,
type AcceptAnyField (line 363) | struct AcceptAnyField {
function AnyFieldCopierType (line 367) | constexpr AnyFieldCopierType AnyFieldCopier() {
function field_number_ (line 376) | constexpr UnboundFieldCopierType(int field_number)
function Status (line 387) | Status HandleVarint(uint64_t repr, Context&... context) const {
function Status (line 395) | Status HandleFixed32(uint32_t repr, Context&... context) const {
function Status (line 403) | Status HandleFixed64(uint64_t repr, Context&... context) const {
function Status (line 412) | Status HandleLengthDelimitedFromReader(ReaderSpan<> repr,
function Status (line 423) | Status HandleLengthDelimitedFromCord(
function Status (line 435) | Status HandleLengthDelimitedFromString(absl::string_view repr,
function Status (line 445) | Status HandleStartGroup(Context&... context) const {
function Status (line 453) | Status HandleEndGroup(Context&... context) const {
FILE: riegeli/messages/field_handler_map.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 40) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/field_handlers.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 28) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/field_handlers.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 44) | ABSL_POINTERS_DEFAULT_NONNULL
function namespace (line 465) | namespace field_handlers_internal {
function Status (line 763) | Status HandleVarint(uint64_t repr, Context&... context) const {
function Status (line 774) | Status HandleFixed32(uint32_t repr, Context&... context) const {
function Status (line 785) | Status HandleFixed64(uint64_t repr, Context&... context) const {
function Status (line 797) | Status HandleLengthDelimitedFromReader(ReaderSpan<> repr,
function Status (line 811) | Status HandleLengthDelimitedFromCord(CordIteratorSpan repr,
function Status (line 826) | Status HandleLengthDelimitedFromString(absl::string_view repr,
function Status (line 858) | Status HandleLengthDelimitedFromReader(ReaderSpan<> repr,
FILE: riegeli/messages/map_entry_field.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 20) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/message_wire_format.h
function WireType (line 30) | enum class WireType : uint32_t {
function MakeTag (line 89) | constexpr uint32_t MakeTag(int field_number, WireType wire_type) {
function WireType (line 94) | constexpr WireType GetTagWireType(uint32_t tag) {
function GetTagFieldNumber (line 98) | constexpr int GetTagFieldNumber(uint32_t tag) {
FILE: riegeli/messages/parse_message.cc
type riegeli (line 47) | namespace riegeli {
function ABSL_ATTRIBUTE_COLD (line 51) | ABSL_ATTRIBUTE_COLD inline absl::Status ParseError(
function ABSL_ATTRIBUTE_COLD (line 57) | ABSL_ATTRIBUTE_COLD inline absl::Status ParseError(
function CheckInitialized (line 62) | inline absl::Status CheckInitialized(google::protobuf::MessageLite& dest,
function CheckInitialized (line 73) | inline absl::Status CheckInitialized(Reader& src,
function ParseMessageFromReaderSpanImpl (line 86) | absl::Status ParseMessageFromReaderSpanImpl(ReaderSpan<ReaderType> src,
function ParseLengthPrefixedMessageImpl (line 134) | absl::Status ParseLengthPrefixedMessageImpl(Src& src,
type parse_message_internal (line 150) | namespace parse_message_internal {
function ParseMessageImpl (line 152) | absl::Status ParseMessageImpl(Reader& src, google::protobuf::Message...
function ParseMessageImpl (line 189) | absl::Status ParseMessageImpl(ReaderSpan<Reader> src,
function ParseMessageImpl (line 195) | absl::Status ParseMessageImpl(ReaderSpan<> src,
function ParseLengthPrefixedMessage (line 203) | absl::Status ParseLengthPrefixedMessage(Reader& src,
function ParseLengthPrefixedMessage (line 209) | absl::Status ParseLengthPrefixedMessage(LimitingReaderBase& src,
function ParseMessage (line 215) | absl::Status ParseMessage(BytesRef src, google::protobuf::MessageLite&...
function ParseMessage (line 238) | absl::Status ParseMessage(const Chain& src, google::protobuf::MessageL...
function ParseMessage (line 274) | absl::Status ParseMessage(const absl::Cord& src,
function ParseMessage (line 311) | absl::Status ParseMessage(CordIteratorSpan src,
FILE: riegeli/messages/parse_message.h
function namespace (line 39) | namespace riegeli {
FILE: riegeli/messages/serialize_message.cc
type riegeli (line 48) | namespace riegeli {
function ABSL_ATTRIBUTE_COLD (line 52) | ABSL_ATTRIBUTE_COLD inline absl::Status FailSizeOverflow(
function ABSL_ATTRIBUTE_COLD (line 59) | ABSL_ATTRIBUTE_COLD inline absl::Status FailSizeOverflow(
function ABSL_ATTRIBUTE_COLD (line 64) | ABSL_ATTRIBUTE_COLD inline absl::Status FailSizeOverflow(
function SerializeMessageUsingStream (line 70) | inline absl::Status SerializeMessageUsingStream(
function SerializeMessageHavingSize (line 95) | inline absl::Status SerializeMessageHavingSize(
function SerializeMessageHavingSize (line 123) | inline absl::Status SerializeMessageHavingSize(
type serialize_message_internal (line 163) | namespace serialize_message_internal {
function SerializeMessageImpl (line 165) | absl::Status SerializeMessageImpl(const google::protobuf::MessageLit...
function SerializeMessageImpl (line 181) | absl::Status SerializeMessageImpl(const google::protobuf::MessageLit...
function SerializeLengthPrefixedMessage (line 200) | absl::Status SerializeLengthPrefixedMessage(
function SerializeMessage (line 218) | absl::Status SerializeMessage(const google::protobuf::MessageLite& src,
function SerializeMessage (line 265) | absl::Status SerializeMessage(const google::protobuf::MessageLite& src,
function SerializeMessage (line 304) | absl::Status SerializeMessage(const google::protobuf::MessageLite& src,
function SerializeMessage (line 346) | absl::Status SerializeMessage(const google::protobuf::MessageLite& src,
FILE: riegeli/messages/serialize_message.h
function namespace (line 42) | namespace riegeli {
FILE: riegeli/messages/serialized_message_assembler.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 39) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/serialized_message_assembler.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 51) | ABSL_POINTERS_DEFAULT_NONNULL
function Status (line 548) | Status SerializedMessageAssembler::RewriteFields(
FILE: riegeli/messages/serialized_message_backward_writer.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 35) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/serialized_message_backward_writer.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 46) | ABSL_POINTERS_DEFAULT_NONNULL
function set_dest (line 273) | inline void SerializedMessageBackwardWriter::set_dest(
FILE: riegeli/messages/serialized_message_internal.h
function namespace (line 29) | namespace riegeli {
FILE: riegeli/messages/serialized_message_reader.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 32) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/serialized_message_reader.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 54) | ABSL_POINTERS_DEFAULT_NONNULL
function absl (line 1189) | inline absl::Status SkipLengthDelimitedFromReader(ReaderSpan<> value) {
function absl (line 1215) | inline absl::Status SkipLengthDelimitedFromCord(CordIteratorSpan value) {
FILE: riegeli/messages/serialized_message_reader_internal.h
function absl (line 604) | inline absl::Status AnnotateWithFieldNumber(absl::Status status,
function absl (line 614) | inline absl::Status AnnotateWithSourceAndFieldNumber(absl::Status status,
function ReadVarintField (line 654) | inline bool ReadVarintField(
function ReadFixed32Field (line 677) | inline bool ReadFixed32Field(
function ReadFixed64Field (line 700) | inline bool ReadFixed64Field(
function ReadLengthDelimitedFieldFromReader (line 723) | inline bool ReadLengthDelimitedFieldFromReader(
function ReadLengthDelimitedFieldFromCord (line 772) | inline bool ReadLengthDelimitedFieldFromCord(
function ReadLengthDelimitedFieldFromString (line 814) | inline bool ReadLengthDelimitedFieldFromString(
function ReadStartGroupField (line 839) | inline bool ReadStartGroupField(
function ReadEndGroupField (line 862) | inline bool ReadEndGroupField(
FILE: riegeli/messages/serialized_message_writer.cc
function ABSL_POINTERS_DEFAULT_NONNULL (line 40) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/serialized_message_writer.h
function ABSL_POINTERS_DEFAULT_NONNULL (line 49) | ABSL_POINTERS_DEFAULT_NONNULL
FILE: riegeli/messages/text_parse_message.cc
type riegeli (line 35) | namespace riegeli {
type text_parse_message_internal (line 37) | namespace text_parse_message_internal {
function TextParseMessageImpl (line 60) | absl::Status TextParseMessageImpl(Reader& src, google::protobuf::Mes...
type text_parse_message_internal (line 58) | namespace text_parse_message_internal {
function TextParseMessageImpl (line 60) | absl::Status TextParseMessageImpl(Reader& src, google::protobuf::Mes...
function TextParseMessage (line 77) | absl::Status TextParseMessage(BytesRef src, google::protobuf::Message&...
function TextParseMessage (line 82) | absl::Status TextParseMessage(const Chain& src, google::protobuf::Mess...
function TextParseMessage (line 87) | absl::Status TextParseMessage(const absl::Cord& src,
FILE: riegeli/messages/text_parse_message.h
function namespace (line 40) | namespace text_parse_message_internal {
function class (line 58) | class TextParseMessageOptions {
FILE: riegeli/messages/text_print_message.cc
type riegeli (line 33) | namespace riegeli {
type text_print_message_internal (line 35) | namespace text_print_message_internal {
function TextPrintMessageImpl (line 37) | absl::Status TextPrintMessageImpl(const google::protobuf::Message& src,
function TextPrintMessage (line 65) | absl::Status TextPrintMessage(const google::protobuf::Message& src,
function TextPrintMessage (line 71) | absl::Status TextPrintMessage(const google::protobuf::Message& src, Ch...
function TextPrintMessage (line 76) | absl::Status TextPrintMessage(const google::protobuf::Message& src,
FILE: riegeli/messages/text_print_message.h
function class (line 34) | class TextPrintMessageOptions {
FILE: riegeli/ordered_varint/ordered_varint_internal.h
function namespace (line 23) | namespace riegeli {
FILE: riegeli/ordered_varint/ordered_varint_reading.cc
type riegeli::ordered_varint_internal (line 24) | namespace riegeli::ordered_varint_internal {
function ReadOrderedVarint32Slow (line 26) | bool ReadOrderedVarint32Slow(Reader& src, uint32_t& dest) {
function ReadOrderedVarint64Slow (line 63) | bool ReadOrderedVarint64Slow(Reader& src, uint64_t& dest) {
FILE: riegeli/ordered_varint/ordered_varint_reading.h
function namespace (line 24) | namespace riegeli {
FILE: riegeli/ordered_varint/ordered_varint_writing.cc
type riegeli::ordered_varint_internal (line 25) | namespace riegeli::ordered_varint_internal {
function WriteOrderedVarint32Slow (line 27) | bool WriteOrderedVarint32Slow(uint32_t data, Writer& dest) {
function WriteOrderedVarint64Slow (line 57) | bool WriteOrderedVarint64Slow(uint64_t data, Writer& dest) {
FILE: riegeli/ordered_varint/ordered_varint_writing.h
function namespace (line 27) | namespace riegeli {
FILE: riegeli/records/block.h
function namespace (line 30) | namespace riegeli::records_internal {
FILE: riegeli/records/chunk_reader.cc
type riegeli (line 39) | namespace riegeli {
FILE: riegeli/records/chunk_reader.h
function namespace (line 34) | namespace riegeli {
function Reset (line 366) | inline void DefaultChunkReaderBase::Reset(Closed) {
function Reset (line 375) | inline void DefaultChunkReaderBase::Reset() {
FILE: riegeli/records/chunk_writer.cc
type riegeli (line 40) | namespace riegeli {
type records_internal (line 42) | namespace records_internal {
function Position (line 44) | Position PosAfterPadding(Position pos, Position padding) {
FILE: riegeli/records/chunk_writer.h
function namespace (line 35) | namespace riegeli {
function ChunkWriter (line 280) | inline ChunkWriter::ChunkWriter(ChunkWriter&& that) noexcept
function Reset (line 289) | inline void ChunkWriter::Reset(Closed) {
function Reset (line 294) | inline void ChunkWriter::Reset() {
function Flush (line 299) | inline bool ChunkWriter::Flush(FlushType flush_type) {
function Position (line 303) | inline Position ChunkWriter::PosAfterWriteChunk(
function DefaultChunkWriterBase (line 308) | inline DefaultChunkWriterBase::DefaultChunkWriterBase(
FILE: riegeli/records/record_position.cc
type riegeli (line 44) | namespace riegeli {
type records_internal (line 115) | namespace records_internal {
type Visitor (line 122) | struct Visitor {
FILE: riegeli/records/record_position.h
function namespace (line 37) | namespace riegeli {
function namespace (line 131) | namespace records_internal {
function class (line 185) | class ABSL_ATTRIBUTE_TRIVIAL_ABI FutureRecordPosition {
function RecordPosition (line 217) | inline RecordPosition::RecordPosition(uint64_t chunk_begin,
function namespace (line 225) | namespace records_internal {
function FutureChunkBegin (line 273) | inline FutureChunkBegin::FutureChunkBegin(FutureChunkBegin&& that) noexcept
function FutureRecordPosition (line 290) | inline FutureRecordPosition::FutureRecordPosition(RecordPosition pos) no...
function FutureRecordPosition (line 293) | inline FutureRecordPosition::FutureRecordPosition(
function FutureRecordPosition (line 297) | inline FutureRecordPosition::FutureRecordPosition(
function FutureRecordPosition (line 308) | inline FutureRecordPosition::FutureRecordPosition(
FILE: riegeli/records/record_reader.cc
type riegeli (line 60) | namespace riegeli {
class RecordsMetadataDescriptors::ErrorCollector (line 62) | class RecordsMetadataDescriptors::ErrorCollector
method RecordError (line 65) | void RecordError(absl::string_view filename, absl::string_view eleme...
method ErrorCollector (line 76) | explicit ErrorCollector(RecordsMetadataDescriptors* descriptors)
function RecordReaderBase (line 117) | RecordReaderBase& RecordReaderBase::operator=(
class RecordReaderBase::ChunkSearchTraits (line 534) | class RecordReaderBase::ChunkSearchTraits {
method ChunkSearchTraits (line 536) | explicit ChunkSearchTraits(RecordReaderBase* self)
method Empty (line 541) | bool Empty(Position low, Position high) const { return low >= high; }
method Middle (line 543) | std::optional<Position> Middle(Position low, Position high) const {
type ChunkSuffix (line 582) | struct ChunkSuffix {
FILE: riegeli/records/record_reader.h
function namespace (line 54) | namespace riegeli {
function RecordPosition (line 326) | RecordPosition last_pos() const;
function explicit (line 642) | explicit RecordReader(Closed) -> RecordReader<DeleteCtad<Closed>>;
function TryRecovery (line 664) | inline bool RecordReaderBase::TryRecovery() {
function namespace (line 688) | namespace record_reader_internal {
FILE: riegeli/records/record_writer.cc
type riegeli (line 66) | namespace riegeli {
class FileDescriptorCollector (line 70) | class FileDescriptorCollector {
method FileDescriptorCollector (line 72) | explicit FileDescriptorCollector(
method AddFile (line 77) | void AddFile(const google::protobuf::FileDescriptor* file_descriptor) {
function SetRecordType (line 93) | void SetRecordType(const google::protobuf::Descriptor& descriptor,
class RecordWriterBase::Worker (line 174) | class RecordWriterBase::Worker {
method Done (line 217) | virtual void Done() {}
class RecordWriterBase::SerialWorker (line 382) | class RecordWriterBase::SerialWorker : public Worker {
method OpenChunk (line 389) | void OpenChunk() override { chunk_encoder_->Clear(); }
function FutureRecordPosition (line 486) | FutureRecordPosition RecordWriterBase::SerialWorker::LastPos() const {
function FutureRecordPosition (line 494) | FutureRecordPosition RecordWriterBase::SerialWorker::Pos() const {
function Position (line 498) | Position RecordWriterBase::SerialWorker::EstimatedSize() const {
class RecordWriterBase::ParallelWorker (line 504) | class RecordWriterBase::ParallelWorker : public Worker {
method OpenChunk (line 513) | void OpenChunk() override { chunk_encoder_ = MakeChunkEncoder(); }
type ChunkPromises (line 531) | struct ChunkPromises {
type DoneRequest (line 537) | struct DoneRequest {
type AnnotateStatusRequest (line 540) | struct AnnotateStatusRequest {
type WriteChunkRequest (line 544) | struct WriteChunkRequest {
type WritePaddingRequest (line 548) | struct WritePaddingRequest {
type FlushRequest (line 551) | struct FlushRequest {
type Visitor (line 573) | struct Visitor {
type Visitor (line 783) | struct Visitor {
function FutureRecordPosition (line 807) | FutureRecordPosition RecordWriterBase::ParallelWorker::LastPos() const {
function FutureRecordPosition (line 817) | FutureRecordPosition RecordWriterBase::ParallelWorker::Pos() const {
function Position (line 826) | Position RecordWriterBase::ParallelWorker::EstimatedSize() const {
function RecordWriterBase (line 858) | RecordWriterBase& RecordWriterBase::operator=(
function FutureRecordPosition (line 1071) | FutureRecordPosition RecordWriterBase::LastPos() const {
function FutureRecordPosition (line 1085) | FutureRecordPosition RecordWriterBase::Pos() const {
function Position (line 1090) | Position RecordWriterBase::EstimatedSize() const {
FILE: riegeli/records/record_writer.h
function namespace (line 55) | namespace riegeli {
FILE: riegeli/records/skipped_region.cc
type riegeli (line 22) | namespace riegeli {
FILE: riegeli/records/skipped_region.h
function namespace (line 27) | namespace riegeli {
FILE: riegeli/records/tools/describe_riegeli_file.cc
type riegeli::tools (line 71) | namespace riegeli::tools {
function DescribeFileMetadataChunk (line 74) | absl::Status DescribeFileMetadataChunk(const Chunk& chunk,
function ReadRecords (line 105) | absl::Status ReadRecords(
function DescribeSimpleChunk (line 121) | absl::Status DescribeSimpleChunk(const Chunk& chunk,
function DescribeTransposedChunk (line 210) | absl::Status DescribeTransposedChunk(
function DescribeFile (line 276) | void DescribeFile(absl::string_view filename, Writer& report) {
function main (line 369) | int main(int argc, char** argv) {
FILE: riegeli/records/tools/records_benchmark.cc
function ABSL_FLAG (line 89) | ABSL_FLAG(uint64_t, max_size, uint64_t{100} * 1000 * 1000,
FILE: riegeli/records/tools/tfrecord_recognizer.cc
type riegeli (line 29) | namespace riegeli {
FILE: riegeli/records/tools/tfrecord_recognizer.h
function namespace (line 24) | namespace riegeli {
FILE: riegeli/snappy/framed/framed_snappy_reader.cc
type riegeli (line 39) | namespace riegeli {
function MaskChecksum (line 44) | inline uint32_t MaskChecksum(uint32_t x) {
function RecognizeFramedSnappy (line 293) | bool RecognizeFramedSnappy(Reader& src) {
FILE: riegeli/snappy/framed/framed_snappy_reader.h
function namespace (line 36) | namespace riegeli {
function Done (line 200) | void Done(FramedSnappyReader& self) {
FILE: riegeli/snappy/framed/framed_snappy_writer.cc
type riegeli (line 44) | namespace riegeli {
function Reader (line 176) | Reader* FramedSnappyWriterBase::ReadModeBehindScratch(Position initial...
FILE: riegeli/snappy/framed/framed_snappy_writer.h
function namespace (line 36) | namespace riegeli {
FILE: riegeli/snappy/hadoop/hadoop_snappy_reader.cc
type riegeli (line 38) | namespace riegeli {
FILE: riegeli/snappy/hadoop/hadoop_snappy_reader.h
function namespace (line 36) | namespace riegeli {
FILE: riegeli/snappy/hadoop/hadoop_snappy_writer.cc
type riegeli (line 40) | namespace riegeli {
function Reader (line 147) | Reader* HadoopSnappyWriterBase::ReadModeBehindScratch(Position initial...
FILE: riegeli/snappy/hadoop/hadoop_snappy_writer.h
function namespace (line 36) | namespace riegeli {
FILE: riegeli/snappy/snappy_reader.cc
type riegeli (line 38) | namespace riegeli {
type snappy_internal (line 83) | namespace snappy_internal {
function SnappyDecompressImpl (line 85) | absl::Status SnappyDecompressImpl(Reader& src, Writer& dest) {
function SnappyUncompressedSize (line 103) | std::optional<size_t> SnappyUncompressedSize(Reader& src) {
FILE: riegeli/snappy/snappy_reader.h
function namespace (line 35) | namespace riegeli {
FILE: riegeli/snappy/snappy_streams.cc
type riegeli::snappy_internal (line 29) | namespace riegeli::snappy_internal {
FILE: riegeli/snappy/snappy_streams.h
function namespace (line 26) | namespace riegeli::snappy_internal {
FILE: riegeli/snappy/snappy_writer.cc
type riegeli (line 44) | namespace riegeli {
function Reader (line 249) | Reader* SnappyWriterBase::ReadModeImpl(Position initial_pos) {
type snappy_internal (line 257) | namespace snappy_internal {
function SnappyCompressImpl (line 259) | absl::Status SnappyCompressImpl(Reader& src, Writer& dest,
function SnappyMaxCompressedSize (line 278) | size_t SnappyMaxCompressedSize(size_t uncompressed_size) {
FILE: riegeli/snappy/snappy_writer.h
function namespace (line 41) | namespace riegeli {
function Reset (line 270) | inline void SnappyWriterBase::Reset(Closed) {
function Reset (line 278) | inline void SnappyWriterBase::Reset() {
function Initialize (line 286) | inline void SnappyWriterBase::Initialize(Writer* dest, int compression_l...
function MoveUncompressed (line 295) | inline void SnappyWriterBase::MoveUncompressed(SnappyWriterBase& that) {
function namespace (line 348) | namespace snappy_internal {
FILE: riegeli/tensorflow/io/file_reader.cc
type riegeli::tensorflow (line 53) | namespace riegeli::tensorflow {
FILE: riegeli/tensorflow/io/file_reader.h
function namespace (line 46) | namespace riegeli {
FILE: riegeli/tensorflow/io/file_writer.cc
type riegeli::tensorflow (line 51) | namespace riegeli::tensorflow {
function Reader (line 304) | Reader* FileWriterBase::ReadModeImpl(Position initial_pos) {
FILE: riegeli/tensorflow/io/file_writer.h
function namespace (line 47) | namespace riegeli {
FILE: riegeli/tensorflow/io/tstring_writer.h
function namespace (line 30) | namespace riegeli::tensorflow {
FILE: riegeli/tensorflow/kernels/riegeli_dataset_ops.cc
type riegeli::tensorflow (line 49) | namespace riegeli::tensorflow {
class RiegeliDatasetOp (line 52) | class RiegeliDatasetOp : public ::tensorflow::data::DatasetOpKernel {
method MakeDataset (line 56) | void MakeDataset(::tensorflow::OpKernelContext* ctx,
class Dataset (line 83) | class Dataset : public ::tensorflow::data::DatasetBase {
method Dataset (line 85) | explicit Dataset(::tensorflow::OpKernelContext* ctx,
method MakeIteratorInternal (line 93) | std::unique_ptr<::tensorflow::data::IteratorBase> MakeIteratorInte...
method DebugString (line 112) | std::string DebugString() const override {
method CheckExternalState (line 116) | absl::Status CheckExternalState() const override {
method InputDatasets (line 120) | absl::Status InputDatasets(
method AsGraphDefInternal (line 128) | absl::Status AsGraphDefInternal(
class Iterator (line 143) | class Iterator : public ::tensorflow::data::DatasetIterator<Datase...
method Iterator (line 145) | explicit Iterator(const Params& params) : DatasetIterator(params...
method ABSL_LOCKS_EXCLUDED (line 150) | ABSL_LOCKS_EXCLUDED(mu_) {
method ABSL_LOCKS_EXCLUDED (line 206) | ABSL_LOCKS_EXCLUDED(mu_) {
method ABSL_LOCKS_EXCLUDED (line 221) | ABSL_LOCKS_EXCLUDED(mu_) {
method ABSL_EXCLUSIVE_LOCKS_REQUIRED (line 258) | ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
method ABSL_GUARDED_BY (line 275) | ABSL_GUARDED_BY(mu_) = 0;
FILE: riegeli/tensorflow/ops/riegeli_dataset_ops.cc
type riegeli::tensorflow (line 19) | namespace riegeli::tensorflow {
FILE: riegeli/text/ascii_align.h
function namespace (line 43) | namespace riegeli {
function friend (line 237) | friend auto RiegeliStringifiedSize(const AsciiCenterType& src) {
function Stringify (line 251) | void Stringify(WriterStringifySink& dest) const& { WriteTo(*dest.dest()); }
function Stringify (line 252) | void Stringify(WriterStringifySink& dest) && {
function friend (line 362) | friend auto RiegeliStringifiedSize(const AsciiRightType& src) {
function Stringify (line 376) | void Stringify(WriterStringifySink& dest) const& { WriteTo(*dest.dest()); }
function Stringify (line 377) | void Stringify(WriterStringifySink& dest) && {
function namespace (line 456) | namespace align_internal {
FILE: riegeli/text/concat.h
function namespace (line 30) | namespace riegeli {
FILE: riegeli/text/join.h
function namespace (line 36) | namespace riegeli {
function StringifiedSize (line 57) | struct DefaultFormatter {
function explicit (line 239) | explicit JoinType(Initializer<Src> src,
function friend (line 264) | friend auto RiegeliStringifiedSize(const JoinType& src) {
function Stringify (line 277) | void Stringify(WriterStringifySink& dest) const { WriteTo(*dest.dest()); }
FILE: riegeli/text/write_int.cc
type riegeli::write_int_internal (line 35) | namespace riegeli::write_int_internal {
function __m128i (line 45) | inline __m128i HexDigits<DigitCase::kLower>() {
function __m128i (line 51) | inline __m128i HexDigits<DigitCase::kUpper>() {
function __m128i (line 57) | inline __m128i WriteHex2Impl(uint8_t src) {
function __m128i (line 71) | inline __m128i WriteHex4Impl(uint16_t src) {
function __m128i (line 88) | inline __m128i WriteHex8Impl(uint32_t src) {
function __m128i (line 105) | inline __m128i WriteHex16Impl(uint64_t src) {
function T (line 124) | inline T DigitCaseDependent(T for_lower, T for_upper) {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 224) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline char* WriteHexImpl(uint8_t src, ch...
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 239) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline char* WriteHexImpl(uint16_t src, c...
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 271) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline char* WriteHexImpl(uint32_t src, c...
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 302) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline char* WriteHexImpl(uint64_t src, c...
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 333) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline char* WriteHexImpl(absl::uint128 src,
FILE: riegeli/text/write_int.h
function namespace (line 37) | namespace riegeli {
function friend (line 120) | friend Position RiegeliStringifiedSize(const DecType& src) {
function DigitCase (line 147) | enum class DigitCase {
function friend (line 233) | friend Position RiegeliStringifiedSize(const HexType& src) {
function namespace (line 267) | namespace write_int_internal {
FILE: riegeli/varint/varint_internal.h
function namespace (line 23) | namespace riegeli {
FILE: riegeli/varint/varint_reading.cc
type riegeli (line 32) | namespace riegeli {
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 45) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline T ReadVarintValue(const char* src,...
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 56) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ReadVarintFromReaderBufferLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 80) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ReadVarintFromReaderLoop(Read...
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 107) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ReadVarintFromCordBufferLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 131) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ReadVarintFromCordLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 159) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline size_t ReadVarintFromArrayLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 182) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline void CopyVarintValue(const char* src,
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 189) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline size_t CopyVarintFromReaderBufferL...
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 212) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline size_t CopyVarintFromReaderLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 236) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline size_t CopyVarintFromCordBufferLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 259) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline size_t CopyVarintFromCordLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 286) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline size_t CopyVarintFromArrayLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 309) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool SkipVarintFromReaderBufferLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 330) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool SkipVarintFromReaderLoop(Read...
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 353) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool SkipVarintFromCordBufferLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 375) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool SkipVarintFromCordLoop(
function ABSL_ATTRIBUTE_ALWAYS_INLINE (line 400) | ABSL_ATTRIBUTE_ALWAYS_INLINE inline size_t SkipVarintFromArrayLoop(
function ReadNativeEndian (line 420) | inline uint64_t ReadNativeEndian(const char* src) {
type varint_internal (line 428) | namespace varint_internal {
function ReadVarintFromReaderBuffer (line 431) | bool ReadVarintFromReaderBuffer(Reader& src, const char* cursor, T acc,
function ReadVarintFromReader (line 464) | bool ReadVarintFromReader(Reader& src, T acc, T& dest) {
function ReadVarintFromCordBuffer (line 483) | bool ReadVarintFromCordBuffer(absl::Cord::CharIterator& src, size_t ...
function ReadVarintFromCord (line 516) | bool ReadVarintFromCord(absl::Cord::CharIterator& src, size_t availa...
function ReadVarintFromArray (line 539) | size_t ReadVarintFromArray(const char* src, size_t available, T acc,...
function CopyVarintFromReaderBuffer (line 564) | size_t CopyVarintFromReaderBuffer(Reader& src, const char* cursor, c...
function CopyVarintFromReader (line 587) | size_t CopyVarintFromReader(Reader& src, char* dest) {
function CopyVarintFromCordBuffer (line 605) | size_t CopyVarintFromCordBuffer(absl::Cord::CharIterator& src, size_...
function CopyVarintFromCord (line 632) | size_t CopyVarintFromCord(absl::Cord::CharIterator& src, size_t avai...
function CopyVarintFromArray (line 651) | size_t CopyVarintFromArray(const char* src, size_t available, char* ...
function SkipVarintFromReaderBuffer (line 672) | bool SkipVarintFromReaderBuffer(Reader& src, const char* cursor) {
function SkipVarintFromReader (line 695) | bool SkipVarintFromReader(Reader& src) {
function SkipVarintFromCordBuffer (line 708) | bool SkipVarintFromCordBuffer(absl::Cord::CharIterator& src, size_t ...
function SkipVarintFromCord (line 734) | bool SkipVarintFromCord(absl::Cord::CharIterator& src, size_t availa...
function SkipVarintFromArray (line 751) | size_t SkipVarintFromArray(const char* src, size_t available) {
function CountVarints (line 768) | size_t CountVarints(absl::string_view value) {
function VerifyBools (line 806) | bool VerifyBools(absl::string_view value) {
FILE: riegeli/varint/varint_reading.h
function namespace (line 31) | namespace riegeli {
FILE: riegeli/varint/varint_writing.h
function namespace (line 29) | namespace riegeli {
FILE: riegeli/xz/xz_error.cc
type riegeli::xz_internal (line 25) | namespace riegeli::xz_internal {
function XzErrorToStatus (line 27) | absl::Status XzErrorToStatus(absl::string_view operation,
FILE: riegeli/xz/xz_error.h
function namespace (line 22) | namespace riegeli::xz_internal {
FILE: riegeli/xz/xz_reader.cc
type riegeli (line 40) | namespace riegeli {
function RecognizeXz (line 301) | bool RecognizeXz(Reader& src) {
FILE: riegeli/xz/xz_reader.h
function class (line 43) | class XzReaderBase : public BufferedReader {
FILE: riegeli/xz/xz_writer.cc
type riegeli (line 41) | namespace riegeli {
function Reader (line 233) | Reader* XzWriterBase::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/xz/xz_writer.h
function class (line 46) | class XzWriterBase : public BufferedWriter {
FILE: riegeli/zlib/zlib_dictionary.h
function namespace (line 30) | namespace riegeli {
FILE: riegeli/zlib/zlib_error.cc
type riegeli::zlib_internal (line 25) | namespace riegeli::zlib_internal {
function ZlibErrorToStatus (line 27) | absl::Status ZlibErrorToStatus(absl::string_view operation, int zlib_c...
FILE: riegeli/zlib/zlib_error.h
function namespace (line 22) | namespace riegeli::zlib_internal {
FILE: riegeli/zlib/zlib_reader.cc
type riegeli (line 43) | namespace riegeli {
function RecognizeZlib (line 312) | bool RecognizeZlib(Reader& src, ZlibReaderBase::Header header,
function GzipUncompressedSizeModulo4G (line 373) | std::optional<uint32_t> GzipUncompressedSizeModulo4G(Reader& src) {
FILE: riegeli/zlib/zlib_reader.h
function namespace (line 40) | namespace riegeli {
FILE: riegeli/zlib/zlib_writer.cc
type riegeli (line 41) | namespace riegeli {
function Reader (line 233) | Reader* ZlibWriterBase::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/zlib/zlib_writer.h
function namespace (line 36) | namespace riegeli {
FILE: riegeli/zstd/zstd_dictionary.cc
type riegeli (line 37) | namespace riegeli {
function ZSTD_DDict (line 74) | inline const ZSTD_DDict* ZstdDictionary::Repr::PrepareDecompressionDic...
function ZSTD_DDict (line 90) | const ZSTD_DDict* ZstdDictionary::PrepareDecompressionDictionary() const
FILE: riegeli/zstd/zstd_dictionary.h
function namespace (line 38) | namespace riegeli {
function class (line 158) | class ZstdDictionary::Repr {
function ZSTD_CDictCache (line 215) | struct ZstdDictionary::ZSTD_CDictCache {
function ZstdDictionary (line 229) | inline ZstdDictionary& ZstdDictionary::set_data(BytesInitializer data,
function ZstdDictionary (line 238) | inline ZstdDictionary& ZstdDictionary::set_data_unowned(
FILE: riegeli/zstd/zstd_reader.cc
type riegeli (line 47) | namespace riegeli {
function GetFrameHeader (line 332) | inline bool GetFrameHeader(Reader& src, ZSTD_frameHeader& header) {
function RecognizeZstd (line 348) | bool RecognizeZstd(Reader& src) {
function ZstdUncompressedSize (line 353) | std::optional<Position> ZstdUncompressedSize(Reader& src) {
function ZstdDictId (line 361) | std::optional<uint32_t> ZstdDictId(Reader& src) {
FILE: riegeli/zstd/zstd_reader.h
function namespace (line 40) | namespace riegeli {
function explicit (line 254) | explicit ZstdReader(Closed) -> ZstdReader<DeleteCtad<Closed>>;
function Reset (line 331) | inline void ZstdReaderBase::Reset(
FILE: riegeli/zstd/zstd_writer.cc
type riegeli (line 47) | namespace riegeli {
function Reader (line 289) | Reader* ZstdWriterBase::ReadModeBehindBuffer(Position initial_pos) {
FILE: riegeli/zstd/zstd_writer.h
function namespace (line 40) | namespace riegeli {
function explicit (line 385) | explicit ZstdWriter(Closed) -> ZstdWriter<DeleteCtad<Closed>>;
function Reset (line 441) | inline void ZstdWriterBase::Reset(
Condensed preview — 498 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (5,402K chars).
[
{
"path": ".bazelrc",
"chars": 571,
"preview": "# Enable Bzlmod by default.\ncommon --enable_bzlmod\n\n# Use C++17.\nbuild --cxxopt=-std=c++17\nbuild --host_cxxopt=-std=c++1"
},
{
"path": "CONTRIBUTING.md",
"chars": 969,
"preview": "# How to Contribute\n\nWe'd love to accept your patches and contributions to this project. There are\njust a few small guid"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "MANIFEST.in",
"chars": 213,
"preview": "include .bazelrc\ninclude *.md\ninclude LICENSE\ninclude MANIFEST.in\ninclude WORKSPACE\ninclude configure\nrecursive-include "
},
{
"path": "MODULE.bazel",
"chars": 1726,
"preview": "module(\n name = \"riegeli\",\n repo_name = \"com_google_riegeli\",\n)\n\nbazel_dep(\n name = \"abseil-cpp\",\n version ="
},
{
"path": "README.md",
"chars": 651,
"preview": "# Riegeli\n\n*Riegeli/records* is a file format for storing a sequence of string records,\ntypically serialized protocol bu"
},
{
"path": "configure",
"chars": 1635,
"preview": "#!/bin/bash\n# Copyright 2019 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may no"
},
{
"path": "doc/index.md",
"chars": 488,
"preview": "# Riegeli\n\n*Riegeli/records* is a file format for storing a sequence of string records,\ntypically serialized protocol bu"
},
{
"path": "doc/record_writer_options.md",
"chars": 5671,
"preview": "# Specifying options for writing Riegeli/records files\n\nOptions for writing Riegeli/records files can be specified as a "
},
{
"path": "doc/riegeli_records_file_format.md",
"chars": 8833,
"preview": "# Riegeli/records file format specification\n\n## Summary\n\nFile contents are interpreted as a sequence of variable-sized *"
},
{
"path": "python/BUILD",
"chars": 670,
"preview": "load(\"@rules_python//python:defs.bzl\", \"py_binary\")\n\npackage(default_visibility = [\"//visibility:private\"])\n\nlicenses([\""
},
{
"path": "python/MANIFEST.in",
"chars": 31,
"preview": "recursive-include riegeli *.py\n"
},
{
"path": "python/README.md",
"chars": 651,
"preview": "# Riegeli\n\n*Riegeli/records* is a file format for storing a sequence of string records,\ntypically serialized protocol bu"
},
{
"path": "python/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "python/build_pip_package.sh",
"chars": 2049,
"preview": "#!/bin/bash\n# Copyright 2018 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may no"
},
{
"path": "python/dummy_binary.py",
"chars": 0,
"preview": ""
},
{
"path": "python/riegeli/BUILD",
"chars": 676,
"preview": "# Riegeli, file format for storing a sequence of records.\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npackage"
},
{
"path": "python/riegeli/BUILD.tpl",
"chars": 1302,
"preview": "load(\"@bazel_tools//tools/python:toolchain.bzl\", \"py_runtime_pair\")\n\nlicenses([\"restricted\"])\n\npackage(\n default_visi"
},
{
"path": "python/riegeli/__init__.py",
"chars": 2819,
"preview": "# Copyright 2018 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this f"
},
{
"path": "python/riegeli/base/BUILD",
"chars": 1072,
"preview": "load(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npackage(\n default_"
},
{
"path": "python/riegeli/base/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "python/riegeli/base/riegeli_error.py",
"chars": 7049,
"preview": "# Copyright 2018 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this f"
},
{
"path": "python/riegeli/base/utils.cc",
"chars": 13939,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/base/utils.h",
"chars": 17676,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/bytes/BUILD",
"chars": 1936,
"preview": "load(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\npackage(\n default_visibility = [\"//python/riegeli:__subpackages__\"],\n "
},
{
"path": "python/riegeli/bytes/python_reader.cc",
"chars": 13735,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/bytes/python_reader.h",
"chars": 6481,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/bytes/python_writer.cc",
"chars": 13922,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/bytes/python_writer.h",
"chars": 6273,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/py_extension.bzl",
"chars": 1509,
"preview": "\"\"\"Supports writing Python modules in C++.\"\"\"\n\nload(\"@rules_cc//cc:defs.bzl\", \"cc_binary\", \"cc_library\")\nload(\"@rules_py"
},
{
"path": "python/riegeli/python_configure.bzl",
"chars": 13699,
"preview": "\"\"\"Repository rule for Python autoconfiguration.\n\n`python_configure` depends on the following environment variables:\n\n "
},
{
"path": "python/riegeli/records/BUILD",
"chars": 3191,
"preview": "load(\"@com_google_protobuf//bazel:proto_library.bzl\", \"proto_library\")\nload(\"@com_google_protobuf//bazel:py_proto_librar"
},
{
"path": "python/riegeli/records/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "python/riegeli/records/examples/BUILD",
"chars": 309,
"preview": "load(\"@rules_python//python:defs.bzl\", \"py_binary\")\n\npackage(features = [\"header_modules\"])\n\nlicenses([\"notice\"])\n\npy_bi"
},
{
"path": "python/riegeli/records/examples/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "python/riegeli/records/examples/write_read_records.py",
"chars": 1880,
"preview": "# Copyright 2019 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this f"
},
{
"path": "python/riegeli/records/record_position.cc",
"chars": 17158,
"preview": "// Copyright 2019 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/records/record_position.h",
"chars": 1650,
"preview": "// Copyright 2019 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/records/record_reader.cc",
"chars": 63147,
"preview": "// Copyright 2019 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/records/record_writer.cc",
"chars": 36895,
"preview": "// Copyright 2019 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "python/riegeli/records/records_metadata.proto",
"chars": 1373,
"preview": "edition = \"2024\";\n\npackage riegeli;\n\nimport \"google/protobuf/descriptor.proto\";\n\n// Information about a Riegeli/records "
},
{
"path": "python/riegeli/records/skipped_region.py",
"chars": 1526,
"preview": "# Copyright 2018 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this f"
},
{
"path": "python/riegeli/records/tests/BUILD",
"chars": 847,
"preview": "load(\"@com_google_protobuf//bazel:proto_library.bzl\", \"proto_library\")\nload(\"@com_google_protobuf//bazel:py_proto_librar"
},
{
"path": "python/riegeli/records/tests/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "python/riegeli/records/tests/records_test.proto",
"chars": 106,
"preview": "edition = \"2024\";\n\npackage riegeli.tests;\n\nmessage SimpleMessage {\n int32 id = 1;\n bytes payload = 2;\n}\n"
},
{
"path": "python/riegeli/records/tests/records_test.py",
"chars": 53356,
"preview": "# Copyright 2018 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this f"
},
{
"path": "python/riegeli/tensorflow/BUILD",
"chars": 1422,
"preview": "load(\"@rules_cc//cc:defs.bzl\", \"cc_binary\")\nload(\"@rules_python//python:defs.bzl\", \"py_library\", \"py_test\")\n\npackage(\n "
},
{
"path": "python/riegeli/tensorflow/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "python/riegeli/tensorflow/kernel_tests/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "python/riegeli/tensorflow/kernel_tests/riegeli_dataset_test.py",
"chars": 9465,
"preview": "# Copyright 2019 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this f"
},
{
"path": "python/riegeli/tensorflow/ops/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "python/riegeli/tensorflow/ops/riegeli_dataset_ops.py",
"chars": 3000,
"preview": "# Copyright 2019 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this f"
},
{
"path": "python/setup.py",
"chars": 2161,
"preview": "# Copyright 2019 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this f"
},
{
"path": "riegeli/.gitignore",
"chars": 8,
"preview": "bazel-*\n"
},
{
"path": "riegeli/BUILD",
"chars": 205,
"preview": "# Riegeli, file format for storing a sequence of records.\n\npackage(\n default_visibility = [\"//visibility:public\"],\n "
},
{
"path": "riegeli/base/BUILD",
"chars": 18980,
"preview": "load(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\npackage(\n default_visibility = [\"//visibility:public\"],\n features = "
},
{
"path": "riegeli/base/any.h",
"chars": 33307,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/any_initializer.h",
"chars": 12642,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/any_internal.h",
"chars": 15518,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/arithmetic.h",
"chars": 13979,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/assert.cc",
"chars": 1843,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/assert.h",
"chars": 20927,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/background_cleaning.cc",
"chars": 4827,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/background_cleaning.h",
"chars": 5766,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/binary_search.h",
"chars": 21431,
"preview": "// Copyright 2020 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/buffer.cc",
"chars": 1135,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/buffer.h",
"chars": 4134,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/buffering.h",
"chars": 2959,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/byte_fill.cc",
"chars": 12883,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/byte_fill.h",
"chars": 19198,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/bytes_ref.h",
"chars": 7223,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/c_string_ref.h",
"chars": 4948,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/chain.cc",
"chars": 77272,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/chain.h",
"chars": 964,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/chain_base.h",
"chars": 34091,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/chain_details.h",
"chars": 42469,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/closing_ptr.h",
"chars": 1603,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/compact_string.cc",
"chars": 7565,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/compact_string.h",
"chars": 32790,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/compare.h",
"chars": 14905,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/constexpr.h",
"chars": 1131,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/cord_iterator_span.cc",
"chars": 2726,
"preview": "// Copyright 2025 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/cord_iterator_span.h",
"chars": 5626,
"preview": "// Copyright 2025 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/cord_utils.cc",
"chars": 3005,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/cord_utils.h",
"chars": 4475,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/debug.cc",
"chars": 5114,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/debug.h",
"chars": 11712,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/dependency.h",
"chars": 32013,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/dependency_base.h",
"chars": 5786,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/dependency_manager.h",
"chars": 16059,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/errno_mapping.cc",
"chars": 5586,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/errno_mapping.h",
"chars": 1142,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/estimated_allocated_size.h",
"chars": 1858,
"preview": "// Copyright 2019 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/external_data.cc",
"chars": 1089,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/external_data.h",
"chars": 1919,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/external_ref.h",
"chars": 985,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/external_ref_base.h",
"chars": 83976,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/external_ref_support.h",
"chars": 3735,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/global.h",
"chars": 5549,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/hybrid_direct_common.h",
"chars": 5709,
"preview": "// Copyright 2026 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/hybrid_direct_internal.h",
"chars": 9756,
"preview": "// Copyright 2026 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/hybrid_direct_map.h",
"chars": 43452,
"preview": "// Copyright 2025 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/hybrid_direct_set.h",
"chars": 24592,
"preview": "// Copyright 2025 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/initializer.h",
"chars": 31489,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/initializer_internal.h",
"chars": 2659,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/intrusive_shared_ptr.h",
"chars": 12833,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/invoker.h",
"chars": 18470,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/iterable.h",
"chars": 13660,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/maker.h",
"chars": 23774,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/memory_estimator.cc",
"chars": 2092,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/memory_estimator.h",
"chars": 24744,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/moving_dependency.h",
"chars": 13139,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/new_aligned.h",
"chars": 5130,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/null_safe_memcpy.h",
"chars": 3527,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/object.cc",
"chars": 4280,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/object.h",
"chars": 12317,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/optional_compact_string.h",
"chars": 7749,
"preview": "// Copyright 2025 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/options_parser.cc",
"chars": 9439,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/options_parser.h",
"chars": 9525,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/ownership.h",
"chars": 1692,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/parallelism.cc",
"chars": 2346,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/parallelism.h",
"chars": 1620,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/port.h",
"chars": 1119,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/recycling_pool.h",
"chars": 34633,
"preview": "// Copyright 2019 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/ref_count.h",
"chars": 2987,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/reset.h",
"chars": 5514,
"preview": "// Copyright 2020 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/shared_buffer.cc",
"chars": 1310,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/shared_buffer.h",
"chars": 4809,
"preview": "// Copyright 2020 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/shared_ptr.h",
"chars": 15873,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/sized_shared_buffer.cc",
"chars": 9124,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/sized_shared_buffer.h",
"chars": 8615,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/stable_dependency.h",
"chars": 10179,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/status.cc",
"chars": 1408,
"preview": "// Copyright 2019 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/status.h",
"chars": 4544,
"preview": "// Copyright 2019 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/stream_utils.cc",
"chars": 1584,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/stream_utils.h",
"chars": 7360,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/string_ref.h",
"chars": 8308,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/string_utils.cc",
"chars": 1166,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/string_utils.h",
"chars": 2237,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/temporary_storage.h",
"chars": 8465,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/type_erased_ref.h",
"chars": 3420,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/type_id.h",
"chars": 2416,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/type_traits.h",
"chars": 19784,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/types.h",
"chars": 2108,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/unicode.cc",
"chars": 3252,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/unicode.h",
"chars": 1365,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/base/uninitialized_vector.h",
"chars": 2847,
"preview": "// Copyright 2026 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/brotli/BUILD",
"chars": 2685,
"preview": "load(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\npackage(\n default_visibility = [\"//visibility:public\"],\n features = "
},
{
"path": "riegeli/brotli/brotli_allocator.cc",
"chars": 1073,
"preview": "// Copyright 2020 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/brotli/brotli_allocator.h",
"chars": 4486,
"preview": "// Copyright 2020 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/brotli/brotli_dictionary.cc",
"chars": 1831,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/brotli/brotli_dictionary.h",
"chars": 11011,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/brotli/brotli_reader.cc",
"chars": 9311,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/brotli/brotli_reader.h",
"chars": 10411,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/brotli/brotli_writer.cc",
"chars": 8610,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/brotli/brotli_writer.h",
"chars": 13452,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/BUILD",
"chars": 47060,
"preview": "load(\"@bazel_skylib//rules:common_settings.bzl\", \"bool_flag\")\nload(\"@rules_cc//cc:defs.bzl\", \"cc_library\")\n\npackage(\n "
},
{
"path": "riegeli/bytes/array_backward_writer.cc",
"chars": 2565,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/array_backward_writer.h",
"chars": 9619,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/array_writer.cc",
"chars": 4391,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/array_writer.h",
"chars": 9715,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/backward_writer.cc",
"chars": 10042,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/backward_writer.h",
"chars": 23012,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/buffer_options.cc",
"chars": 3921,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/buffer_options.h",
"chars": 15497,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/buffered_reader.cc",
"chars": 24302,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/buffered_reader.h",
"chars": 9095,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/buffered_writer.cc",
"chars": 8149,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/buffered_writer.h",
"chars": 6271,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_handle.cc",
"chars": 5118,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_handle.h",
"chars": 20154,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_internal.cc",
"chars": 1019,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_internal.h",
"chars": 1148,
"preview": "// Copyright 2024 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_internal_for_cc.h",
"chars": 2773,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_reader.cc",
"chars": 14570,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_reader.h",
"chars": 18100,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_writer.cc",
"chars": 18678,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cfile_writer.h",
"chars": 23709,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/chain_backward_writer.cc",
"chars": 8217,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/chain_backward_writer.h",
"chars": 13411,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/chain_reader.cc",
"chars": 9563,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/chain_reader.h",
"chars": 7561,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/chain_writer.cc",
"chars": 15067,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/chain_writer.h",
"chars": 15498,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/compact_string_writer.h",
"chars": 5597,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/copy_all.cc",
"chars": 5170,
"preview": "// Copyright 2018 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/copy_all.h",
"chars": 6477,
"preview": "// Copyright 2022 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cord_backward_writer.cc",
"chars": 13133,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cord_backward_writer.h",
"chars": 14023,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cord_reader.cc",
"chars": 9223,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cord_reader.h",
"chars": 10316,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cord_writer.cc",
"chars": 18896,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/cord_writer.h",
"chars": 16502,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/fd_handle.cc",
"chars": 5357,
"preview": "// Copyright 2021 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/fd_handle.h",
"chars": 21066,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/fd_internal.cc",
"chars": 1814,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/fd_internal.h",
"chars": 1396,
"preview": "// Copyright 2023 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/fd_internal_for_cc.h",
"chars": 2118,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/fd_mmap_reader.cc",
"chars": 11099,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
},
{
"path": "riegeli/bytes/fd_mmap_reader.h",
"chars": 20029,
"preview": "// Copyright 2017 Google LLC\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use th"
}
]
// ... and 298 more files (download for full content)
About this extraction
This page contains the full source code of the google/riegeli GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 498 files (5.0 MB), approximately 1.3M tokens, and a symbol index with 1901 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.