Repository: eevee/camel
Branch: master
Commit: 1f9132ce43f6
Files: 16
Total size: 85.2 KB
Directory structure:
gitextract_jx6rvyvg/
├── .gitignore
├── LICENSE
├── MANIFEST.in
├── README.txt
├── camel/
│ ├── __init__.py
│ └── tests/
│ ├── __init__.py
│ ├── test_docs.py
│ └── test_general.py
├── docs/
│ ├── Makefile
│ ├── api.rst
│ ├── camel.rst
│ ├── conf.py
│ ├── index.rst
│ ├── make.bat
│ └── yamlref.rst
└── setup.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
*.pyc
*.pyo
.cache/
*.egg-info/
__pycache__/
build/
dist/
docs/_build/
================================================
FILE: LICENSE
================================================
This project is licensed under the ISC license, reproduced below.
Copyright (c) 2012, Lexy "eevee" Munroe <eevee.camel@veekun.com>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
================================================
FILE: MANIFEST.in
================================================
include LICENSE
include README.txt
================================================
FILE: README.txt
================================================
Camel
=====
Camel is a library that lets you describe how to serialize your objects to
YAML — and refuses to serialize them if you don't.
Quick example:
.. code-block:: python
from camel import Camel, CamelRegistry
class DieRoll(tuple):
def __new__(cls, a, b):
return tuple.__new__(cls, [a, b])
def __repr__(self):
return "DieRoll(%s,%s)" % self
reg = CamelRegistry()
@reg.dumper(DieRoll, u'roll', version=None)
def _dump_dice(data):
return u"{}d{}".format(*data)
@reg.loader(u'roll', version=None)
def _load_dice(data, version):
a, _, b = data.partition(u'd')
return DieRoll(int(a), int(b))
value = DieRoll(3, 6)
camel = Camel([reg])
print(camel.dump(value))
# !roll 3d6
# ...
Docs: http://camel.readthedocs.org/en/latest/
GitHub: https://github.com/eevee/camel
================================================
FILE: camel/__init__.py
================================================
# encoding: utf8
# TODO
# i kinda need for pokedex purposes:
# - consider creating an @inherited_dumper? would need a way to pattern-match on the tag or something though, yikes. can this be done in a more controlled way, like you only get to pick a suffix to add to the tag? or is it useful in some other way to be able to pattern match on tags?
# - easy way to explicitly set the representer style (maybe a decorator arg?)
# - document tag_prefix! or was i thinking about the design of it? should you be able to override the tag prefix when adding to the camel?
# - tag_prefix=YAML_TAG_PREFIX is different from passing it as the actual tag name and i don't get why -- maybe because the default tag prefix is !
# - test and document inherit arg to @dumper
# - comment, test, document add_registry and tag_shorthand
#
# - expose the plain scalar parser things
#
# - support %TAG directives some nice reasonable way
# - support the attrs module, if installed, somehow
# - consider using (optionally?) ruamel.yaml, which roundtrips comments, merges, anchors, ...
# - DWIM formatting: block style except for very short sequences (if at all?), quotey style for long text...
# - make dumper/loader work on methods? ehh
# - "raw" loaders and dumpers that get to deal with raw yaml nodes?
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import base64
import collections
import functools
from io import StringIO
import types
import yaml
try:
from yaml import CSafeDumper as SafeDumper
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeDumper
from yaml import SafeLoader
YAML_TAG_PREFIX = 'tag:yaml.org,2002:'
_str = type('')
_bytes = type(b'')
_long = type(18446744073709551617) # 2**64 + 1
class CamelDumper(SafeDumper):
"""Subclass of yaml's `SafeDumper` that scopes representers to the
instance, rather than to the particular class, because damn.
"""
def __init__(self, *args, **kwargs):
# TODO this isn't quite good enough; pyyaml still escapes anything
# outside the BMP
kwargs.setdefault('allow_unicode', True)
super(CamelDumper, self).__init__(*args, **kwargs)
self.yaml_representers = SafeDumper.yaml_representers.copy()
self.yaml_multi_representers = SafeDumper.yaml_multi_representers.copy()
# Always dump bytes as binary, even on Python 2
self.add_representer(bytes, CamelDumper.represent_binary)
def represent_binary(self, data):
# This is copy-pasted, because it only exists in pyyaml in python 3 (?!)
if hasattr(base64, 'encodebytes'):
data = base64.encodebytes(data).decode('ascii')
else:
data = base64.encodestring(data).decode('ascii')
return self.represent_scalar(
YAML_TAG_PREFIX + 'binary', data, style='|')
def add_representer(self, data_type, representer):
self.yaml_representers[data_type] = representer
def add_multi_representer(self, data_type, representer):
self.yaml_multi_representers[data_type] = representer
class CamelLoader(SafeLoader):
"""Subclass of yaml's `SafeLoader` that scopes constructors to the
instance, rather than to the particular class, because damn.
"""
def __init__(self, *args, **kwargs):
super(CamelLoader, self).__init__(*args, **kwargs)
self.yaml_constructors = SafeLoader.yaml_constructors.copy()
self.yaml_multi_constructors = SafeLoader.yaml_multi_constructors.copy()
self.yaml_implicit_resolvers = SafeLoader.yaml_implicit_resolvers.copy()
def add_constructor(self, data_type, constructor):
self.yaml_constructors[data_type] = constructor
def add_multi_constructor(self, data_type, constructor):
self.yaml_multi_constructors[data_type] = constructor
def add_implicit_resolver(self, tag, regexp, first):
if first is None:
first = [None]
for ch in first:
self.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
def add_path_resolver(self, *args, **kwargs):
# This API is non-trivial and claims to be experimental and unstable
raise NotImplementedError
class Camel(object):
"""Class responsible for doing the actual dumping to and loading from YAML.
"""
def __init__(self, registries=()):
self.registries = collections.OrderedDict()
self.version_locks = {} # class => version
self.add_registry(STANDARD_TYPES)
for registry in registries:
self.add_registry(registry)
def add_registry(self, registry, tag_prefix=None, tag_shorthand=None):
self.registries[registry] = (
tag_prefix or registry.tag_prefix,
tag_shorthand or registry.tag_shorthand,
)
def lock_version(self, cls, version):
self.version_locks[cls] = version
def make_dumper(self, stream):
tag_shorthands = {}
for registry, (prefix, shorthand) in self.registries.items():
if shorthand is None:
continue
if shorthand in tag_shorthands:
raise ValueError(
"Conflicting tag shorthands: {!r} is short for both {!r} and {!r}"
.format(shorthand, tag_shorthands[shorthand], prefix))
tag_shorthands[shorthand] = prefix
dumper = CamelDumper(stream, default_flow_style=False, tags=tag_shorthands)
for registry in self.registries:
registry.inject_dumpers(dumper, version_locks=self.version_locks)
return dumper
def dump(self, data):
stream = StringIO()
dumper = self.make_dumper(stream)
dumper.open()
dumper.represent(data)
dumper.close()
return stream.getvalue()
def make_loader(self, stream):
loader = CamelLoader(stream)
for registry in self.registries:
registry.inject_loaders(loader)
return loader
def load(self, data):
stream = StringIO(data)
loader = self.make_loader(stream)
obj = loader.get_data()
if loader.check_node():
raise RuntimeError("Multiple documents found in stream; use load_all")
return obj
def load_first(self, data):
stream = StringIO(data)
loader = self.make_loader(stream)
return loader.get_data()
def load_all(self, data):
stream = StringIO(data)
loader = self.make_loader(stream)
while loader.check_node():
yield loader.get_data()
class DuplicateVersion(ValueError):
pass
class CamelRegistry(object):
frozen = False
def __init__(self, tag_prefix='!', tag_shorthand=None):
self.tag_prefix = tag_prefix
self.tag_shorthand = tag_shorthand
# type => {version => function)
self.dumpers = collections.defaultdict(dict)
self.multi_dumpers = collections.defaultdict(dict)
# base tag => {version => function}
self.loaders = collections.defaultdict(dict)
def freeze(self):
self.frozen = True
# Dumping
def _check_tag(self, tag):
# Good a place as any, I suppose
if self.frozen:
raise RuntimeError("Can't add to a frozen registry")
if ';' in tag:
raise ValueError(
"Tags may not contain semicolons: {0!r}".format(tag))
def dumper(self, cls, tag, version, inherit=False):
self._check_tag(tag)
if inherit:
store_in = self.multi_dumpers
else:
store_in = self.dumpers
if version in store_in[cls]:
raise DuplicateVersion
tag = self.tag_prefix + tag
if version is None:
full_tag = tag
elif isinstance(version, (int, _long)) and version > 0:
full_tag = "{0};{1}".format(tag, version)
else:
raise TypeError(
"Expected None or a positive integer version; "
"got {0!r} instead".format(version))
def decorator(f):
store_in[cls][version] = functools.partial(
self.run_representer, f, full_tag)
return f
return decorator
def run_representer(self, representer, tag, dumper, data):
canon_value = representer(data)
# Note that we /do not/ support subclasses of the built-in types here,
# to avoid complications from returning types that have their own
# custom representers
canon_type = type(canon_value)
# TODO this gives no control over flow_style, style, and implicit. do
# we intend to figure it out ourselves?
if canon_type is dict:
return dumper.represent_mapping(tag, canon_value, flow_style=False)
elif canon_type is collections.OrderedDict:
# pyyaml tries to sort the items of a dict, which defeats the point
# of returning an OrderedDict. Luckily, it only does this if the
# value it gets has an 'items' method; otherwise it skips the
# sorting and iterates the value directly, assuming it'll get
# key/value pairs. So pass in the dict's items iterator.
return dumper.represent_mapping(tag, canon_value.items(), flow_style=False)
elif canon_type in (tuple, list):
return dumper.represent_sequence(tag, canon_value, flow_style=False)
elif canon_type in (int, _long, float, bool, _str, type(None)):
return dumper.represent_scalar(tag, canon_value)
else:
raise TypeError(
"Representers must return native YAML types, but the representer "
"for {!r} returned {!r}, which is of type {!r}"
.format(data, canon_value, canon_type))
def inject_dumpers(self, dumper, version_locks=None):
if not version_locks:
version_locks = {}
for add_method, dumpers in [
(dumper.add_representer, self.dumpers),
(dumper.add_multi_representer, self.multi_dumpers),
]:
for cls, versions in dumpers.items():
version = version_locks.get(cls, max)
if versions and version is max:
if None in versions:
representer = versions[None]
else:
representer = versions[max(versions)]
elif version in versions:
representer = versions[version]
else:
raise KeyError(
"Don't know how to dump version {0!r} of type {1!r}"
.format(version, cls))
add_method(cls, representer)
# Loading
# TODO implement "upgrader", which upgrades from one version to another
def loader(self, tag, version):
self._check_tag(tag)
if version in self.loaders[tag]:
raise DuplicateVersion
tag = self.tag_prefix + tag
def decorator(f):
self.loaders[tag][version] = functools.partial(
self.run_constructor, f, version)
return f
return decorator
def run_constructor(self, constructor, version, *yaml_args):
# Two args for add_constructor, three for add_multi_constructor
if len(yaml_args) == 3:
loader, suffix, node = yaml_args
version = int(suffix)
else:
loader, node = yaml_args
if isinstance(node, yaml.ScalarNode):
data = loader.construct_scalar(node)
elif isinstance(node, yaml.SequenceNode):
data = loader.construct_sequence(node, deep=True)
elif isinstance(node, yaml.MappingNode):
data = loader.construct_mapping(node, deep=True)
else:
raise TypeError("Not a primitive node: {!r}".format(node))
return constructor(data, version)
def inject_loaders(self, loader):
for tag, versions in self.loaders.items():
# "all" loader overrides everything
if all in versions:
if None in versions:
loader.add_constructor(tag, versions[None])
else:
loader.add_constructor(tag, versions[all])
loader.add_multi_constructor(tag + ";", versions[all])
continue
# Otherwise, add each constructor individually
for version, constructor in versions.items():
if version is None:
loader.add_constructor(tag, constructor)
elif version is any:
loader.add_multi_constructor(tag + ";", versions[any])
if None not in versions:
loader.add_constructor(tag, versions[any])
else:
full_tag = "{0};{1}".format(tag, version)
loader.add_constructor(full_tag, constructor)
# YAML's "language-independent types" — not builtins, but supported with
# standard !! tags. Most of them are built into pyyaml, but OrderedDict is
# curiously overlooked. Loaded first by default into every Camel object.
# Ref: http://yaml.org/type/
# TODO pyyaml supports tags like !!python/list; do we care?
STANDARD_TYPES = CamelRegistry(tag_prefix=YAML_TAG_PREFIX)
@STANDARD_TYPES.dumper(frozenset, 'set', version=None)
def _dump_frozenset(data):
return dict.fromkeys(data)
@STANDARD_TYPES.dumper(collections.OrderedDict, 'omap', version=None)
def _dump_ordered_dict(data):
pairs = []
for key, value in data.items():
pairs.append({key: value})
return pairs
@STANDARD_TYPES.loader('omap', version=None)
def _load_ordered_dict(data, version):
return collections.OrderedDict(
pair for datum in data for (pair,) in [datum.items()]
)
# Extra Python types that don't have native YAML equivalents, but that PyYAML
# supports with !!python/foo tags. Dumping them isn't supported by default,
# but loading them is, since there's no good reason for it not to be.
# A couple of these dumpers override builtin type support. For example, tuples
# are dumped as lists by default, but this registry will dump them as
# !!python/tuple.
PYTHON_TYPES = CamelRegistry(tag_prefix=YAML_TAG_PREFIX)
@PYTHON_TYPES.dumper(tuple, 'python/tuple', version=None)
def _dump_tuple(data):
return list(data)
@STANDARD_TYPES.loader('python/tuple', version=None)
def _load_tuple(data, version):
return tuple(data)
@PYTHON_TYPES.dumper(complex, 'python/complex', version=None)
def _dump_complex(data):
ret = repr(data)
if str is bytes:
ret = ret.decode('ascii')
# Complex numbers become (1+2j), but the parens are superfluous
if ret[0] == '(' and ret[-1] == ')':
return ret[1:-1]
else:
return ret
@STANDARD_TYPES.loader('python/complex', version=None)
def _load_complex(data, version):
return complex(data)
@PYTHON_TYPES.dumper(frozenset, 'python/frozenset', version=None)
def _dump_frozenset(data):
try:
return list(sorted(data))
except TypeError:
return list(data)
@STANDARD_TYPES.loader('python/frozenset', version=None)
def _load_frozenset(data, version):
return frozenset(data)
if hasattr(types, 'SimpleNamespace'):
@PYTHON_TYPES.dumper(types.SimpleNamespace, 'python/namespace', version=None)
def _dump_simple_namespace(data):
return data.__dict__
@STANDARD_TYPES.loader('python/namespace', version=None)
def _load_simple_namespace(data, version):
return types.SimpleNamespace(**data)
STANDARD_TYPES.freeze()
PYTHON_TYPES.freeze()
================================================
FILE: camel/tests/__init__.py
================================================
================================================
FILE: camel/tests/test_docs.py
================================================
"""Make sure the documentation examples actually, uh, work."""
from __future__ import unicode_literals
import textwrap
def test_docs_table_v1():
class Table(object):
def __init__(self, size):
self.size = size
def __repr__(self):
return "<Table {self.size!r}>".format(self=self)
from camel import CamelRegistry
my_types = CamelRegistry()
@my_types.dumper(Table, 'table', version=1)
def _dump_table(table):
return {
'size': table.size,
}
@my_types.loader('table', version=1)
def _load_table(data, version):
return Table(data["size"])
from camel import Camel
table = Table(25)
assert Camel([my_types]).dump(table) == "!table;1\nsize: 25\n"
data = {'chairs': [], 'tables': [Table(25), Table(36)]}
assert Camel([my_types]).dump(data) == textwrap.dedent("""
chairs: []
tables:
- !table;1
size: 25
- !table;1
size: 36
""").lstrip()
table, = Camel([my_types]).load("[!table;1 {size: 100}]")
assert isinstance(table, Table)
assert table.size == 100
def test_docs_table_v2():
# Tables can be rectangles now!
class Table(object):
def __init__(self, height, width):
self.height = height
self.width = width
def __repr__(self):
return "<Table {self.height!r}x{self.width!r}>".format(self=self)
from camel import Camel, CamelRegistry
my_types = CamelRegistry()
@my_types.dumper(Table, 'table', version=2)
def _dump_table_v2(table):
return {
'height': table.height,
'width': table.width,
}
@my_types.loader('table', version=2)
def _load_table_v2(data, version):
return Table(data["height"], data["width"])
@my_types.loader('table', version=1)
def _load_table_v1(data, version):
edge = data["size"] ** 0.5
return Table(edge, edge)
table = Table(7, 10)
assert Camel([my_types]).dump(table) == textwrap.dedent("""
!table;2
height: 7
width: 10
""").lstrip()
@my_types.dumper(Table, 'table', version=1)
def _dump_table_v1(table):
return {
# not really, but the best we can manage
'size': table.height * table.width,
}
camel = Camel([my_types])
camel.lock_version(Table, 1)
assert camel.dump(Table(5, 7)) == "!table;1\nsize: 35\n"
def test_docs_deleted():
class DummyData(object):
def __init__(self, data):
self.data = data
from camel import Camel, CamelRegistry
my_types = CamelRegistry()
@my_types.loader('deleted-type', version=all)
def _load_deleted_type(data, version):
return DummyData(data)
camel = Camel([my_types])
assert isinstance(camel.load("""!deleted-type;4 foo"""), DummyData)
def test_docs_table_any():
class Table(object):
def __init__(self, height, width):
self.height = height
self.width = width
def __repr__(self):
return "<Table {self.height!r}x{self.width!r}>".format(self=self)
from camel import Camel, CamelRegistry
my_types = CamelRegistry()
@my_types.loader('table', version=any)
def _load_table(data, version):
if 'size' in data:
# version 1
edge = data['size'] ** 0.5
return Table(edge, edge)
else:
# version 2?)
return Table(data['height'], data['width'])
camel = Camel([my_types])
table1, table2 = camel.load(
"[!table;1 {size: 49}, !table;2 {height: 5, width: 9}]")
assert table1.height == 7
assert table1.width == 7
assert table2.height == 5
assert table2.width == 9
================================================
FILE: camel/tests/test_general.py
================================================
# encoding: utf8
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import collections
import datetime
import pytest
from camel import Camel, CamelRegistry, PYTHON_TYPES
# Round-trips for simple values of built-in types
@pytest.mark.parametrize(('value', 'expected_serialization'), [
# TODO the trailing ... for non-container values is kinda weird
(None, "null\n...\n"),
('ⓤⓝⓘⓒⓞⓓⓔ', "ⓤⓝⓘⓒⓞⓓⓔ\n...\n"),
(b'bytes', "!!binary |\n Ynl0ZXM=\n"),
(True, "true\n...\n"),
(133, "133\n...\n"),
# long, for python 2
(2**133, "10889035741470030830827987437816582766592\n...\n"),
(3.52, "3.52\n...\n"),
([1, 2, 'three'], "- 1\n- 2\n- three\n"),
({'x': 7, 'y': 8, 'z': 9}, "x: 7\ny: 8\nz: 9\n"),
# TODO this should use ? notation
(set("qvx"), "!!set\nq: null\nv: null\nx: null\n"),
(datetime.date(2015, 10, 21), "2015-10-21\n...\n"),
(datetime.datetime(2015, 10, 21, 4, 29), "2015-10-21 04:29:00\n...\n"),
# TODO case with timezone... unfortunately can't preserve the whole thing
(collections.OrderedDict([('a', 1), ('b', 2), ('c', 3)]), "!!omap\n- a: 1\n- b: 2\n- c: 3\n"),
])
def test_basic_roundtrip(value, expected_serialization):
camel = Camel()
dumped = camel.dump(value)
assert dumped == expected_serialization
assert camel.load(dumped) == value
def test_tuple_roundtrip():
# By default, tuples become lists
value = (4, 3, 2)
camel = Camel()
dumped = camel.dump(value)
# TODO short list like this should be flow style?
assert dumped == "- 4\n- 3\n- 2\n"
assert camel.load(dumped) == list(value)
def test_frozenset_roundtrip():
# By default, frozensets become sets
value = frozenset((4, 3, 2))
camel = Camel()
dumped = camel.dump(value)
# TODO this should use ? notation
assert dumped == "!!set\n2: null\n3: null\n4: null\n"
assert camel.load(dumped) == set(value)
# Round-trips for built-in Python types with custom representations
@pytest.mark.parametrize(('value', 'expected_serialization'), [
((4, 3, 2), "!!python/tuple\n- 4\n- 3\n- 2\n"),
(5 + 12j, "!!python/complex 5+12j\n...\n"),
(2j, "!!python/complex 2j\n...\n"),
(frozenset((4, 3, 2)), "!!python/frozenset\n- 2\n- 3\n- 4\n"),
])
def test_python_roundtrip(value, expected_serialization):
camel = Camel([PYTHON_TYPES])
dumped = camel.dump(value)
assert dumped == expected_serialization
# Should be able to load them without the python types
vanilla_camel = Camel()
assert vanilla_camel.load(dumped) == value
# -----------------------------------------------------------------------------
# Simple custom type
class DieRoll(tuple):
def __new__(cls, a, b):
return tuple.__new__(cls, [a, b])
def __repr__(self):
return "DieRoll(%s,%s)" % self
# Dump/load as a compact string
reg = CamelRegistry()
@reg.dumper(DieRoll, 'roll', version=None)
def dump_dice(data):
return "{}d{}".format(*data)
@reg.loader('roll', version=None)
def load_dice(data, version):
# TODO enforce incoming data is a string?
a, _, b = data.partition('d')
return DieRoll(int(a), int(b))
def test_dieroll():
value = DieRoll(3, 6)
camel = Camel([reg])
dumped = camel.dump(value)
assert dumped == '!roll 3d6\n...\n'
assert camel.load(dumped) == value
# Dump/load as a dict
reg2 = CamelRegistry()
@reg2.dumper(DieRoll, 'roll', version=None)
def dump_dice(data):
return collections.OrderedDict([
# NOTE: These are deliberately arranged in reverse alphabetical order,
# to ensure that we avoid pyyaml's default behavior of sorting a
# dict-like when dumping a mapping
('numdice', data[0]),
('faces', data[1]),
])
@reg2.loader('roll', version=None)
def load_dice(data, version):
# TODO enforce data is a dict?
# TODO enforce data contains only these keys?
return DieRoll(
int(data['numdice']),
int(data['faces']),
)
def test_dieroll2():
value = DieRoll(3, 6)
camel = Camel([reg2])
dumped = camel.dump(value)
assert dumped == '!roll\nnumdice: 3\nfaces: 6\n'
assert camel.load(dumped) == value
================================================
FILE: docs/Makefile
================================================
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build2
PAPER =
BUILDDIR = _build
# User-friendly check for sphinx-build2
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
clean:
rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Camel.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Camel.qhc"
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/Camel"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Camel"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
================================================
FILE: docs/api.rst
================================================
API reference
=============
.. automodule:: camel
:members:
:undoc-members:
================================================
FILE: docs/camel.rst
================================================
Camel overview
==============
Camel is intended as a replacement for libraries like :py:mod:`pickle` or
`PyYAML`_, which automagically serialize any type they come across. That seems
convenient at first, but in any large or long-lived application, the benefits
are soon outweighed by the costs:
.. _PyYAML: http://pyyaml.org/
* You can't move, rename, or delete any types that are encoded in a pickle.
* Even private implementation details of your class are encoded in a pickle by
default, which means you can't change them either.
* Because pickle's behavior is recursive, it can be difficult to know which
types are pickled.
* Because pickle's behavior is recursive, you may inadvertently pickle far more
data than necessary, if your objects have caches or reified properties. In
extreme cases you may pickle configuration that's no longer correct when the
pickle is loaded.
* Since pickles aren't part of your codebase and are rarely covered by tests,
you may not know you've broken pickles until your code hits production... or
much later.
* Pickle in particular is very opaque, even when using the ASCII format. It's
easy to end up with a needlessly large pickle by accident and have no
visibility into what's being stored in it, or to break loading a large pickle
and be unable to recover gracefully or even tell where the problem is.
* Automagically serialized data is hard enough to load back into your *own*
application. Loading it anywhere else is effectively out of the question.
It's certainly possible to whip pickle or PyYAML into shape manually by writing
``__reduce__`` or representer functions, but their default behavior is still
automagic, so you can't be sure you didn't miss something. Also, nobody
actually does it, so merely knowing it's possible doesn't help much.
Camel's philosophy
------------------
Explicit is better than implicit.
Complex is better than complicated.
Readability counts.
If the implementation is hard to explain, it's a bad idea.
*In the face of ambiguity, refuse the temptation to guess.*
— `The Zen of Python`_
.. _The Zen of Python: https://www.python.org/dev/peps/pep-0020/
Serialization is hard. We can't hide that difficulty, only delay it for a
while. And it *will* catch up with you.
A few people in the Python community have been rallying against pickle and its
ilk for a while, but when asked for alternatives, all we can do is mumble
something about writing functions. Well, that's not very helpful.
Camel forces you to write all your own serialization code, then wires it all
together for you. It's backed by YAML, which is ostensibly easy for humans to
read — and has explicit support for custom types. Hopefully, after using
Camel, you'll discover you've been tricked into making a library of every type
you serialize, the YAML name you give it, and exactly how it's formatted. All
of this lives in your codebase, so someone refactoring a class will easily
stumble upon its serialization code. Why, you could even use this knowledge to
load your types into an application written in a different language, or turn
them into a documented format!
Let's see some code already
---------------------------
Let's!
Here's the Table example from `a talk Alex Gaynor gave at PyCon US 2014`_.
Initially we have some square tables.
.. _a talk Alex Gaynor gave at PyCon US 2014: https://www.youtube.com/watch?v=7KnfGDajDQw&t=1292
.. code-block:: python
class Table(object):
def __init__(self, size):
self.size = size
def __repr__(self):
return "<Table {self.size!r}>".format(self=self)
We want to be able to serialize these, so we write a *dumper* and a
corresponding *loader* function. We'll also need a *registry* to store these
functions::
from camel import CamelRegistry
my_types = CamelRegistry()
@my_types.dumper(Table, 'table', version=1)
def _dump_table(table):
return dict(
size=table.size,
)
@my_types.loader('table', version=1)
def _load_table(data, version):
return Table(data["size"])
.. note:: This example is intended for Python 3. With Python 2,
``dict(size=...)`` will create a "size" key that's a :py:class:`bytes`,
which will be serialized as ``!!binary``. It will still work, but it'll be
ugly, and won't interop with Python 3. If you're still on Python 2, you
should definitely use dict literals with :py:class:`unicode` keys.
Now we just give this registry to a :py:class:`Camel` object and ask it to dump
for us::
from camel import Camel
table = Table(25)
print(Camel([my_types]).dump(table))
.. code-block:: yaml
!table;1
size: 25
Unlike the simple example given in the talk, we can also dump arbitrary
structures containing Tables with no extra effort::
data = dict(chairs=[], tables=[Table(25), Table(36)])
print(Camel([my_types]).dump(data))
.. code-block:: yaml
chairs: []
tables:
- !table;1
size: 25
- !table;1
size: 36
And load them back in::
print(Camel([my_types]).load("[!table;1 {size: 100}]"))
.. code-block:: python
[<Table 100>]
Versioning
..........
As you can see, all serialized Tables are tagged as ``!table;1``. The
``table`` part is the argument we gave to ``@dumper`` and ``@loader``, and the
``1`` is the version number.
Version numbers mean that when the time comes to change your class, you don't
have anything to worry about. Just write a new loader and dumper with a higher
version number, and fix the old loader to work with the new code::
# Tables can be rectangles now!
class Table(object):
def __init__(self, height, width):
self.height = height
self.width = width
def __repr__(self):
return "<Table {self.height!r}x{self.width!r}>".format(self=self)
@my_types.dumper(Table, 'table', version=2)
def _dump_table_v2(table):
return dict(
height=table.height,
width=table.width,
)
@my_types.loader('table', version=2)
def _load_table_v2(data, version):
return Table(data["height"], data["width"])
@my_types.loader('table', version=1)
def _load_table_v1(data, version):
edge = data["size"] ** 0.5
return Table(edge, edge)
table = Table(7, 10)
print(Camel([my_types]).dump(table))
.. code-block:: yaml
!table;2
height: 7
width: 10
More on versions
----------------
Versions are expected to be positive integers, presumably starting at 1.
Whenever your class changes, you have two options:
1. Fix the dumper and loader to preserve the old format but work with the new
internals.
2. Failing that, write new dumpers and loaders and bump the version.
One of the advantages of Camel is that your serialization code is nothing more
than functions returning Python structures, so it's very easily tested. Even
if you end up with dozens of versions, you can write test cases for each
without ever dealing with YAML at all.
You might be wondering whether there's any point to having more than one
version of a dumper function. By default, only the dumper with the highest
version for a type is used. But it's possible you may want to stay
backwards-compatible with other code — perhaps an older version of your
application or library — and thus retain the ability to write out older
formats. You can do this with :py:meth:`Camel.lock_version`::
@my_types.dumper(Table, 'table', version=1)
def _dump_table_v1(table):
return dict(
# not really, but the best we can manage
size=table.height * table.width,
)
camel = Camel([my_types])
camel.lock_version(Table, 1)
print(camel.dump(Table(5, 7)))
.. code-block:: yaml
!table;1
size: 35
Obviously you might lose some information when round-tripping through an old
format, but sometimes it's necessary until you can fix old code.
Note that version locking only applies to dumping, not to loading. For
loading, there are a couple special versions you can use.
Let's say you delete an old class whose information is no longer useful. While
cleaning up all references to it, you discover it has Camel dumpers and
loaders. What about all your existing data? No problem! Just use a version
of ``all`` and return a dummy object::
class DummyData(object):
def __init__(self, data):
self.data = data
@my_types.loader('deleted-type', version=all)
def _load_deleted_type(data, version):
return DummyData(data)
``all`` overrides *all* other loader versions (hence the name). You might
instead want to use ``any``, which is a fallback for when the version isn't
recognized::
@my_types.loader('table', version=any)
def _load_table(data, version):
if 'size' in data:
# version 1
edge = data['size'] ** 0.5
return Table(edge, edge)
else:
# version 2?
return Table(data['height'], data['width'])
Versions must still be integers; a non-integer version will cause an immediate
parse error.
Going versionless
.................
You might be thinking that the version numbers everywhere are an eyesore, and
your data would be much prettier if it only used ``!table``.
Well, yes, it would. But you'd lose your ability to bump the version, so you'd
have to be *very very sure* that your chosen format can be adapted to any
possible future changes to your class.
If you are, in fact, *very very sure*, then you can use a version of ``None``.
This is treated like an *infinite* version number, so it will always be used
when dumping (unless overridden by a version lock).
Similarly, an unversioned tag will look for a loader with a ``None`` version,
then fall back to ``all`` or ``any``. The order versions are checked for is
thus:
* ``None``, if appropriate
* ``all``
* Numeric version, if appropriate
* ``any``
There are deliberately no examples of unversioned tags here. Designing an
unversioned format requires some care, and a trivial documentation example
can't do it justice.
Supported types
---------------
By default, Camel knows how to load and dump all types in the `YAML type
registry`_ to their Python equivalents, which are as follows.
.. _YAML type registry: http://yaml.org/type/
=============== ========================================
YAML tag Python type
=============== ========================================
``!!binary`` :py:class:`bytes`
``!!bool`` :py:class:`bool`
``!!float`` :py:class:`float`
``!!int`` :py:class:`int` (or :py:class:`long` on Python 2)
``!!map`` :py:class:`dict`
``!!merge`` —
``!!null`` :py:class:`NoneType`
``!!omap`` :py:class:`collections.OrderedDict`
``!!seq`` :py:class:`list` or :py:class:`tuple` (dump only)
``!!set`` :py:class:`set` or :py:class:`frozenset` (dump only)
``!!str`` :py:class:`str` (:py:class:`unicode` on Python 2)
``!!timestamp`` :py:class:`datetime.date` or :py:class:`datetime.datetime` as appropriate
=============== ========================================
.. note:: PyYAML tries to guess whether a bytestring is "really" a string on
Python 2, but Camel does not. Serializing *any* bytestring produces an ugly
base64-encoded ``!!binary`` representation.
This is a **feature**.
.. note:: A dumper function must return a value that can be expressed in YAML
without a tag — that is, any of the above Python types *except*
:py:class:`bytes`, :py:class:`set`/:py:class:`frozenset`, and
:py:class:`datetime.date`/:py:class:`datetime.datetime`. (Of course, if the
value is a container, its contents can be anything and will be serialized
recursively.)
If a dumper returns a :py:class:`collections.OrderedDict`, it will be
serialized like a plain dict, but the order of its keys will be preserved.
The following additional types are loaded by default, but **not dumped**. If
you want to dump these types, you can use the existing ``camel.PYTHON_TYPES``
registry.
====================== =====================================
YAML tag Python type
====================== =====================================
``!!python/complex`` :py:class:`complex`
``!!python/frozenset`` :py:class:`frozenset`
``!!python/namespace`` :py:class:`types.SimpleNamespace` (Python 3.3+)
``!!python/tuple`` :py:class:`tuple`
====================== =====================================
Other design notes
------------------
* Camel will automatically use the C extension if available, and fall back to a
Python implementation otherwise. The PyYAML documentation says it doesn't
have this behavior because there are some slight differences between the
implementations, but fails to explain what they are.
* :py:meth:`Camel.load` is safe by default. There is no calling of arbitrary
functions or execution of arbitrary code just from loading data. There is no
"dangerous" mode. PyYAML's ``!!python/object`` and similar tags are not
supported. (Unless you write your own loaders for them, of course.)
* There is no "OO" interface, where dumpers or loaders can be written as
methods with special names. That approach forces a class to have only a
single representation, and more importantly litters your class with junk
unrelated to the class itself. Consider this a cheap implementation of
traits. You can fairly easily build support for this in your application if
you really *really* want it.
* Yes, you may have to write a lot of boring code like this::
@my_types.dumper(SomeType, 'sometype')
def _dump_sometype(data):
return dict(
foo=data.foo,
bar=data.bar,
baz=data.baz,
...
)
I strongly encourage you *not* to do this automatically using introspection,
which would defeat the point of using Camel. If it's painful, step back and
consider whether you really need to be serializing as much as you are, or
whether your classes need to be so large.
* There's no guarantee that the data you get will actually be in the correct
format for that version. YAML is meant for human beings, after all, and
human beings make mistakes. If you're concerned about this, you could
combine Camel with something like the `Colander`_ library.
.. _Colander: http://docs.pylonsproject.org/projects/colander/en/latest/
Known issues
------------
Camel is a fairly simple wrapper around `PyYAML`_, and inherits many of its
problems. Only YAML 1.1 is supported, not 1.2, so a handful of syntactic edge
cases may not parse correctly. Loading and dumping are certainly slower and
more memory-intensive than pickle or JSON. Unicode handling is slightly
clumsy. Python-specific types use tags starting with ``!!``, which is supposed
for be for YAML's types only.
.. _PyYAML: http://pyyaml.org/
Formatting and comments are not preserved during a round-trip load and dump.
The `ruamel.yaml`_ library is a fork of PyYAML that solves this problem, but it
only works when using the pure-Python implementation, which would hurt Camel's
performance even more. Opinions welcome.
.. _ruamel.yaml: https://pypi.python.org/pypi/ruamel.yaml
PyYAML has several features that aren't exposed in Camel yet: dumpers that work
on subclasses, loaders that work on all tags with a given prefix, and parsers
for plain scalars in custom formats.
================================================
FILE: docs/conf.py
================================================
# -*- coding: utf-8 -*-
#
# Camel documentation build configuration file, created by
# sphinx-quickstart2 on Fri May 8 11:39:04 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys
import os
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('..'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.intersphinx',
]
intersphinx_mapping = {
'python': ('https://docs.python.org/3', None),
}
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'Camel'
copyright = u'2015, Eevee (Lexy Munroe)'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1.2'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'classic'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'Cameldoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'Camel.tex', u'Camel Documentation',
u'Eevee (Alex Munroe)', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'camel', u'Camel Documentation',
[u'Eevee (Lexy Munroe)'], 1)
]
# If true, show URL addresses after external links.
#man_show_urls = False
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'Camel', u'Camel Documentation',
u'Eevee (Lexy Munroe)', 'Camel', 'One line description of project.',
'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
================================================
FILE: docs/index.rst
================================================
.. Camel documentation master file, created by
sphinx-quickstart2 on Fri May 8 11:39:04 2015.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to Camel's documentation!
=================================
Camel is a Python serialization library that forces you to explicitly describe
how to dump or load your types. It's good for you, just like eating your
vegetables.
Contents:
.. toctree::
:maxdepth: 2
camel
yamlref
api
================================================
FILE: docs/make.bat
================================================
@ECHO OFF
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build2
)
set BUILDDIR=_build
set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
set I18NSPHINXOPTS=%SPHINXOPTS% .
if NOT "%PAPER%" == "" (
set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
)
if "%1" == "" goto help
if "%1" == "help" (
:help
echo.Please use `make ^<target^>` where ^<target^> is one of
echo. html to make standalone HTML files
echo. dirhtml to make HTML files named index.html in directories
echo. singlehtml to make a single large HTML file
echo. pickle to make pickle files
echo. json to make JSON files
echo. htmlhelp to make HTML files and a HTML help project
echo. qthelp to make HTML files and a qthelp project
echo. devhelp to make HTML files and a Devhelp project
echo. epub to make an epub
echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
echo. text to make text files
echo. man to make manual pages
echo. texinfo to make Texinfo files
echo. gettext to make PO message catalogs
echo. changes to make an overview over all changed/added/deprecated items
echo. xml to make Docutils-native XML files
echo. pseudoxml to make pseudoxml-XML files for display purposes
echo. linkcheck to check all external links for integrity
echo. doctest to run all doctests embedded in the documentation if enabled
goto end
)
if "%1" == "clean" (
for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
del /q /s %BUILDDIR%\*
goto end
)
%SPHINXBUILD% 2> nul
if errorlevel 9009 (
echo.
echo.The 'sphinx-build2' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build2' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
if "%1" == "html" (
%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/html.
goto end
)
if "%1" == "dirhtml" (
%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
goto end
)
if "%1" == "singlehtml" (
%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
goto end
)
if "%1" == "pickle" (
%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the pickle files.
goto end
)
if "%1" == "json" (
%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the JSON files.
goto end
)
if "%1" == "htmlhelp" (
%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run HTML Help Workshop with the ^
.hhp project file in %BUILDDIR%/htmlhelp.
goto end
)
if "%1" == "qthelp" (
%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run "qcollectiongenerator" with the ^
.qhcp project file in %BUILDDIR%/qthelp, like this:
echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Camel.qhcp
echo.To view the help file:
echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Camel.ghc
goto end
)
if "%1" == "devhelp" (
%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished.
goto end
)
if "%1" == "epub" (
%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The epub file is in %BUILDDIR%/epub.
goto end
)
if "%1" == "latex" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
if errorlevel 1 exit /b 1
echo.
echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "latexpdf" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
cd %BUILDDIR%/latex
make all-pdf
cd %BUILDDIR%/..
echo.
echo.Build finished; the PDF files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "latexpdfja" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
cd %BUILDDIR%/latex
make all-pdf-ja
cd %BUILDDIR%/..
echo.
echo.Build finished; the PDF files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "text" (
%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The text files are in %BUILDDIR%/text.
goto end
)
if "%1" == "man" (
%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The manual pages are in %BUILDDIR%/man.
goto end
)
if "%1" == "texinfo" (
%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
goto end
)
if "%1" == "gettext" (
%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
goto end
)
if "%1" == "changes" (
%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
if errorlevel 1 exit /b 1
echo.
echo.The overview file is in %BUILDDIR%/changes.
goto end
)
if "%1" == "linkcheck" (
%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
if errorlevel 1 exit /b 1
echo.
echo.Link check complete; look for any errors in the above output ^
or in %BUILDDIR%/linkcheck/output.txt.
goto end
)
if "%1" == "doctest" (
%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
if errorlevel 1 exit /b 1
echo.
echo.Testing of doctests in the sources finished, look at the ^
results in %BUILDDIR%/doctest/output.txt.
goto end
)
if "%1" == "xml" (
%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The XML files are in %BUILDDIR%/xml.
goto end
)
if "%1" == "pseudoxml" (
%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
goto end
)
:end
================================================
FILE: docs/yamlref.rst
================================================
Brief YAML reference
====================
There is no official YAML reference guide. The YAML website only offers the
`YAML specification`_, which is a dense and thorny tome clearly aimed at
implementers. I suspect this has greatly hampered YAML's popularity.
.. _YAML specification: http://www.yaml.org/spec/1.2/spec.html
In the hopes of improving this situation, here is a very quick YAML overview
that should describe the language almost entirely. Hopefully it's useful
whether or not you use Camel.
Overall structure and design
----------------------------
As I see it, YAML has two primary goals: to support encoding any arbitrary data
structure; and to be easily read and written by humans. If only the spec
shared that last goal.
Human-readability means that much of YAML's syntax is optional, wherever it
would be unambiguous and easier on a human. The trade-off is more complexity
in parsers and emitters.
Here's an example document, configuration for some hypothetical app:
.. code-block:: yaml
database:
username: admin
password: foobar # TODO get prod passwords out of config
socket: /var/tmp/database.sock
options: {use_utf8: true}
memcached:
host: 10.0.0.99
workers:
- host: 10.0.0.101
port: 2301
- host: 10.0.0.102
port: 2302
YAML often has more than one way to express the same data, leaving a human free
to use whichever is most convenient. More convenient syntax tends to be more
contextual or whitespace-sensitive. In the above document, you can see that
indenting is enough to make a nested mapping. Integers and booleans are
automatically distinguished from unquoted strings, as well.
General syntax
--------------
As of 1.2, YAML is a strict superset of JSON. Any valid JSON can be parsed in
the same structure with a YAML 1.2 parser.
YAML is designed around Unicode, not bytes, and its syntax assumes Unicode
input. There is no syntactic mechanism for giving a character encoding; the
parser is expected to recognize BOMs for UTF-8, UTF-16, and UTF-32, but
otherwise a byte stream is assumed to be UTF-8.
The only vertical whitespace characters are U+000A LINE FEED and U+000D
CARRIAGE RETURN. The only horizontal whitespace characters are U+0009 TAB and
U+0020 SPACE. Other control characters are not allowed anywhere. Otherwise,
anything goes.
YAML operates on *streams*, which can contain multiple distinct structures,
each parsed individually. Each structure is called a *document*.
A document begins with ``---`` and ends with ``...``. Both are optional,
though a ``...`` can only be followed by directives or ``---``. You don't see
multiple documents very often, but it's a very useful feature for sending
intermittent chunks of data over a single network connection. With JSON you'd
usually put each chunk on its own line and delimit with newlines; YAML has
support built in.
Documents may be preceded by *directives*, in which case the ``---`` is
required to indicate the end of the directives. Directives are a ``%``
followed by an identifier and some parameters. (This is how directives are
distinguished from a bare document without ``---``, so the first non-blank
non-comment line of a document can't start with a ``%``.)
There are only two directives at the moment: ``%YAML`` specifies the YAML
version of the document, and ``%TAG`` is used for tag shorthand, described
in :ref:`yamlref-more-tags`. Use of directives is, again, fairly uncommon.
*Comments* may appear anywhere. ``#`` begins a comment, and it runs until the
end of the line. In most cases, comments are whitespace: they don't affect
indentation level, they can appear between any two tokens, and a comment on its
own line is the same as a blank line. The few exceptions are not too
surprising; for example, you can't have a comment between the key and colon in
``key:``.
A YAML document is a graph of values, called *nodes*. See
:ref:`yamlref-kinds`.
Nodes may be prefixed with up to two properties: a *tag* and an *anchor*.
Order doesn't matter, and both are optional. Properties can be given to any
value, regardless of kind or style.
Tags
....
Tags are prefixed with ``!`` and describe the *type* of a node. This allows
for adding new types without having to extend the syntax or mingle type
information with data. Omitting the tag leaves the type to the parser's
discretion; usually that means you'll get lists, dicts, strings, numbers, and
other simple types.
You'll probably only see tags in two forms:
* ``!foo`` is a "local" tag, used for some custom type that's specific to the
document.
* ``!!bar`` is a built-in YAML type from the `YAML tag repository`_. Most of
these are inferred from plain data — ``!!seq`` for sequences, ``!!int`` for
numbers, and so on — but a few don't have dedicated syntax and have to be
given explicitly.
For example, ``!!binary`` is used for representing arbitrary binary data
encoded as base64. So ``!!binary aGVsbG8=`` would parse as the bytestring
``hello``.
.. _YAML tag repository: http://yaml.org/type/
There's much more to tags, most of which is rarely used in practice. See
:ref:`yamlref-more-tags`.
Anchors
.......
The other node property is the *anchor*, which is how YAML can store recursive
data structures. Anchor names are prefixed with ``&`` and can't contain
whitespace, brackets, braces, or commas.
An *alias node* is an anchor name prefixed with ``*``, and indicates that the
node with that anchor name should occur in both places. (Alias nodes can't
have properties themselves; the properties of the anchored node are used.)
For example, you might share configuration::
host1:
&common-host
os: linux
arch: x86_64
host2: *common-host
Or serialize a list that contains itself::
&me [*me]
.. note:: This is **not** a copy. The exact same value is reused.
Anchor names act somewhat like variable assignments: at any point in the
document, the parser only knows about the anchors it's seen so far, and a
second anchor with the same name takes precedence. This means that aliases
cannot refer to anchors that appear later in the document.
Anchor names aren't intended to carry information, which unfortunately means
that most YAML parsers throw them away, and re-serializing a document will get
you anchor names like ``ANCHOR1``.
.. _yamlref-kinds:
Kinds of value
--------------
Values come in one of three *kinds*, which reflect the general "shape" of
the data. Scalars are individual values; sequences are ordered collections;
mappings are unordered associations. Each can be written in either a
whitespace-sensitive *block style* or a more compact and explicit *flow style*.
Scalars
.......
Most values in a YAML document will be *plain scalars*. They're defined by
exclusion: if it's not anything else, it's a plain scalar. Technically, they
can only be flow style, so they're really "plain flow scalar style" scalars.
Plain scalars are the most flexible kind of value, and may resolve to a variety
of types from the `YAML tag repository`_:
* Integers become, well, integers (``!!int``). Leading ``0``, ``0b``, and
``0x`` are recognized as octal, binary, and hexadecimal. ``_`` is allowed,
and ignored. Curiously, ``:`` is allowed and treated as a base 60 delimiter,
so you can write a time as ``1:59`` and it'll be loaded as the number of
seconds, 119.
* Floats become floats (``!!float``). Scientific notation using ``e`` is also
recognized. As with integers, ``_`` is ignored and ``:`` indicates base 60,
though only the last component can have a fractional part. Positive
infinity, negative infinity, and not-a-number are recognized with a leading
dot: ``.inf``, ``-.inf``, and ``.nan``.
* ``true`` and ``false`` become booleans (``!!bool``). ``y``, ``n``, ``yes``, ``no``,
``on``, and ``off`` are allowed as synonyms. Uppercase and title case are
also recognized.
* ``~`` and ``null`` become nulls (``!!null``), which is ``None`` in Python. A
completely empty value also becomes null.
* ISO8601 dates are recognized (``!!timestamp``), with whitespace allowed
between the date and time. The time is also optional, and defaults to
midnight UTC.
* ``=`` is a special value (``!!value``) used as a key in mappings. I've never
seen it actually used, and the thing it does is nonsense in many languages
anyway, so don't worry about it. Just remember you can't use ``=`` as a
plain string.
* ``<<`` is another special value (``!!merge``) used as a key in mappings.
This one is actually kind of useful; it's described below in
:ref:`yamlref-merge-keys`.
.. note:: The YAML spec has a notion of *schemas*, sets of types which are
recognized. The recommended schema is "core", which doesn't actually
require ``!!timestamp`` support. I think the idea is to avoid requiring
support for types that may not exist natively — a Perl YAML parser can't
reasonably handle ``!!timestamp`` out of the box, because Perl has no
built-in timestamp type. So while you could technically run into a parser
that doesn't support floats (the "failsafe" schema only does strings!), it
probably won't come as a surprise.
Otherwise, it's a string. Well. Probably. As part of tag resolution (see
:ref:`yamlref-more-tags`), an application is allowed to parse plain scalars
however it wants; you might add logic that parses ``1..5`` as a range type, or
you might recognize keywords and replace them with special objects. But if
you're doing any of that, you're hopefully aware of it.
Between the above parsing and conflicts with the rest of YAML's syntax, for a
plain scalar to be a string, it must meet these restrictions:
* It must not be ``true``, ``false``, ``yes``, ``no``, ``y``, ``n``, ``on``,
``off``, ``null``, or any of those words in uppercase or title case, which
would all be parsed as booleans or nulls.
* It must not be ``~``, which is null. If it's a mapping key, it must not be
``=`` or ``<<``, which are special key values.
* It must not be something that looks like a number or timestamp. I wouldn't
bet on anything that consists exclusively of digits, dashes, underscores, and
colons.
* The first character must not be any of: ``[`` ``]`` ``{`` ``}`` ``,`` ``#``
``&`` ``*`` ``!`` ``|`` ``>`` ``'`` ``"`` ``%`` ``@`` `````. All of these
are YAML syntax for some other kind of construct.
* If the first character is ``?``, ``:``, or ``-``, the next character must not
be whitespace. Otherwise it'll be parsed as a block mapping or sequence.
* It must not contain `` #`` or ``: ``, which would be parsed as a comment or a
key. A hash not preceded by space or a colon not followed by space is fine.
* If the string is inside a flow collection (i.e., inside ``[...]`` or
``{...}``), it must not contain any of ``[`` ``]`` ``{`` ``}`` ``,``, which
would all be parsed as part of the collection syntax.
* Leading and trailing whitespace are ignored.
* If the string is broken across lines, then the newline and any adjacent
whitespace are collapsed into a single space.
That actually leaves you fairly wide open; the biggest restriction is on the
first character. You can have spaces, you can wrap across lines, you can
include whatever (non-control) Unicode you want.
If you need explicit strings, you have some other options.
Strings
```````
YAML has lots of ways to write explicit strings. Aside from plain scalars,
there are two other *flow scalar styles*.
Single-quoted strings are surrounded by ``'``. Single quotes may be escaped as
``''``, but otherwise no escaping is done at all. You may wrap over multiple
lines, but the newline and any surrounding whitespace becomes a single space.
A line containing only whitespace becomes a newline.
Double-quoted strings are surrounded by ``"``. Backslash escapes are recognized:
============== ======
Sequence Result
============== ======
``\0`` U+0000 NULL
``\a`` U+0007 BELL
``\b`` U+0008 BACKSPACE
``\t`` U+0009 CHARACTER TABULATION
``\n`` U+000A LINE FEED
``\v`` U+000B LINE TABULATION
``\f`` U+000C FORM FEED
``\r`` U+000D CARRIAGE RETURN
``\e`` U+001B ESCAPE
``\"`` U+0022 QUOTATION MARK
``\/`` U+002F SOLIDUS
``\\`` U+005C REVERSE SOLIDUS
``\N`` U+0085 NEXT LINE
``\_`` U+00A0 NO-BREAK SPACE
``\L`` U+2028 LINE SEPARATOR
``\P`` U+2029 PARAGRAPH SEPARATOR
``\xNN`` Unicode character ``NN``
``\uNNNN`` Unicode character ``NNNN``
``\UNNNNNNNN`` Unicode character ``NNNNNNNN``
============== ======
As usual, you may wrap a double-quoted string across multiple lines, but the
newline and any surrounding whitespace becomes a single space. As with
single-quoted strings, a line containing only whitespace becomes a newline.
You can escape spaces and tabs to protect them from being thrown away. You
can also escape a newline to preserve any trailing whitespace on that line, but
throw away the newline and any leading whitespace on the next line.
These rules are weird, so here's a contrived example::
"line \
one
line two\n\
\ \ line three\nline four\n
line five
"
Which becomes::
line one
line two
line three
line four
line five
Right, well, I hope that clears that up.
There are also two *block scalar styles*, both consisting of a header followed by an
indented block. The header is usually just a single character, indicating
which block style to use.
``|`` indicates *literal style*, which preserves all newlines in the indented
block. ``>`` indicates *folded style*, which performs the same line folding as
with quoted strings. Escaped characters are not recognized in either style.
Indentation, the initial newline, and any leading blank lines are always
ignored.
So to represent this string::
This is paragraph one.
This is paragraph two.
You could use either literal style::
|
This is paragraph one.
This is paragraph two.
Or folded style::
>
This is
paragraph one.
This
is paragraph
two.
Obviously folded style is more useful if you have paragraphs with longer lines.
Note that there are two blank lines between paragraphs in folded style; a
single blank line would be parsed as a single newline.
The header has some other features, but I've never seen them used. It consists
of up to three parts, with no intervening whitespace.
1. The character indicating which block style to use.
2. Optionally, the indentation level of the indented block, relative to its
parent. You only need this if the first line of the block starts with a
space, because the space would be interpreted as indentation.
3. Optionally, a "chomping" indicator. The default behavior is to include the
final newline as part of the string, but ignore any subsequent empty lines.
You can use ``-`` here to ignore the final newline as well, or use ``+`` to
preserve all trailing whitespace verbatim.
You can put a comment on the same line as the header, but a comment on the next
line would be interpreted as part of the indented block. You can also put a
tag or an anchor before the header, as with any other node.
Sequences
.........
Sequences are ordered collections, with type ``!!seq``. They're pretty simple.
Flow style is a comma-delimited list in square brackets, just like JSON:
``[one, two, 3]``. A trailing comma is allowed, and whitespace is generally
ignored. The contents must also be written in flow style.
Block style is written like a bulleted list::
- one
- two
- 3
- a plain scalar that's
wrapped across multiple lines
Indentation determines where each element ends, and where the entire sequence
ends.
Other blocks may be nested without intervening newlines::
- - one one
- one two
- - two one
- two two
Mappings
........
Mappings are unordered, er, mappings, with type ``!!map``. The keys must be
unique, but may be of any type. Also, they're unordered.
Did I mention that mappings are **unordered**? The order of the keys in the
document is irrelevant and arbitrary. If you need order, you need a sequence.
Flow style looks unsurprisingly like JSON: ``{x: 1, y: 2}``. Again, a trailing
comma is allowed, and whitespace doesn't matter.
As a special case, inside a sequence, you can write a single-pair mapping
without the braces. So ``[a: b, c: d, e: f]`` is a sequence containing three
mappings. This is allowed in block sequences too, and is used for the ordered
mapping type ``!!omap``.
Block style is actually a little funny. The canonical form is a little
surprising::
? x
: 1
? y
: 2
``?`` introduces a key, and ``:`` introduces a value. You very rarely see this
form, because the ``?`` is optional as long as the key and colon are all on one
line (to avoid ambiguity) and the key is no more than 1024 characters long (to
avoid needing infinite lookahead).
So that's more commonly written like this::
x: 1
y: 2
The explicit ``?`` syntax is more useful for complex keys. For example, it's
the only way to use block styles in the key::
? >
If a train leaves Denver at 5:00 PM traveling at 90 MPH, and another
train leaves New York City at 10:00 PM traveling at 80 MPH, by how many
minutes are you going to miss your connection?
: Depends whether we're on Daylight Saving Time or not.
Other than the syntactic restrictions, an implicit key isn't special in any way
and can also be of any type::
true: false
null: null
up: down
[0, 1]: [1, 0]
It's fairly uncommon to see anything but strings as keys, though, since
languages often don't support it. Python can't have lists and dicts as dict
keys; Perl 5 and JavaScript only support string keys; and so on.
Unlike sequences, you may **not** nest another block inside a block mapping on
the same line. This is invalid::
one: two: buckle my shoe
But this is fine::
- one: 1
two: 2
- three: 3
four: 4
You can also nest a block sequence without indenting::
foods:
- burger
- fries
drinks:
- soda
- iced tea
One slight syntactic wrinkle: in either style, the colon must be followed by
whitespace. ``foo:bar`` is a single string, remember. (For JSON's sake, the
whitespace can be omitted if the colon immediately follows a flow sequence, a
flow mapping, or a quoted string.)
.. _yamlref-merge-keys:
Merge keys
``````````
These are written ``<<`` and have type ``!!merge``. A merge key should have
another mapping (or sequence of mappings) as its value. Each mapping is merged
into the containing mapping, with any existing keys left alone. The actual
``<<`` key is never shown to the application.
This is generally used in conjunction with anchors to share default values::
defaults: &DEFAULTS
use-tls: true
verify-host: true
host1:
<<: *DEFAULTS
hostname: example.com
host2:
<<: *DEFAULTS
hostname: example2.com
host3:
<<: *DEFAULTS
hostname: example3.com
# we have a really, really good reason for doing this, really
verify-host: false
.. _yamlref-more-tags:
More on tags
------------
``!!str`` is actually an illusion.
Tag names are actually URIs, using UTF-8 percent-encoding. YAML suggests using
the ``tag:`` scheme and your domain name to help keep tags globally unique; for
example, the string tag is really ``tag:yaml.org,2002:str``. (Domain names can
change hands over time, hence the inclusion of a year.)
That's quite a mouthful, and wouldn't be recognized as a tag anyway, because
tags have to start with ``!``. So tags are written in shorthand with a prefix,
like ``!foo!bar``. The ``!foo!`` is a *named tag handle* that expands to a
given prefix, kind of like XML namespacing. Named tag handles must be defined
by a ``%TAG`` directive before the document::
%TAG !foo! tag:example.com,2015:app/
A tag of ``!foo!bar`` would then resolve to ``tag:example.com,2015:app/bar``.
I've never seen ``%TAG`` used in practice. Instead, everyone uses the two
special tag handles.
* The *primary tag handle* is ``!``, which by default expands to ``!``. So
``!bar`` just resolves to ``!bar``, a *local tag*, specific to the document
and not expected to be unique.
* The *secondary tag handle* is ``!!``, which by default expands to
``tag:yaml.org,2002:``, the prefix YAML uses for its own built-in types. So
``!!bar`` resolves to ``tag:yaml.org,2002:bar``, and the tag for a string
would more commonly be written as ``!!str``. Defining new tags that use
``!!`` is impolite.
Both special handles can be reassigned with ``%TAG``, just like any other
handle. An important (and confusing) point here is that the **resolved** name
determines whether or not a tag is local; how it's written is irrelevant.
You're free to do this::
%TAG !foo! !foo-types/
Now ``!foo!bar`` is shorthand for ``!foo-types/bar``, which is a local tag.
You can also do the reverse::
%TAG ! tag:example.com,2015:legacy-types/
Which would make ``!bar`` a global tag! This is deliberate, as a quick way to
convert an entire document from local tags to global tags.
You can reassign ``!!``, too. But let's not.
Tags can also be written *verbatim* as ``!<foo>``, in which case ``foo`` is
taken to be the resolved final name of the tag, ignoring ``%TAG`` and any other
resolution mechanism. This is the only way to write a global tag without using
``%TAG``, since tags must start with a ``!``.
Every node has a tag, whether it's given one explicitly or not. Nodes without
explicit tags are given one of two special *non-specific* tags: ``!`` for
quoted and folded scalars; or ``?`` for sequences, mappings, and plain scalars.
The ``?`` tag tells the application to do *tag resolution*. Technically, this
means the application can do any kind of arbitrary inspection to figure out the
type of the node. In practice, it just means that scalars are inspected to see
whether they're booleans, integers, floats, whatever else, or just strings.
The ``!`` tag forces a node to be interpreted as a basic built-in type, based
on its kind: ``!!str``, ``!!seq``, or ``!!map``. You can explicitly give the
``!`` tag to a node if you want, for example writing ``! true`` or ``! 133`` to
force parsing as strings. Or you could use quotes. Just saying.
================================================
FILE: setup.py
================================================
from setuptools import find_packages, setup
from io import open
setup(
name='camel',
version='0.1.2',
description="Python serialization for adults",
long_description=open('README.txt', encoding='utf8').read(),
url="https://github.com/eevee/camel",
author="Eevee (Lexy Munroe)",
author_email="eevee.camel@veekun.com",
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'Intended Audience :: Developers',
'License :: OSI Approved :: ISC License (ISCL)',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.2',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
],
packages=find_packages(),
install_requires=['pyyaml'],
tests_require=['pytest'],
)
gitextract_jx6rvyvg/ ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.txt ├── camel/ │ ├── __init__.py │ └── tests/ │ ├── __init__.py │ ├── test_docs.py │ └── test_general.py ├── docs/ │ ├── Makefile │ ├── api.rst │ ├── camel.rst │ ├── conf.py │ ├── index.rst │ ├── make.bat │ └── yamlref.rst └── setup.py
SYMBOL INDEX (60 symbols across 3 files)
FILE: camel/__init__.py
class CamelDumper (line 48) | class CamelDumper(SafeDumper):
method __init__ (line 52) | def __init__(self, *args, **kwargs):
method represent_binary (line 63) | def represent_binary(self, data):
method add_representer (line 72) | def add_representer(self, data_type, representer):
method add_multi_representer (line 75) | def add_multi_representer(self, data_type, representer):
class CamelLoader (line 79) | class CamelLoader(SafeLoader):
method __init__ (line 83) | def __init__(self, *args, **kwargs):
method add_constructor (line 89) | def add_constructor(self, data_type, constructor):
method add_multi_constructor (line 92) | def add_multi_constructor(self, data_type, constructor):
method add_implicit_resolver (line 95) | def add_implicit_resolver(self, tag, regexp, first):
method add_path_resolver (line 101) | def add_path_resolver(self, *args, **kwargs):
class Camel (line 106) | class Camel(object):
method __init__ (line 109) | def __init__(self, registries=()):
method add_registry (line 117) | def add_registry(self, registry, tag_prefix=None, tag_shorthand=None):
method lock_version (line 123) | def lock_version(self, cls, version):
method make_dumper (line 126) | def make_dumper(self, stream):
method dump (line 142) | def dump(self, data):
method make_loader (line 150) | def make_loader(self, stream):
method load (line 156) | def load(self, data):
method load_first (line 164) | def load_first(self, data):
method load_all (line 169) | def load_all(self, data):
class DuplicateVersion (line 176) | class DuplicateVersion(ValueError):
class CamelRegistry (line 180) | class CamelRegistry(object):
method __init__ (line 183) | def __init__(self, tag_prefix='!', tag_shorthand=None):
method freeze (line 193) | def freeze(self):
method _check_tag (line 198) | def _check_tag(self, tag):
method dumper (line 207) | def dumper(self, cls, tag, version, inherit=False):
method run_representer (line 236) | def run_representer(self, representer, tag, dumper, data):
method inject_dumpers (line 263) | def inject_dumpers(self, dumper, version_locks=None):
method loader (line 289) | def loader(self, tag, version):
method run_constructor (line 304) | def run_constructor(self, constructor, version, *yaml_args):
method inject_loaders (line 322) | def inject_loaders(self, loader):
function _dump_frozenset (line 355) | def _dump_frozenset(data):
function _dump_ordered_dict (line 360) | def _dump_ordered_dict(data):
function _load_ordered_dict (line 368) | def _load_ordered_dict(data, version):
function _dump_tuple (line 384) | def _dump_tuple(data):
function _load_tuple (line 389) | def _load_tuple(data, version):
function _dump_complex (line 394) | def _dump_complex(data):
function _load_complex (line 406) | def _load_complex(data, version):
function _dump_frozenset (line 411) | def _dump_frozenset(data):
function _load_frozenset (line 419) | def _load_frozenset(data, version):
function _dump_simple_namespace (line 425) | def _dump_simple_namespace(data):
function _load_simple_namespace (line 430) | def _load_simple_namespace(data, version):
FILE: camel/tests/test_docs.py
function test_docs_table_v1 (line 6) | def test_docs_table_v1():
function test_docs_table_v2 (line 46) | def test_docs_table_v2():
function test_docs_deleted (line 94) | def test_docs_deleted():
function test_docs_table_any (line 110) | def test_docs_table_any():
FILE: camel/tests/test_general.py
function test_basic_roundtrip (line 34) | def test_basic_roundtrip(value, expected_serialization):
function test_tuple_roundtrip (line 41) | def test_tuple_roundtrip():
function test_frozenset_roundtrip (line 51) | def test_frozenset_roundtrip():
function test_python_roundtrip (line 68) | def test_python_roundtrip(value, expected_serialization):
class DieRoll (line 81) | class DieRoll(tuple):
method __new__ (line 82) | def __new__(cls, a, b):
method __repr__ (line 85) | def __repr__(self):
function dump_dice (line 94) | def dump_dice(data):
function load_dice (line 99) | def load_dice(data, version):
function test_dieroll (line 105) | def test_dieroll():
function dump_dice (line 118) | def dump_dice(data):
function load_dice (line 129) | def load_dice(data, version):
function test_dieroll2 (line 138) | def test_dieroll2():
Condensed preview — 16 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (92K chars).
[
{
"path": ".gitignore",
"chars": 71,
"preview": "*.pyc\n*.pyo\n.cache/\n*.egg-info/\n__pycache__/\nbuild/\ndist/\ndocs/_build/\n"
},
{
"path": "LICENSE",
"chars": 830,
"preview": "This project is licensed under the ISC license, reproduced below.\n\nCopyright (c) 2012, Lexy \"eevee\" Munroe <eevee.camel@"
},
{
"path": "MANIFEST.in",
"chars": 35,
"preview": "include LICENSE\ninclude README.txt\n"
},
{
"path": "README.txt",
"chars": 888,
"preview": "Camel\n=====\n\nCamel is a library that lets you describe how to serialize your objects to\nYAML — and refuses to serialize "
},
{
"path": "camel/__init__.py",
"chars": 15791,
"preview": "# encoding: utf8\n# TODO\n# i kinda need for pokedex purposes:\n# - consider creating an @inherited_dumper? would need a w"
},
{
"path": "camel/tests/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "camel/tests/test_docs.py",
"chars": 3792,
"preview": "\"\"\"Make sure the documentation examples actually, uh, work.\"\"\"\nfrom __future__ import unicode_literals\nimport textwrap\n\n"
},
{
"path": "camel/tests/test_general.py",
"chars": 4276,
"preview": "# encoding: utf8\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_fun"
},
{
"path": "docs/Makefile",
"chars": 6760,
"preview": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD "
},
{
"path": "docs/api.rst",
"chars": 83,
"preview": "API reference\n=============\n\n.. automodule:: camel\n :members:\n :undoc-members:\n"
},
{
"path": "docs/camel.rst",
"chars": 15590,
"preview": "Camel overview\n==============\n\nCamel is intended as a replacement for libraries like :py:mod:`pickle` or\n`PyYAML`_, whic"
},
{
"path": "docs/conf.py",
"chars": 8286,
"preview": "# -*- coding: utf-8 -*-\n#\n# Camel documentation build configuration file, created by\n# sphinx-quickstart2 on Fri May 8 "
},
{
"path": "docs/index.rst",
"chars": 520,
"preview": ".. Camel documentation master file, created by\n sphinx-quickstart2 on Fri May 8 11:39:04 2015.\n You can adapt this "
},
{
"path": "docs/make.bat",
"chars": 6702,
"preview": "@ECHO OFF\r\n\r\nREM Command file for Sphinx documentation\r\n\r\nif \"%SPHINXBUILD%\" == \"\" (\r\n\tset SPHINXBUILD=sphinx-build2\r\n)\r"
},
{
"path": "docs/yamlref.rst",
"chars": 22702,
"preview": "Brief YAML reference\n====================\n\nThere is no official YAML reference guide. The YAML website only offers the\n"
},
{
"path": "setup.py",
"chars": 953,
"preview": "from setuptools import find_packages, setup\nfrom io import open\n\n\nsetup(\n name='camel',\n version='0.1.2',\n desc"
}
]
About this extraction
This page contains the full source code of the eevee/camel GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 16 files (85.2 KB), approximately 23.1k tokens, and a symbol index with 60 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.