Repository: adriank/ObjectPath Branch: master Commit: 8472264c57e8 Files: 30 Total size: 98.2 KB Directory structure: gitextract_4ckekqd1/ ├── .coveragerc ├── .gitignore ├── .travis.yml ├── .vscode/ │ ├── launch.json │ └── settings.json ├── LICENSE ├── MANIFEST.in ├── README.md ├── README.rst ├── README.rst.original ├── VER ├── build.sh ├── objectpath/ │ ├── __init__.py │ ├── core/ │ │ ├── __init__.py │ │ ├── interpreter.py │ │ └── parser.py │ ├── shell.py │ └── utils/ │ ├── __init__.py │ ├── colorify.py │ ├── debugger.py │ ├── json_ext.py │ └── timeutils.py ├── requirements.txt ├── setup.cfg ├── setup.py ├── shell.py ├── testObjectPath.py └── tests/ ├── __init__.py ├── test_ObjectPath.py └── test_utils.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ [run] omit = */tests* */shell.py */debugger.py ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ ================================================ FILE: .travis.yml ================================================ language: python python: - '2.7' - '3.8' - pypy install: - pip install -r requirements.txt - pip install coveralls script: - nosetests - coverage run --source=objectpath setup.py test deploy: provider: pypi user: adriankal password: secure: bWTP43XxjAIwC6PMmfjt4/dCBenky+zF7empMgzfmkUq5jCoQjCJm4IUdYIdjCKLYEIU1DaOTp7+rqmIZf2d8wWvGAc4+k3vinV9k8WzycpBM+YgnW2knQ5eko93H0lpNOIrat4J0wvc51JjHfj4uqib6SCTaXmBS/kRHmiRkx8= on: tags: true all_branches: true repo: adriank/ObjectPath distributions: "sdist bdist_wheel" after_success: - coveralls ================================================ FILE: .vscode/launch.json ================================================ { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { "name": "Python: Current File (Integrated Terminal)", "type": "python", "request": "launch", "program": "/Users/adrian/projects/ObjectPath/shell.py", "args": ["/Users/adrian/projects/nutrient-db/butter.json"], "console": "integratedTerminal" }, { "name": "Python: Attach", "type": "python", "request": "attach", "port": 5678, "host": "localhost" }, { "name": "Python: Module", "type": "python", "request": "launch", "module": "objectpath", "console": "integratedTerminal" }, { "name": "Python: Django", "type": "python", "request": "launch", "program": "${workspaceFolder}/manage.py", "console": "integratedTerminal", "args": [ "runserver", "--noreload", "--nothreading" ], "django": true }, { "name": "Python: Flask", "type": "python", "request": "launch", "module": "flask", "env": { "FLASK_APP": "app.py" }, "args": [ "run", "--no-debugger", "--no-reload" ], "jinja": true }, { "name": "Python: Current File (External Terminal)", "type": "python", "request": "launch", "program": "${file}", "console": "externalTerminal" } ] } ================================================ FILE: .vscode/settings.json ================================================ { "python.linting.pylintEnabled": true } ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2017 Adrian Kalbarczyk Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MANIFEST.in ================================================ include *.rst include VER ================================================ FILE: README.md ================================================ ObjectPath ========== [![Downloads](https://img.shields.io/pypi/dm/objectpath.svg)](https://pypi.python.org/pypi/objectpath/) [![Build Status](https://travis-ci.org/adriank/ObjectPath.svg?branch=master)](https://travis-ci.org/adriank/ObjectPath) [![Code Health](https://landscape.io/github/adriank/ObjectPath/master/landscape.png)](https://landscape.io/github/adriank/ObjectPath/master) [![Coverage Status](https://coveralls.io/repos/adriank/ObjectPath/badge.png?branch=master)](https://coveralls.io/r/adriank/ObjectPath?branch=master) The agile NoSQL query language for semi-structured data ----------------------------------------------- **#Python #NoSQL #Javascript #JSON #nested-array-object** ObjectPath is a query language similar to XPath or JSONPath, but much more powerful thanks to embedded arithmetic calculations, comparison mechanisms and built-in functions. This makes the language more like SQL in terms of expressiveness, but it works over JSON documents rather than relations. ObjectPath can be considered a full-featured expression language. Besides selector mechanism there is also boolean logic, type system and string concatenation available. On top of that, the language implementations (Python at the moment; Javascript is in beta!) are secure and relatively fast. More at [ObjectPath site](https://adriank.github.io/ObjectPath/) ![ObjectPath img](http://adriank.github.io/ObjectPath/img/op-colors.png) ObjectPath makes it easy to find data in big nested JSON documents. It borrows the best parts from E4X, JSONPath, XPath and SQL. ObjectPath is to JSON documents what XPath is to XML. Other examples to ilustrate this kind of relationship are: | Scope | Language | |---|---| | text documents | regular expression | | XML | XPath | | HTML | CSS selectors | | JSON documents | ObjectPath | Documentation ------------- [ObjectPath Reference](https://adriank.github.io/ObjectPath/reference.html) Command line usage ----- `````sh $ sudo pip install objectpath $ objectpath file.json ````` or `````sh $ git clone https://github.com/adriank/ObjectPath.git $ cd ObjectPath $ python shell.py file.json ````` Python usage ---------------- `````sh $ sudo pip install objectpath $ python >>> from objectpath import * >>> tree=Tree({"a":1}) >>> tree.execute("$.a") 1 >>> ````` `````sh $ git clone https://github.com/adriank/ObjectPath.git $ cd ObjectPath $ python >>> from objectpath import * >>> tree=Tree({"a":1}) >>> tree.execute("$.a") 1 >>> ````` Contributing & bugs ------------------- I appreciate all contributions and bugfix requests for ObjectPath, however since I don't code in Python anymore, this library is not maintained as of now. Since I can't fully assure that code contributed by others meets quality standards, I can't accept PRs. If you feel you could maintain this code, ping me. I'd be more than happy to transfer this repo to a dedicated ObjectPath organization on GitHub and give the ownership to someone with more time for this project than me. License ------- **MIT** ================================================ FILE: README.rst ================================================ ObjectPath ========== |Downloads| |Build Status| |Code Health| |Coverage Status| The agile NoSQL query language for semi-structured data ------------------------------------------------------- **#Python #NoSQL #Javascript #JSON #nested-array-object** ObjectPath is a query language similar to XPath or JSONPath, but much more powerful thanks to embedded arithmetic calculations, comparison mechanisms and built-in functions. This makes the language more like SQL in terms of expressiveness, but it works over JSON documents rather than relations. ObjectPath can be considered a full-featured expression language. Besides selector mechanism there is also boolean logic, type system and string concatenation available. On top of that, the language implementations (Python at the moment; Javascript is in beta!) are secure and relatively fast. More at `ObjectPath site `__ .. figure:: http://adriank.github.io/ObjectPath/img/op-colors.png :alt: ObjectPath img ObjectPath img ObjectPath makes it easy to find data in big nested JSON documents. It borrows the best parts from E4X, JSONPath, XPath and SQL. ObjectPath is to JSON documents what XPath is to XML. Other examples to ilustrate this kind of relationship are: ============== ================== Scope Language ============== ================== text documents regular expression XML XPath HTML CSS selectors JSON documents ObjectPath ============== ================== Documentation ------------- `ObjectPath Reference `__ Command line usage ------------------ .. code:: sh $ sudo pip install objectpath $ objectpath file.json or .. code:: sh $ git clone https://github.com/adriank/ObjectPath.git $ cd ObjectPath $ python shell.py file.json Python usage ------------ .. code:: sh $ sudo pip install objectpath $ python >>> from objectpath import * >>> tree=Tree({"a":1}) >>> tree.execute("$.a") 1 >>> .. code:: sh $ git clone https://github.com/adriank/ObjectPath.git $ cd ObjectPath $ python >>> from objectpath import * >>> tree=Tree({"a":1}) >>> tree.execute("$.a") 1 >>> Contributing & bugs ------------------- I appreciate all contributions and bugfix requests for ObjectPath, however since I don’t code in Python any more, this library is not maintained as of now. Since I can’t fully assure that code contributed by others meets quality standards, I can’t accept PRs. If you feel you could maintain this code, ping me. I’d be more than happy to transfer this repo to a dedicated ObjectPath organization on GitHub and give the ownership to someone with more time for this project than me. License ------- **MIT** .. |Downloads| image:: https://img.shields.io/pypi/dm/objectpath.svg :target: https://pypi.python.org/pypi/objectpath/ .. |Build Status| image:: https://travis-ci.org/adriank/ObjectPath.svg?branch=master :target: https://travis-ci.org/adriank/ObjectPath .. |Code Health| image:: https://landscape.io/github/adriank/ObjectPath/master/landscape.png :target: https://landscape.io/github/adriank/ObjectPath/master .. |Coverage Status| image:: https://coveralls.io/repos/adriank/ObjectPath/badge.png?branch=master :target: https://coveralls.io/r/adriank/ObjectPath?branch=master ================================================ FILE: README.rst.original ================================================ ObjectPath ========== |Downloads| |Build Status| |Code Health| |Coverage Status| The agile NoSQL query language for semi-structured data ------------------------------------------------------- **#Python #NoSQL #Javascript #JSON #nested-array-object** ObjectPath is a query language similar to XPath or JSONPath, but much more powerful thanks to embedded arithmetic calculations, comparison mechanisms and built-in functions. This makes the language more like SQL in terms of expressiveness, but it works over JSON documents rather than relations. ObjectPath can be considered a full-featured expression language. Besides selector mechanism there is also boolean logic, type system and string concatenation available. On top of that, the language implementations (Python at the moment; Javascript is in beta!) are secure and relatively fast. More at `ObjectPath site `__ .. figure:: http://adriank.github.io/ObjectPath/img/op-colors.png :alt: ObjectPath img ObjectPath img ObjectPath makes it easy to find data in big nested JSON documents. It borrows the best parts from E4X, JSONPath, XPath and SQL. ObjectPath is to JSON documents what XPath is to XML. Other examples to ilustrate this kind of relationship are: ============== ================== Scope Language ============== ================== text documents regular expression XML XPath HTML CSS selectors JSON documents ObjectPath ============== ================== Documentation ------------- `ObjectPath Reference `__ Command line usage ------------------ .. code:: sh $ sudo pip install objectpath $ objectpath file.json or .. code:: sh $ git clone https://github.com/adriank/ObjectPath.git $ cd ObjectPath $ python shell.py file.json Python usage ------------ .. code:: sh $ sudo pip install objectpath $ python >>> from objectpath import * >>> tree=Tree({"a":1}) >>> tree.execute("$.a") 1 >>> .. code:: sh $ git clone https://github.com/adriank/ObjectPath.git $ cd ObjectPath $ python >>> from objectpath import * >>> tree=Tree({"a":1}) >>> tree.execute("$.a") 1 >>> Contributing & bugs ------------------- I appreciate all contributions and bugfix requests for ObjectPath, however since I don’t code in Python any more, this library is not maintained as of now. Since I can’t fully assure that code contributed by others meets quality standards, I can’t accept PRs. If you feel you could maintain this code, ping me. I’d be more than happy to transfer this repo to a dedicated ObjectPath organization on GitHub and give the ownership to someone with more time for this project than me. License ------- **MIT** .. |Downloads| image:: https://img.shields.io/pypi/dm/objectpath.svg :target: https://pypi.python.org/pypi/objectpath/ .. |Build Status| image:: https://travis-ci.org/adriank/ObjectPath.svg?branch=master :target: https://travis-ci.org/adriank/ObjectPath .. |Code Health| image:: https://landscape.io/github/adriank/ObjectPath/master/landscape.png :target: https://landscape.io/github/adriank/ObjectPath/master .. |Coverage Status| image:: https://coveralls.io/repos/adriank/ObjectPath/badge.png?branch=master :target: https://coveralls.io/r/adriank/ObjectPath?branch=master ================================================ FILE: VER ================================================ 0.6.2 ================================================ FILE: build.sh ================================================ #!/bin/sh pandoc -f markdown -t rst -o README.rst README.md mdTable="aaa Scope aaa Language aaa aaa---aaa---aaa aaa text documents aaa regular expression aaa aaa XML aaa XPath aaa aaa HTML aaa CSS selectors aaa aaa JSON documents aaa ObjectPath aaa" rstTable="============== ================== Scope Language ============== ================== text documents regular expression XML XPath HTML CSS selectors JSON documents ObjectPath ============== ==================" sed -i "s/\\\|/aaa/g" README.rst perl -0777 -i.original -pe "s/$mdTable/$rstTable/g" README.rst python setup.py build python setup.py sdist bdist_wheel upload ================================================ FILE: objectpath/__init__.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- from .core.interpreter import * from .utils import * ================================================ FILE: objectpath/core/__init__.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of ObjectPath released under MIT license. # Copyright (C) 2010-2014 Adrian Kalbarczyk from types import GeneratorType as generator from itertools import chain SELECTOR_OPS = [ "is", ">", "<", "is not", ">=", "<=", "in", "not in", ":", "and", "or", "matches", "fn" ] # it must be list because of further concatenations NUM_TYPES = [int, float] try: NUM_TYPES += [long] except NameError: pass STR_TYPES = [str] try: STR_TYPES += [unicode] except NameError: pass ITER_TYPES = [list, generator, chain] try: ITER_TYPES += [map, filter] except NameError: pass class ProgrammingError(Exception): pass class ExecutionError(Exception): pass PY_TYPES_MAP = { "int": "number", "float": "number", "str": "string", "dict": "object", "list": "array" } ================================================ FILE: objectpath/core/interpreter.py ================================================ #!/usr/bin/env python # This file is part of ObjectPath released under MIT license. # Copyright (C) 2010-2014 Adrian Kalbarczyk import sys, re from .parser import parse from objectpath.core import * import objectpath.utils.colorify as color # pylint: disable=W0614 from objectpath.utils import flatten, filter_dict, timeutils, skip from objectpath.utils.json_ext import py2JSON from objectpath.core import ITER_TYPES, generator, chain from objectpath.utils.debugger import Debugger EPSILON = 0.0000000000000001 #this is used in float comparison EXPR_CACHE = {} RE_TYPE = type(re.compile('')) # setting external modules to 0, thus enabling lazy loading. 0 ensures that Pythonic types are never matched. # this way is efficient because if statement is fast and once loaded these variables are pointing to libraries. ObjectId = generateID = calendar = escape = escapeDict = unescape = unescapeDict = 0 class Tree(Debugger): _REGISTERED_FUNCTIONS = {} @classmethod def register_function(cls, name, func): """ This method is used to add custom functions not catered for by default :param str name: The name by which the function will be referred to in the expression :param callable func: The function :return: """ cls._REGISTERED_FUNCTIONS[name] = func def __init__(self, obj, cfg=None): if not cfg: cfg = {} self.D = cfg.get("debug", False) self.setObjectGetter(cfg.get("object_getter", None)) self.setData(obj) self.current = self.node = None if self.D: super(Tree, self).__init__() def setData(self, obj): if type(obj) in ITER_TYPES + [dict]: self.data = obj def setObjectGetter(self, object_getter_cb): if callable(object_getter_cb): self.object_getter = object_getter_cb else: def default_getter(obj, attr): try: return obj.__getattribute__(attr) except AttributeError: if self.D: self.end(color.op(".") + " returning '%s'", color.bold(obj)) return obj self.object_getter = default_getter def compile(self, expr): if expr in EXPR_CACHE: return EXPR_CACHE[expr] ret = EXPR_CACHE[expr] = parse(expr, self.D) return ret def execute(self, expr): D = self.D if D: self.start("Tree.execute") TYPES = [str, int, float, bool, generator, chain] try: TYPES += [long] except NameError: pass # TODO change to yield? def exe(node): """ node[0] - operator name node[1:] - params """ types = [ str, timeutils.datetime.time, timeutils.datetime.date, timeutils.datetime.datetime ] try: types += [unicode] except: pass if D: self.start("executing node %s", color.bold(self.cleanOutput(node))) type_node = type(node) if node is None or type_node in TYPES: return node elif type_node in types: return node elif type_node is list: return (exe(n) for n in node) elif type_node is dict: ret = {} for i in node.items(): ret[exe(i[0])] = exe(i[1]) return ret op = node[0] if op == "or": if D: self.debug("%s or %s", node[1], node[2]) return exe(node[1]) or exe(node[2]) elif op == "and": if D: self.debug("%s and %s", node[1], node[2]) return exe(node[1]) and exe(node[2]) elif op == "+": if len(node) > 2: fst = exe(node[1]) snd = exe(node[2]) if None in (fst, snd): return fst or snd typefst = type(fst) typesnd = type(snd) if typefst is dict: try: fst.update(snd) except Exception: if type(snd) is not dict: raise ProgrammingError( "Can't add value of type %s to %s" % ( color.bold( PY_TYPES_MAP. get(type(snd).__name__, type(snd).__name__) ), color.bold("object") ) ) return fst if typefst is list and typesnd is list: if D: self.debug("both sides are lists, returning '%s'", fst + snd) return fst + snd if typefst in ITER_TYPES or typesnd in ITER_TYPES: if typefst not in ITER_TYPES: fst = [fst] elif typesnd not in ITER_TYPES: snd = [snd] if D: self.debug("at least one side is a generator and the other is an iterable, returning chain") return chain(fst, snd) if typefst in NUM_TYPES: try: return fst + snd except Exception: return fst + float(snd) if typefst in STR_TYPES or typesnd in STR_TYPES: if D: self.info("doing string comparison '%s' is '%s'", fst, snd) if sys.version_info[0] < 3: if typefst is unicode: fst = fst.encode("utf-8") if typesnd is unicode: snd = snd.encode("utf-8") return str(fst) + str(snd) try: timeType = timeutils.datetime.time if typefst is timeType and typesnd is timeType: return timeutils.addTimes(fst, snd) except Exception: pass if D: self.debug("standard addition, returning '%s'", fst + snd) return fst + snd else: return exe(node[1]) elif op == "-": if len(node) > 2: fst = exe(node[1]) snd = exe(node[2]) try: return fst - snd except Exception: typefst = type(fst) typesnd = type(snd) timeType = timeutils.datetime.time if typefst is timeType and typesnd is timeType: return timeutils.subTimes(fst, snd) else: return -exe(node[1]) elif op == "*": return exe(node[1])*exe(node[2]) elif op == "%": return exe(node[1]) % exe(node[2]) elif op == "/": return exe(node[1])/float(exe(node[2])) elif op == ">": if D: self.debug("%s > %s, %s", node[1], node[2], node[1] > node[2]) return exe(node[1]) > exe(node[2]) elif op == "<": return exe(node[1]) < exe(node[2]) elif op == ">=": return exe(node[1]) >= exe(node[2]) elif op == "<=": return exe(node[1]) <= exe(node[2]) # TODO this algorithm produces 3 for 1<2<3 and should be true # elif op in "<=>=": # fst=exe(node[1]) # snd=exe(node[2]) # if op==">": # return fst > snd and snd or False # elif op=="<": # return fst < snd and snd or False # elif op==">=": # return fst >= snd and snd or False # elif op=="<=": # return fst <= snd and snd or False elif op == "not": fst = exe(node[1]) if D: self.debug("doing not '%s'", fst) return not fst elif op == "in": fst = exe(node[1]) snd = exe(node[2]) if D: self.debug("doing '%s' in '%s'", node[1], node[2]) if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES: return any( x in max(fst, snd, key=len) for x in min(fst, snd, key=len) ) return exe(node[1]) in exe(node[2]) elif op == "not in": fst = exe(node[1]) snd = exe(node[2]) if D: self.debug("doing '%s' not in '%s'", node[1], node[2]) if type(fst) in ITER_TYPES and type(snd) in ITER_TYPES: return not any( x in max(fst, snd, key=len) for x in min(fst, snd, key=len) ) return exe(node[1]) not in exe(node[2]) elif op in ("is", "is not"): if D: self.debug("found operator '%s'", op) # try: fst = exe(node[1]) # except Exception as e: # if D: self.debug("NOT ERROR! Can't execute node[1] '%s', error: '%s'. Falling back to orginal value.",node[1],str(e)) # fst=node[1] # try: snd = exe(node[2]) # except Exception as e: # if D: self.debug("NOT ERROR! Can't execute node[2] '%s', error: '%s'. Falling back to orginal value.",node[2],str(e)) # snd=node[2] if op == "is" and fst == snd: return True # this doesn't work for 3 is not '3' # if op == "is not" and fst != snd: # return True typefst = type(fst) typesnd = type(snd) if D: self.debug("type fst: '%s', type snd: '%s'", typefst, typesnd) if typefst in STR_TYPES: if D: self.info("doing string comparison '\"%s\" is \"%s\"'", fst, snd) ret = str(fst) == str(snd) elif typefst is float or typesnd is float: if D: self.info("doing float comparison '%s is %s'", fst, snd) try: ret = abs(float(fst) - float(snd)) < EPSILON except: ret = False elif typefst is int or typesnd is int: if D: self.info("doing integer comparison '%s is %s'", fst, snd) try: ret = int(fst) == int(snd) except: ret = False elif typefst is list and typesnd is list: if D: self.info("doing array comparison '%s' is '%s'", fst, snd) ret = fst == snd elif typefst is dict and typesnd is dict: if D: self.info("doing object comparison '%s' is '%s'", fst, snd) ret = fst == snd elif fst is None or snd is None: if fst is None and snd is None: # this executes only for "is not" ret = True else: ret = (fst or snd) is None if D: self.info( "doing None comparison %s is %s = %s", color.bold(fst), color.bold(snd), color.bold(not not (fst or snd)) ) else: if D: self.info("can't compare %s and %s. Returning False", self.cleanOutput(fst), self.cleanOutput(snd)) ret = False # else: # try: # global ObjectId # if not ObjectId: # from bson.objectid import ObjectId # if typefst is ObjectId or typesnd is ObjectId: # if D: self.info("doing MongoDB objectID comparison '%s' is '%s'",fst,snd) # ret=str(fst)==str(snd) # else: # if D: self.info("doing standard comparison '%s' is '%s'",fst,snd) # ret=fst is snd # except Exception: # pass if op == "is not": if D: self.info("'is not' found. Returning %s", not ret) return not ret else: if D: self.info("returning %s is %s => %s", color.bold(self.cleanOutput(fst)), color.bold(self.cleanOutput(snd)), color.bold(ret)) return ret elif op == "re": return re.compile(exe(node[1])) elif op == "matches": fst = exe(node[1]) snd = exe(node[2]) if type(fst) not in STR_TYPES+[RE_TYPE]: raise Exception("operator " + color.bold("matches") + " expects regexp on the left. Example: 'a.*d' matches 'abcd'") if type(snd) in ITER_TYPES: for i in snd: if not not re.match(fst, i): return True return False else: # regex matches string return not not re.match(fst, snd) # elif op=="(literal)": # fstLetter=node[1][0] # if fstLetter is "'": # return node[1][1:-1] # elif fstLetter.isdigit: # return int(node[1]) elif op == "(root)": # this is $ return self.data # elif op=="(node)":# this is ! # if D: self.debug("returning node %s",self.node) # return self.node elif op == "(current)": # this is @ if D: self.debug("returning current node: \n %s", color.bold(self.current)) return self.current elif op == "name": return node[1] elif op == ".": fst = node[1] if type(fst) is tuple: fst = exe(fst) typefst = type(fst) if D: self.debug(color.op(".") + " left is '%s'", color.bold(self.cleanOutput(fst))) # try: if node[2][0] == "*": if D: self.end( color.op(".") + " returning '%s'", color.bold(typefst in ITER_TYPES and fst or [fst]) ) return fst # typefst in ITER_TYPES and fst or [fst] # except: # pass snd = exe(node[2]) if D: self.debug(color.op(".") + " right is '%s'", color.bold(snd)) if typefst in ITER_TYPES: if D: self.debug( color.op(".") + " filtering %s by %s", color.bold(self.cleanOutput(fst)), color.bold(snd) ) if type(snd) in ITER_TYPES: return filter_dict(fst, list(snd)) else: # if D: self.debug(list(fst)) return (e[snd] for e in fst if type(e) is dict and snd in e) try: if D: self.end(color.op(".") + " returning '%s'", fst.get(snd)) return fst.get(snd) except Exception: if isinstance(fst, object): return self.object_getter(fst, snd) if D: self.end(color.op(".") + " returning '%s'", color.bold(fst)) return fst elif op == "..": fst = flatten(exe(node[1])) if node[2][0] == "*": if D: self.debug(color.op("..") + " returning '%s'", color.bold(fst)) return fst # reduce objects to selected attributes snd = exe(node[2]) if D: self.debug( color.op("..") + " finding all %s in %s", color.bold(snd), color.bold(self.cleanOutput(fst)) ) if type(snd) in ITER_TYPES: ret = filter_dict(fst, list(snd)) if D: self.debug(color.op("..") + " returning %s", color.bold(ret)) return ret else: ret = chain.from_iterable( type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e) ) # print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e)))) if D: self.debug(color.op("..") + " returning %s", color.bold(self.cleanOutput(ret))) return ret elif op == "[": len_node = len(node) # TODO move it to tree generation phase if len_node == 1: # empty list if D: self.debug("returning an empty list") return [] if len_node == 2: # list - preserved to catch possible event of leaving it as '[' operator if D: self.debug("doing list mapping") return [exe(x) for x in node[1]] if len_node == 3: # selector used [] fst = exe(node[1]) # check against None if not fst: return fst selector = node[2] if D: self.debug( "\n found selector '%s'.\n executing on %s", color.bold(selector), color.bold(fst) ) selectorIsTuple = type(selector) is tuple if selectorIsTuple and selector[0] == "[": nodeList = [] nodeList_append = nodeList.append for i in fst: if D: self.debug("setting self.current to %s", color.bold(i)) self.current = i nodeList_append( exe((selector[0], exe(selector[1]), exe(selector[2]))) ) if D: self.debug( "returning %s objects: %s", color.bold(len(nodeList)), color.bold(nodeList) ) return nodeList if selectorIsTuple and selector[0] == "(current)": if D: self.warning( color.bold("$.*[@]") + " is eqivalent to " + color.bold("$.*") + "!" ) return fst if selectorIsTuple and selector[0] in SELECTOR_OPS: if D: self.debug( "found %s operator in selector, %s", color.bold(selector[0]), color.bold(selector) ) if type(fst) is dict: fst = [fst] # TODO move it to tree building phase if type(selector[1]) is tuple and selector[1][0] == "name": selector = (selector[0], selector[1][1], selector[2]) selector0 = selector[0] selector1 = selector[1] selector2 = selector[2] def exeSelector(fst): for i in fst: if D: self.debug("setting self.current to %s", color.bold(i)) self.debug(" s0: %s\n s1: %s\n s2: %s\n Current: %s", selector0, selector1, selector2, i) self.current = i if selector0 == "fn": yield exe(selector) # elif type(selector1) in STR_TYPES and False: # if D: self.debug("found string %s", type(i)) # try: # if exe((selector0,i[selector1],selector2)): # yield i # if D: self.debug("appended") # if D: self.debug("discarded") # except Exception as e: # if D: self.debug("discarded, Exception: %s",color.bold(e)) else: try: # TODO optimize an event when @ is not used. exe(selector1) can be cached if exe((selector0, exe(selector1), exe(selector2))): yield i if D: self.debug("appended %s", i) elif D: self.debug("discarded") except Exception: if D: self.debug("discarded") # if D and nodeList: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList)) return exeSelector(fst) self.current = fst snd = exe(node[2]) typefst = type(fst) if typefst in [tuple] + ITER_TYPES + STR_TYPES: typesnd = type(snd) # nodes[N] if typesnd in NUM_TYPES or typesnd is str and snd.isdigit(): n = int(snd) if D: self.info( "getting %sth element from '%s'", color.bold(n), color.bold(fst) ) if typefst in (generator, chain): if n > 0: return skip(fst, n) elif n == 0: return next(fst) else: fst = list(fst) else: try: return fst[n] except (IndexError, TypeError): return None # $.*['string']==$.string if type(snd) in STR_TYPES: return exe((".", fst, snd)) else: # $.*[@.string] - bad syntax, but allowed return snd else: try: if D: self.debug("returning %s", color.bold(fst[snd])) return fst[snd] except KeyError: # CHECK - is it ok to do that or should it be ProgrammingError? if D: self.debug("returning an empty list") return [] raise ProgrammingError( "Wrong usage of " + color.bold("[") + " operator" ) elif op == "fn": # Built-in functions fnName = node[1] args = None try: args = [exe(x) for x in node[2:]] except IndexError: if D: self.debug("NOT ERROR: can't map '%s' with '%s'", node[2:], exe) # arithmetic if fnName == "sum": args = args[0] if type(args) in NUM_TYPES: return args return sum((x for x in args if type(x) in NUM_TYPES)) elif fnName == "max": args = args[0] if type(args) in NUM_TYPES: return args return max((x for x in args if type(x) in NUM_TYPES)) elif fnName == "min": args = args[0] if type(args) in NUM_TYPES: return args return min((x for x in args if type(x) in NUM_TYPES)) elif fnName == "avg": args = args[0] if type(args) in NUM_TYPES: return args if type(args) not in ITER_TYPES: raise Exception("Argument for avg() is not an array") else: args = list(args) try: return sum(args)/float(len(args)) except TypeError: args = [x for x in args if type(x) in NUM_TYPES] self.warning("Some items in array were ommited") return sum(args)/float(len(args)) elif fnName == "round": return round(*args) # casting elif fnName == "int": return int(args[0]) elif fnName == "float": return float(args[0]) elif fnName == "str": return str(py2JSON(args[0])) elif fnName in ("list", "array"): try: a = args[0] except IndexError: return [] targs = type(a) if targs is timeutils.datetime.datetime: return timeutils.date2list(a) + timeutils.time2list(a) if targs is timeutils.datetime.date: return timeutils.date2list(a) if targs is timeutils.datetime.time: return timeutils.time2list(a) return list(a) # string elif fnName == "upper": return args[0].upper() elif fnName == "lower": return args[0].lower() elif fnName == "capitalize": return args[0].capitalize() elif fnName == "title": return args[0].title() elif fnName == "split": return args[0].split(*args[1:]) elif fnName == "slice": if args and type(args[1]) not in ITER_TYPES: raise ExecutionError( "Wrong usage of slice(STRING, ARRAY). Second argument is not an array but %s." % color.bold(type(args[1]).__name__) ) try: pos = list(args[1]) if type(pos[0]) in ITER_TYPES: if D: self.debug("run slice() for a list of slicers") return (args[0][x[0]:x[1]] for x in pos) return args[0][pos[0]:pos[1]] except IndexError: if len(args) != 2: raise ProgrammingError( "Wrong usage of slice(STRING, ARRAY). Provided %s argument, should be exactly 2." % len(args) ) elif fnName == "escape": global escape, escapeDict if not escape: from objectpath.utils import escape, escapeDict return escape(args[0], escapeDict) elif fnName == "unescape": global unescape, unescapeDict if not unescape: from objectpath.utils import unescape, unescapeDict return unescape(args[0], unescapeDict) elif fnName == "replace": if sys.version_info[0] < 3 and type(args[0]) is unicode: args[0] = args[0].encode("utf8") return str.replace(args[0], args[1], args[2]) # TODO this should be supported by /regex/ # elif fnName=="REsub": # return re.sub(args[1],args[2],args[0]) elif fnName == "sort": if len(args) > 1: key = args[1] a = {"key": lambda x: x.get(key, 0)} else: a = {} args = args[0] if D: self.debug("doing sort on '%s'", args) try: return sorted(args, **a) except TypeError: return args elif fnName == "reverse": args = args[0] try: args.reverse() return args except TypeError: return args elif fnName == "unique": try: return list(set(args[0])) except TypeError: return args[0] elif fnName == "map": return chain.from_iterable(map(lambda x: exe(("fn", args[0], x)), args[1])) elif fnName in ("count", "len"): args = args[0] if args in (True, False, None): return args if type(args) in ITER_TYPES: return len(list(args)) return len(args) elif fnName == "join": try: joiner = args[1] except Exception: joiner = "" try: return joiner.join(args[0]) except TypeError: try: return joiner.join(map(str, args[0])) except Exception: return args[0] # time elif fnName in ("now", "age", "time", "date", "dateTime"): if fnName == "now": return timeutils.now() if fnName == "date": return timeutils.date(args) if fnName == "time": return timeutils.time(args) if fnName == "dateTime": return timeutils.dateTime(args) # TODO move lang to localize() entirely! if fnName == "age": a = {} if len(args) > 1: a["reference"] = args[1] if len(args) > 2: a["lang"] = args[2] return list(timeutils.age(args[0], **a)) elif fnName == "toMillis": args = args[0] if args.utcoffset() is not None: args = args - args.utcoffset() # pylint: disable=E1103 global calendar if not calendar: import calendar return int( calendar.timegm(args.timetuple())*1000 + args.microsecond/1000 ) elif fnName == "localize": if type(args[0]) is timeutils.datetime.datetime: return timeutils.UTC2local(*args) # polygons elif fnName == "area": def segments(p): p = list(map(lambda x: x[0:2], p)) return zip(p, p[1:] + [p[0]]) return 0.5*abs( sum(x0*y1 - x1*y0 for ((x0, y0), (x1, y1)) in segments(args[0])) ) # misc elif fnName == "keys": try: return list(args[0].keys()) except AttributeError: raise ExecutionError( "Argument is not " + color.bold("object") + " but %s in keys()" % color.bold(type(args[0]).__name__) ) elif fnName == "values": try: return list(args[0].values()) except AttributeError: raise ExecutionError( "Argument is not " + color.bold("object") + " but %s in values()" % color.bold(type(args[0]).__name__) ) elif fnName == "type": ret = type(args[0]) if ret in ITER_TYPES: return "array" if ret is dict: return "object" return ret.__name__ elif fnName in self._REGISTERED_FUNCTIONS: return self._REGISTERED_FUNCTIONS[fnName](*args) else: raise ProgrammingError( "Function " + color.bold(fnName) + " does not exist." ) else: return node D = self.D if type(expr) in STR_TYPES: tree = self.compile(expr) elif type(expr) not in (tuple, list, dict): return expr ret = exe(tree) if D: self.end("Tree.execute with: %s", color.bold(self.cleanOutput(ret))) return ret def __str__(self): return "TreeObject()" def __repr__(self): return self.__str__() ================================================ FILE: objectpath/core/parser.py ================================================ #!/usr/bin/env python # This file is part of ObjectPath released under MIT license. # Copyright (C) 2010-2014 Adrian Kalbarczyk # Code from http://effbot.org/zone/simple-top-down-parsing.htm was used in this file. # Licence of the code is public domain. # Relicenced to MIT by Adrian Kalbarczyk and: # - specialized to work with ObjectPath, # - optimized import sys if sys.version_info[0] >= 3: from io import StringIO else: from cStringIO import StringIO from objectpath.core import SELECTOR_OPS, NUM_TYPES symbol_table = {} token = nextToken = None # TODO optimization ('-',1) -> -1 # TODO optimization operators should be numbers TRUE = ["true", "t"] FALSE = ["false", "f"] NONE = ["none", "null", "n", "nil"] class symbol_base(object): id = None value = None fst = snd = third = None def nud(self): raise SyntaxError("Syntax error (%r)." % self.id) def led(self): raise SyntaxError("Unknown operator (%r)." % self.id) def getTree(self): if self.id == "(name)": val = self.value.lower() if val in TRUE: return True elif val in FALSE: return False elif val in NONE: return None return (self.id[1:-1], self.value) elif self.id == "(number)": return self.value elif self.id == "(literal)": fstLetter = self.value[0] if fstLetter in ["'", "\""]: return self.value[1:-1] # elif fstLetter.isdigit(): # try: # return int(self.value) # except: # return float(self.value) else: if self.value == "True": return True elif self.value == "False": return False elif self.value == "None": return None ret = [self.id] ret_append = ret.append L = (dict, tuple, list) for i in filter(None, [self.fst, self.snd, self.third]): if type(i) is str: ret_append(i) elif type(i) in L: t = [] t_append = t.append if self.id == "{": ret = {} for j in list(self.fst.items()): ret[j[0].getTree()] = j[1].getTree() return ret for j in i: try: t_append(j.getTree()) except Exception: t_append(j) if self.id in ("[", ".", ".."): ret.append(t) else: ret.extend(t) # ret_append(t) # return (self.id,ret[1:]) else: if type(self.fst.value) in NUM_TYPES and self.snd is None: if self.id == "-": return -self.fst.value if self.id == "+": return self.fst.value ret_append(i.getTree()) if self.id == "{": return {} # if self.id == "[" and self.fst == []: # return [] if self.id == "(": # this will produce ("fn","fnName",arg1,arg2,...argN) # try: return tuple(["fn", ret[1][1]] + ret[2:]) # except: # pass return tuple(ret) def __repr__(self): if self.id == "(name)" or self.id == "(literal)": return "(%s:%s)" % (self.id[1:-1], self.value) out = [self.id, self.fst, self.snd, self.third] # out=list(map(str, filter(None, out))) return "(" + " ".join(out) + ")" def symbol(ID, bp=0): try: s = symbol_table[ID] except KeyError: class s(symbol_base): pass s.__name__ = "symbol-" + ID # for debugging s.id = ID s.value = None s.lbp = bp symbol_table[ID] = s else: s.lbp = max(bp, s.lbp) return s # helpers def infix(ID, bp): def led(self, left): self.fst = left self.snd = expression(bp) return self symbol(ID, bp).led = led def infix_r(ID, bp): def led(self, left): self.fst = left self.snd = expression(bp - 1) return self symbol(ID, bp).led = led def prefix(ID, bp): def nud(self): self.fst = expression(bp) return self symbol(ID).nud = nud def advance(ID=None): global token if ID and token.id != ID: raise SyntaxError("Expected %r, got %s" % (ID, token.id)) token = nextToken() def method(s): # decorator assert issubclass(s, symbol_base) def bind(fn): setattr(s, fn.__name__, fn) return bind infix_r("or", 30) infix_r("and", 40) prefix("not", 50) infix("in", 60) infix("not", 60) # not in infix("is", 60) infix("matches", 60) infix("<", 60) infix("<=", 60) infix(">", 60) infix(">=", 60) # infix(" ", 60); infix("!=", 60); infix("==", 60) # infix("&", 90) # infix("<<", 100); infix(">>", 100) infix("+", 110) infix("-", 110) infix("*", 120) infix("/", 120) infix("//", 120) infix("%", 120) prefix("-", 130) prefix("+", 130) #prefix("~", 130) # infix_r("**", 140) symbol(".", 150) symbol("[", 150) symbol("{", 150) symbol("(", 150) # additional behavior symbol("(name)").nud = lambda self: self symbol("(literal)").nud = lambda self: self symbol("(number)").nud = lambda self: self symbol("(end)") symbol(")") # REGEX infix("|", 0) infix("^", 0) infix("?", 0) infix("\\", 0) symbol("@") @method(symbol("@")) def nud(self): # pylint: disable=E0102 self.id = "(current)" return self symbol("!") @method(symbol("!")) def nud(self): # pylint: disable=E0102 self.id = "(node)" return self # RegEx @method(symbol("/")) def nud(self): # pylint: disable=E0102 self.id = "re" regex = [] if token.id != "/": self_fst_append = regex.append while 1: if token.id == "/": break if token.id in ["(name)", "(number)"]: self_fst_append(str(token.value)) else: self_fst_append(token.id) advance() self.fst = "".join(regex).replace("\\", "\\\\") advance("/") return self @method(symbol("(")) def nud(self): # pylint: disable=E0102,W0613 expr = expression() advance(")") return expr symbol(",") @method(symbol(".")) def led(self, left): # pylint: disable=E0102 attr = False if token.id == ".": self.id = ".." advance() if token.id == "@": attr = True advance() if token.id == "(": advance() self.fst = left self.snd = [] if token.id != ")": self_snd_append = self.snd.append while 1: self_snd_append(expression()) if token.id != ",": break advance(",") advance(")") return self if token.id not in ["(name)", "*", "(literal)", "("]: raise SyntaxError("Expected an attribute name.") self.fst = left if attr: token.value = "@" + token.value self.snd = token advance() return self # handling namespaces; e.g $.a.b.c or $ss.a.b.c # default storage is the request namespace symbol("$") @method(symbol("$")) def nud(self): # pylint: disable=E0102 global token # pylint: disable=W0602 self.id = "(root)" if token.id == ".": self.fst = "rs" else: self.fst = token.value advance() return self symbol("]") @method(symbol("[")) def led(self, left): # pylint: disable=E0102 self.fst = left self.snd = expression() advance("]") return self symbol(",") # this is for built-in functions @method(symbol("(")) def led(self, left): # pylint: disable=E0102 # self.id="fn" self.fst = left self.snd = [] if token.id != ")": self_snd_append = self.snd.append while 1: self_snd_append(expression()) if token.id != ",": break advance(",") advance(")") return self symbol(":") symbol("=") # constants def constant(ID): @method(symbol(ID)) def nud(self): # pylint: disable=W0612 self.id = "(literal)" self.value = ID return self constant("None") constant("True") constant("False") # multitoken operators @method(symbol("not")) def led(self, left): # pylint: disable=E0102 if token.id != "in": raise SyntaxError("Invalid syntax") advance() self.id = "not in" self.fst = left self.snd = expression(60) return self @method(symbol("is")) def led(self, left): # pylint: disable=E0102 if token.id == "not": advance() self.id = "is not" self.fst = left self.snd = expression(60) return self symbol("]") @method(symbol("[")) def nud(self): # pylint: disable=E0102 self.fst = [] if token.id != "]": while 1: if token.id == "]": break self.fst.append(expression()) if token.id not in SELECTOR_OPS + [","]: break advance(",") advance("]") return self symbol("}") @method(symbol("{")) def nud(self): # pylint: disable=E0102 self.fst = {} if token.id != "}": while 1: if token.id == "}": break key = expression() advance(":") self.fst[key] = expression() if token.id != ",": break advance(",") advance("}") return self import tokenize as tokenizer type_map = { tokenizer.NUMBER: "(number)", tokenizer.STRING: "(literal)", tokenizer.OP: "(operator)", tokenizer.NAME: "(name)", tokenizer.ERRORTOKEN: "(operator)" #'$' is recognized in python tokenizer as error token! } # python tokenizer def tokenize_python(program): if sys.version_info[0] < 3: tokens = tokenizer.generate_tokens(StringIO(program).next) else: tokens = tokenizer.generate_tokens(StringIO(program).__next__) for t in tokens: # print type_map[t[0]], t[1] try: # change this to output python values in correct type yield type_map[t[0]], t[1] except KeyError: if t[0] in [tokenizer.NL, tokenizer.COMMENT, tokenizer.NEWLINE]: continue if t[0] == tokenizer.ENDMARKER: break else: raise SyntaxError("Syntax error") yield "(end)", "(end)" def tokenize(program): if isinstance(program, list): source = program else: source = tokenize_python(program) for ID, value in source: if ID == "(literal)": symbol = symbol_table[ID] s = symbol() s.value = value elif ID == "(number)": symbol = symbol_table[ID] s = symbol() try: s.value = int(value) except Exception: s.value = float(value) elif value == " ": continue else: # name or operator symbol = symbol_table.get(value) if symbol: s = symbol() elif ID == "(name)": symbol = symbol_table[ID] s = symbol() s.value = value else: raise SyntaxError("Unknown operator '%s', '%s'" % (ID, value)) yield s # parser engine def expression(rbp=0): global token t = token token = nextToken() left = t.nud() while rbp < token.lbp: t = token token = nextToken() left = t.led(left) return left def parse(expr, D=False): if sys.version_info[0] < 3 and type(expr) is unicode: expr = expr.encode("utf8") if type(expr) is not str: return expr expr = expr.strip() global token, nextToken if sys.version_info[0] >= 3: nextToken = tokenize(expr).__next__ else: nextToken = tokenize(expr).next token = nextToken() r = expression().getTree() if D: print("PARSE STAGE") print(r) return r ================================================ FILE: objectpath/shell.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of ObjectPath released under MIT license. # Copyright (C) 2010-2014 Adrian Kalbarczyk import argparse import os import sys try: import readline # this is to prevent various tools from deleting import readline ___x = readline.__doc__ except ImportError: import pyreadline as readline from objectpath import Tree, ITER_TYPES from objectpath.utils.colorify import * # pylint: disable=W0614 from objectpath.utils import json_ext as json from objectpath.utils.json_ext import printJSON try: import pytz except ImportError: if os.isatty(sys.stdin.fileno()) and sys.stdout.isatty(): print("WARNING! pytz is not installed. Localized times are not supported.") def main(): parser = argparse.ArgumentParser(description='Command line options') parser.add_argument( '-u', '--url', dest='URL', help='URL containing JSON document.' ) # parser.add_argument('-xml', dest='xml', help='[EXPERIMENTAL] Expect XML input.',action='store_true') parser.add_argument( '-d', '--debug', dest='debug', help='Debbuging on/off.', action='store_true' ) parser.add_argument( '-p', '--profile', dest='profile', help='Profiling on/off.', action='store_true' ) parser.add_argument( '-e', '--expr', dest='expr', help='Expression/query to execute on file, print on stdout and exit.' ) parser.add_argument('file', metavar='FILE', nargs="?", help='File name') args = parser.parse_args() a = {} expr = args.expr if not expr: print( bold("ObjectPath interactive shell") + "\n" + bold("ctrl+c") + " to exit, documentation at " + const("http://adriank.github.io/ObjectPath") + ".\n" ) if args.debug: a["debug"] = True if args.profile: try: from guppy import hpy except: pass File = args.file # if args.xml: # from utils.xmlextras import xml2tree src = False if args.URL: if sys.version_info[0] >= 3: from urllib.request import Request, build_opener # pylint: disable=E0611 else: from urllib2 import Request, build_opener request = Request(args.URL) opener = build_opener() request.add_header('User-Agent', 'ObjectPath/1.0 +http://objectpath.org/') src = opener.open(request) elif File: src = open(File, "r") if not src: if not expr: print( "JSON document source not specified. Working with an empty object {}." ) tree = Tree({}, a) else: if not expr: sys.stdout.write( "Loading JSON document from " + str(args.URL or File) + "..." ) sys.stdout.flush() # if args.xml: # tree=Tree(json.loads(json.dumps(xml2tree(src))),a) # else: tree = Tree(json.load(src), a) if not expr: print(" " + bold("done") + ".") if expr: if args.profile: import cProfile, pstats, StringIO pr = cProfile.Profile() pr.enable() try: ret = tree.execute(expr) except Exception as e: print(e.__class__.__name__ + ": " + str(e)) exit(1) if type(ret) in ITER_TYPES: ret = list(ret) print(json.dumps(ret)) if args.profile: pr.disable() s = StringIO.StringIO() sortby = 'cumulative' ps = pstats.Stats(pr, stream=s).sort_stats(sortby) ps.print_stats() print(s.getvalue()) return try: while True: limitResult = 5 try: if sys.version >= '3': r = input(">>> ") else: r = raw_input(">>> ") if r.startswith("all"): limitResult = -1 r = tree.execute(r[3:].strip()) else: r = tree.execute(r) # python 3 raises error here - unicode is not a proper type there try: if type(r) is unicode: r = r.encode("utf8") except NameError: pass print(printJSON(r, length=limitResult)) if args.profile: h = hpy() print(h.heap()) except Exception as e: print(e) except KeyboardInterrupt: pass # new line at the end forces command prompt to apear at left print(bold("\nbye!")) if __name__ == "__main__": main() ================================================ FILE: objectpath/utils/__init__.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of ObjectPath released under MIT license. # Copyright (C) 2010-2014 Adrian Kalbarczyk from itertools import islice from xml.sax.saxutils import escape, unescape from objectpath.core import NUM_TYPES, generator, chain, ITER_TYPES escape = escape unescape = unescape unescapeDict = {"'": "'", """: "\""} escapeDict = {"'": "'", "\"": """} # islice=islice is an optimization def skip(iterable, n, islice=islice): try: return next(islice(iterable, n, None)) except StopIteration: return None # raise IndexError("generator index out of range") def filter_dict(iterable, keys): """ filters keys of each element of iterable $.(a,b) returns all objects from array that have at least one of the keys: [1,"aa",{"a":2,"c":3},{"c":3},{"a":1,"b":2}].(a,b) -> [{"a":2},{"a":1,"b":2}] """ if type(keys) is not list: keys = [keys] for i in iterable: try: d = {} for a in keys: try: d[a] = i[a] except KeyError: pass if d != {}: yield d except Exception: pass def flatten(fragment, skip=False): def rec(frg): typefrg = type(frg) if typefrg in ITER_TYPES: for i in frg: for j in rec(i): yield j elif typefrg is dict: yield frg for i in frg.items(): for j in rec(i[1]): yield j g = rec(fragment) if skip: for i in xrange(skip): g.next() for i in g: yield i ================================================ FILE: objectpath/utils/colorify.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- def color(c, s): return '\033[%sm%s\033[0m' % (c, s) def bold(s): return color(1, s) def op(s): return color(32, bold(s)) def const(s): return color(36, bold(s)) def string(s): return color(33, bold(s)) ================================================ FILE: objectpath/utils/debugger.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # move dbgMap outside import inspect from objectpath.core import ITER_TYPES class Debugger(object): dbg = None _debugStr = None dbgfn = None level = 40 CUT_AFTER = 100 CRITICAL = 50 ERROR = 40 WARNING = 30 INFO = START = END = 20 DEBUG = 10 doDebug = False def __init__(self): self._debugStr = [] self.dbgfn = self.consolelog self.dbgMap = { "debug": self.DEBUG, "info": self.INFO, "start": self.START, "end": self.END, "warning": self.WARNING, "error": self.ERROR, "critical": self.CRITICAL } try: self.doDebug = True self.level = self.dbgMap["debug"] self.info("All strings will be cut to %s chatacters." % self.CUT_AFTER) except (KeyError, TypeError): pass def debug(self, *s): if self.dbgfn and self.level <= self.DEBUG: self.dbgfn("DEBUG", s) def info(self, *s): if self.dbgfn and self.level <= self.INFO: self.dbgfn("INFO", s) def start(self, *s): if self.dbgfn and self.level <= self.INFO: self.dbgfn("START", s) def end(self, *s): if self.dbgfn and self.level <= self.INFO: self.dbgfn("END", s) def warning(self, *s): if self.dbgfn and self.level <= self.WARNING: self.dbgfn("WARNING", s) def error(self, *s): if self.dbgfn and self.level <= self.ERROR: self.dbgfn("ERROR", s) def critical(self, *s): if self.dbgfn and self.level <= self.CRITICAL: self.dbgfn("CRITICAL", s) def lineno(self): """Returns the current line number in our program.""" return inspect.currentframe().f_back.f_back.f_back.f_lineno def cleanOutput(self, o): toOutput = o if type(toOutput) in ITER_TYPES: toOutput = list(toOutput) if type(toOutput) is tuple: return tuple(map(lambda x: type(x) in ITER_TYPES and list(x) or x, toOutput)) return toOutput def consolelog(self, lvl, s): def f(x): try: if type(x) is unicode: x = x.encode("utf8") except NameError: pass if self.CUT_AFTER and type(x) is dict: s = [] for i in x.items(): s.append("'%s': %s" % (i[0], repr(i[1])[:self.CUT_AFTER])) if len(s[-1]) > self.CUT_AFTER: s.append("...\033[0m") return "{\n\t" + ",\n\t".join(s) + "\n}" s = str(x).replace("\n", "").replace("\t", "").replace("u'", "'") if self.CUT_AFTER and len(s) > self.CUT_AFTER: return s[:self.CUT_AFTER] + "...\033[0m" else: return x if len(s) > 1: v = tuple(map(f, s[1:])) self._debugStr.append((lvl, s[0] % v)) print( lvl + "@" + str(self.lineno()) + " " + (s[0] % v).replace("u'", "'") ) else: self._debugStr.append((lvl, s[0])) print(lvl + "@" + str(self.lineno()) + " " + f(s[0])) ================================================ FILE: objectpath/utils/json_ext.py ================================================ #!/usr/bin/env python try: import simplejson as json except ImportError: try: import json except ImportError: raise Exception("JSONNotFound") try: import ujson as json_fast except ImportError: json_fast = json from types import GeneratorType as generator from objectpath.core import ITER_TYPES, STR_TYPES, NUM_TYPES from objectpath.utils.colorify import * # pylint: disable=W0614 load = json_fast.load def loads(s): if s.find("u'") != -1: s = s.replace("u'", "'") s = s.replace("'", '"') try: return json_fast.loads(s) # ,object_hook=object_hook) except ValueError as e: raise Exception(str(e) + " " + s) def default(obj): if isinstance(obj, generator): return list(obj) def dumps(s, default=default, indent=None): return json.dumps(s, default=default, indent=indent, separators=(',', ':')) dump = json.dump def py2JSON(o): if o is True: return 'true' if o is False: return 'false' if o is None: return 'null' if type(o) in NUM_TYPES: return o # TODO - check if that is correct if type(o) is tuple: return list(o) elif type(o) in ITER_TYPES + [list, str]: return o try: return str(o) except UnicodeEncodeError: return o.encode("utf8") except Exception: return o LAST_LIST = None def printJSON(o, length=5, depth=5): spaces = 2 def plus(): currIndent[0] += 1 def minus(): currIndent[0] -= 1 def out(s): try: s = str(s) except Exception: pass if not ret: ret.append(s) elif ret[-1][-1] == "\n": ret.append(currIndent[0]*spaces*" " + s) else: ret.append(s) def rec(o): if type(o) in ITER_TYPES: o = list(o) if currDepth[0] >= depth: out("") return out("[") if len(o) > 0: if len(o) > 1: out("\n") plus() for i in o[0:length]: rec(i) out(",\n") if length == -1: rec(o[-1]) out(",\n") if length != -1 and len(o) > length: out("... (" + str(len(o) - length) + " more items)\n") else: ret.pop() if len(o) > 1: out("\n") if len(o) > 1: minus() out("]") elif type(o) is dict: currDepth[0] += 1 if currDepth[0] > depth: out("{...}") return keys = o.keys() out("{") if len(keys) > 0: if len(keys) > 1: plus() out("\n") for k in o.keys(): out(string('"' + str(k) + '"') + ": ") rec(o[k]) out(",\n") ret.pop() if len(keys) > 1: minus() out("\n") out("}") else: if type(o) in NUM_TYPES: out(const(o)) elif o in [None, False, True]: out(const(py2JSON(o))) elif type(o) in STR_TYPES: out(string('"' + o + '"')) else: out(string(o)) currDepth[0] -= 1 currIndent = [0] currDepth = [0] ret = [] rec(o) currIndent[0] = 0 currDepth[0] = 0 return "".join(ret) ================================================ FILE: objectpath/utils/timeutils.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of ObjectPath released under MIT license. # Copyright (C) 2008-2010 Adrian Kalbarczyk import datetime import sys, os try: import pytz TIMEZONE_CACHE = {"UTC": pytz.utc} except ImportError: pass from objectpath.core import STR_TYPES HOURS_IN_DAY = 24 now = datetime.datetime.now def round9_10(n): i = int(n) if n - i > 0.9: return i + 1 return i # TODO its 31 minuta, should be 31 minut - probably done def age(date, reference=None, lang="en"): if reference is None: reference = now() td = reference - date #TimeDelta days = float(td.days) langIsPL = lang == "pl" if days: years = round9_10(days/356) if years: if langIsPL: return (years, years == 1 and "rok" or years < 5 and "lata" or "lat") else: return (years, years == 1 and "year" or "years") months = round9_10(days/30) if months: if langIsPL: return ( months, months == 1 and "miesiąc" or 1 < months < 5 and "miesiące" or "miesięcy" ) else: return (months, months == 1 and "month" or "months") weeks = round9_10(days/7) if weeks: if langIsPL: return ( weeks, weeks == 1 and "tydzień" or weeks % 10 in [0, 1, 5, 6, 7, 8, 9] and "tygodni" or "tygodnie" ) else: return (weeks, weeks == 1 and "week" or "weeks") days = int(days) if langIsPL: return (days, days == 1 and "dzień" or "dni") else: return (days, days == 1 and "day" or "days") seconds = float(td.seconds) if seconds is not None: hours = round9_10(seconds/3600) if hours: if langIsPL: return ( hours, hours == 1 and "godzina" or 1 < hours < 5 and "godziny" or "godzin" ) else: return (hours, hours == 1 and "hour" or "hours") minutes = round9_10(seconds/60) if minutes: if langIsPL: return ( minutes, minutes == 1 and "minuta" or 1 < minutes < 5 and "minuty" or "minut" ) else: return (minutes, minutes == 1 and "minute" or "minutes") seconds = int(seconds) if langIsPL: return ( seconds, seconds == 1 and "sekunda" or 1 < seconds < 5 and "sekundy" or "sekund" ) else: return (seconds, seconds == 1 and "second" or "seconds") # return (0,"seconds") def date(d): if d: d = d[0] t = type(d) if t is datetime.datetime: return datetime.date(d.year, d.month, d.day) if t in (tuple, list): return datetime.date(*d) return datetime.date.today() def date2list(d): return [d.year, d.month, d.day] def time(d): if not d or not d[0]: d = now() else: d = d[0] t = type(d) if t in (tuple, list): return datetime.time(*d) return datetime.time(d.hour, d.minute, d.second, d.microsecond) def time2list(t): return [t.hour, t.minute, t.second, t.microsecond] def addTimes(fst, snd): l1 = time2list(fst) l2 = time2list(snd) t = [l1[0] + l2[0], l1[1] + l2[1], l1[2] + l2[2], l1[3] + l2[3]] t2 = [] one = 0 ms = t[3] if ms >= 1000000: t2.append(ms - 1000000) one = 1 else: t2.append(ms) for i in (t[2], t[1]): i = i + one one = 0 if i >= 60: t2.append(i - 60) one = 1 # elif i==60: # t2.append(0) # one=1 else: t2.append(i) hour = t[0] + one if hour >= HOURS_IN_DAY: t2.append(hour - HOURS_IN_DAY) else: t2.append(hour) return datetime.time(*reversed(t2)) def subTimes(fst, snd): l1 = time2list(fst) l2 = time2list(snd) t = [l1[0] - l2[0], l1[1] - l2[1], l1[2] - l2[2], l1[3] - l2[3]] t2 = [] one = 0 ms = t[3] if ms < 0: t2.append(1000000 + ms) one = 1 else: t2.append(ms) for i in (t[2], t[1]): i = i - one one = 0 if i >= 0: t2.append(i) else: t2.append(60 + i) one = 1 hour = t[0] - one if hour < 0: t2.append(HOURS_IN_DAY + hour) else: t2.append(hour) return datetime.time(*reversed(t2)) def dateTime(arg): """ d may be: - datetime() - [y,m,d,h[,m[,ms]]] - [date(),time()] - [[y,m,d],[h,m,s,ms]] and permutations of above """ l = len(arg) if l == 1: dt = arg[0] typed = type(dt) if typed is datetime.datetime: return dt if typed in (tuple, list) and len(dt) in [5, 6, 7]: return datetime.datetime(*dt) if l == 2: date = time = None typeArg0 = type(arg[0]) typeArg1 = type(arg[1]) if typeArg0 in STR_TYPES: return datetime.datetime.strptime(arg[0], arg[1]) if typeArg0 is datetime.date: d = arg[0] date = [d.year, d.month, d.day] if typeArg0 in (tuple, list): date = arg[0] if typeArg1 is datetime.time: t = arg[1] time = [t.hour, t.minute, t.second, t.microsecond] if typeArg1 in (tuple, list): time = arg[1] return datetime.datetime(*date + time) # dt - dateTime, tzName is e.g. 'Europe/Warsaw' def UTC2local(dt, tzName="UTC"): try: if tzName in TIMEZONE_CACHE: tz = TIMEZONE_CACHE[tzName] else: tz = TIMEZONE_CACHE[tzName] = pytz.timezone(tzName) return TIMEZONE_CACHE["UTC"].localize(dt).astimezone(tz) except Exception: return dt ================================================ FILE: requirements.txt ================================================ pytz ================================================ FILE: setup.cfg ================================================ [bdist_wheel] universal = 1 [yapf] # split_all_comma_separated_values=True no_spaces_around_selected_binary_operators="*,/" indent_width=2 dedent_closing_brackets=True coalesce_brackets=True blank_lines_around_top_level_definition=1 continuation_indent_width=2 ================================================ FILE: setup.py ================================================ import os from setuptools import setup def read(*rnames): return open(os.path.join(os.path.dirname(__file__), *rnames)).read() long_description = (read('README.rst') + '\n' + 'Download\n' '********\n') setup( name='objectpath', version=read('VER').strip(), description='The agile query language for semi-structured data. #JSON', long_description=long_description, url='http://adriank.github.io/ObjectPath', author='Adrian Kalbarczyk', author_email='adrian.kalbarczyk@gmail.com', license='MIT License', packages=['objectpath', 'objectpath.utils', 'objectpath.core'], # package_dir={'': 'objectpath'}, keywords="query, tree, JSON, nested structures", classifiers=[ "Development Status :: 6 - Mature", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Topic :: Software Development :: Libraries :: Python Modules" ], install_requires=[ 'pytz', ], zip_safe=True, entry_points={'console_scripts': ['objectpath = objectpath.shell:main']}, test_suite="tests" ) ================================================ FILE: shell.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- from objectpath import shell shell.main() ================================================ FILE: testObjectPath.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- from tests import doTests # starts all test suites doTests() ================================================ FILE: tests/__init__.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- #unit tests for ACR functions from .test_ObjectPath import op_test from .test_utils import utils_test import unittest def doTests(): print('Started ObjectPath Python implementation testing.\n') unittest.TextTestRunner(verbosity=2).run(op_test) unittest.TextTestRunner(verbosity=2).run(utils_test) ================================================ FILE: tests/test_ObjectPath.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- from objectpath.core.interpreter import * from objectpath.core import ProgrammingError, ExecutionError from random import randint, choice #from bson.objectid import ObjectId import sys, unittest, os sys.setrecursionlimit(20000) object1 = { "__lang__": "en", "test": { "_id": 1, "name": "aaa", "o": { "_id": 2 }, "l": [{ "_id": 3, "aaa": "ddd", "false": 2 }, { "_id": 4 }] } } object2 = { "store": { "book": [{ "id":1, "category": "reference", "author": "Nigel Rees", "title": "Sayings of the Century", "price": 8.95 }, { "category": "fiction", "author": "Evelyn Waugh", "title": "Sword of Honour", "price": 12.99 }, { "category": "fiction", "author": "Herman Melville", "title": "Moby Dick", "isbn": "0-553-21311-3", "price": 8.99 }, { "category": "fiction", "author": "J. R. R. Tolkien", "title": "The Lord of the Rings", "isbn": "0-395-19395-8", "price": 22.99 }], "bicycle": { "color": "red", "price": 19.95 }, "k": [{ "_id": 4 }] } } object3 = { "item_1": { "value": "foo", "x": 5.6, "y": 9 }, "item_2": { "value": "bar", "x": 5.6, "y": 9.891 }, "item_3": { "value": "foobar", "x": 5.6, "y": 9.8 } } object4 = { "root": { "response": { "200": { "value": 5, }, "201": { "value": 4, }, "404": { "value": 0, } } } } tree1 = Tree(object1) tree2 = Tree(object2) tree3 = Tree(object3) tree4 = Tree(object4) def execute_raw(expr): return tree1.execute(expr) TYPES = [generator, chain] if sys.version_info.major > 2: TYPES += [map] TYPES = tuple(TYPES) def execute(expr): r = tree1.execute(expr) if isinstance(r, TYPES): return list(r) else: return r def execute2(expr): r = tree2.execute(expr) if isinstance(r, TYPES): return list(r) else: return r def execute3(expr): r = tree3.execute(expr) if isinstance(r, TYPES): return list(r) else: return r def execute4(expr): r = tree4.execute(expr) if isinstance(r, TYPES): return list(r) else: return r class ObjectPath(unittest.TestCase): def test_simple_types(self): self.assertEqual(execute("null"), None) self.assertEqual(execute("true"), True) self.assertEqual(execute("false"), False) self.assertEqual(execute("''"), "") self.assertEqual(execute('""'), "") self.assertEqual(execute("2"), 2) self.assertEqual(execute("2.0"), 2.0) def test_arrays(self): self.assertEqual(execute("[]"), []) self.assertEqual(list(execute("[1,2,3]")), [1, 2, 3]) self.assertEqual( list(execute("[false,null,true,'',\"\",2,2.0,{}]")), [False, None, True, '', "", 2, 2.0, {}] ) def test_objects(self): self.assertEqual(execute("{}"), {}) self.assertEqual( execute("{a:1,b:false,c:'string'}"), { "a": 1, "b": False, "c": 'string' } ) self.assertEqual( execute("{'a':1,'b':false,'c':'string'}"), { "a": 1, "b": False, "c": 'string' } ) def test_arithm_add(self): self.assertEqual(execute("2+3"), 5) self.assertEqual(execute("2+3+4"), 9) self.assertEqual(execute("++3"), 3) # null is treated as neutral value self.assertEqual(execute("null+3"), 3) self.assertEqual(execute("3+null"), 3) def test_arithm_sub(self): self.assertEqual(execute("-1"), -1) self.assertEqual(execute("2-3"), 2 - 3) self.assertEqual(execute("2.2-3.4"), 2.2 - 3.4) self.assertEqual(execute("-+-3"), 3) self.assertEqual(execute("+-+3"), -3) def test_arithm_mul(self): self.assertEqual(execute("2*3*5*6"), 180) def test_arithm_mod(self): self.assertEqual(execute("2%3"), 2.0 % 3) self.assertEqual(execute("2.0%3"), 2.0 % 3) self.assertEqual(execute("float(2)%3"), float(2) % 3) def test_arithm_div(self): self.assertEqual(execute("2/3"), 2.0/3) self.assertEqual(execute("2.0/3"), 2.0/3) self.assertEqual(execute("float(2)/3"), float(2)/3) def test_arithm_group(self): self.assertEqual(execute("2-3+4+5-7"), 2 - 3 + 4 + 5 - 7) self.assertEqual(execute("33*2/5-2"), 33*2/5.0 - 2) self.assertEqual(execute("33-4*5+2/6"), 33 - 4*5 + 2/6.0) #self.assertEqual(execute("2//3//4//5"), ('//', ('//', ('//', 2, 3), 4), 5)) def test_arithm_parentheses(self): self.assertEqual(execute("+6"), 6) self.assertEqual(execute("2+2*2"), 6) self.assertEqual(execute("2+(2*2)"), 6) self.assertEqual(execute("(2+2)*2"), 8) self.assertEqual(execute("(33-4)*5+2/6"), (33 - 4)*5 + 2/6.0) self.assertEqual(execute("2/3/(4/5)*6"), 2/3.0/(4/5.0)*6) self.assertEqual(execute("((2+4))+6"), ((2 + 4)) + 6) def test_logic_negatives(self): self.assertEqual(execute("not false"), True) self.assertEqual(execute("not null"), True) self.assertEqual(execute("not 0"), True) self.assertEqual(execute("not 0.0"), True) self.assertEqual(execute("not ''"), True) self.assertEqual(execute("not []"), True) self.assertEqual(execute("not {}"), True) def test_logic_not(self): self.assertEqual(execute("not false"), True) self.assertEqual(execute("not not not false"), True) def test_logic_or(self): self.assertEqual(execute("1 or 2"), 1) self.assertEqual(execute("0 or 2"), 2) self.assertEqual(execute("'a' or 0 or 3"), 'a') self.assertEqual( execute("null or false or 0 or 0.0 or '' or [] or {}"), {} ) def test_logic_and(self): self.assertEqual(execute("1 and 2"), 2) self.assertEqual(execute("0 and 2"), 0) self.assertEqual(execute("'a' and false and 3"), False) self.assertEqual( execute("true and 1 and 1.0 and 'foo' and [1] and {a:1}"), {"a": 1} ) def test_comparison_regex(self): self.assertIsInstance(execute("/aaa/"), type(re.compile(""))) self.assertEqual(execute("/.*aaa/ matches 'xxxaaaadddd'"), True) self.assertEqual(execute("'.*aaa' matches 'xxxaaaadddd'"), True) self.assertEqual(execute("'.*aaa' matches ['xxxaaaadddd', 'xxx']"), True) def test_comparison_is(self): self.assertEqual(execute("2 is 2"), True) self.assertEqual(execute("'2' is 2"), True) self.assertEqual(execute("2 is '2'"), True) self.assertEqual(execute("2 is 2.0"), True) self.assertEqual(execute("0.1+0.2 is 0.3"), True) self.assertEqual(execute("[] is []"), True) self.assertEqual(execute("[1] is [1]"), True) self.assertEqual(execute("{} is {}"), True) self.assertEqual(execute("{} is []"), False) self.assertEqual(execute("None is 'aaa'"), False) self.assertEqual(execute("None is None"), True) self.assertEqual(execute("{'aaa':1} is {'aaa':1}"), True) #oid=ObjectId() #self.assertEqual(execute("ObjectID('"+str(oid)+"') is '"+str(oid)+"'"), True) def test_comparison_isnot(self): self.assertEqual(execute("None is not None"), False) self.assertEqual(execute("None is not 'aaa'"), True) self.assertEqual(execute("{} is not []"), True) self.assertEqual(execute("3 is not 6"), True) self.assertEqual(execute("3 is not '3'"), False) self.assertEqual(execute("[] is not [1]"), True) self.assertEqual(execute("[] is not []"), False) self.assertEqual(execute("{'aaa':2} is not {'bbb':2}"), True) self.assertEqual(execute("{} is not {}"), False) def test_membership_in(self): self.assertEqual(execute("4 in [6,4,3]"), True) self.assertEqual(execute("4 in {4:true}"), True) self.assertEqual(execute("[2,3] in [6,4,3]"), True) def test_membership_notin(self): self.assertEqual(execute("4 not in []"), True) self.assertEqual(execute("1 not in {'232':2}"), True) self.assertEqual(execute("[2,5] not in [6,4,3]"), True) def test_complex(self): self.assertEqual(execute("23 is not 56 or 25 is 57"), True) self.assertEqual(execute("2+3/4-6*7>0 or 10 is not 11 and 14"), 14) def test_comparison_lt(self): self.assertEqual(execute("2<3"), True) self.assertEqual(execute("3<3"), False) self.assertEqual(execute("2<=2"), True) self.assertEqual(execute("2<=1"), False) def test_comparison_gt(self): self.assertEqual(execute("5>4"), True) self.assertEqual(execute("5>5"), False) self.assertEqual(execute("5>=5"), True) def test_concatenation(self): self.assertEqual(execute("'a'+'b'+\"c\""), 'abc') self.assertEqual(execute("'5'+5"), '55') self.assertEqual(execute("5+'5'"), 10) self.assertEqual(list(execute("[1,2,4] + [3,5]")), [1, 2, 4, 3, 5]) self.assertEqual( execute('{"a":1,"b":2} + {"a":2,"c":3}'), { "a": 2, "b": 2, "c": 3 } ) self.assertRaises( ProgrammingError, lambda: execute('{"a":1,"b":2} + "sss"') ) def test_builtin_casting(self): self.assertEqual(execute("str('foo')"), 'foo') self.assertEqual(execute("str(1)"), '1') self.assertEqual(execute("str(1.0)"), '1.0') self.assertEqual(execute("str(1 is 1)"), 'true') self.assertEqual(execute("int(1)"), 1) self.assertEqual(execute("int(1.0)"), 1) self.assertEqual(execute("int('1')"), 1) #Python can't handle that #self.assertEqual(execute("int('1.0')"), 1) self.assertEqual(execute("float(1.0)"), 1.0) self.assertEqual(execute("float(1)"), 1.0) self.assertEqual(execute("float('1')"), 1.0) self.assertEqual(execute("float('1.0')"), 1.0) self.assertEqual(execute("array()"), []) self.assertEqual(execute("array([])"), []) self.assertEqual(execute("array('abc')"), ['a', 'b', 'c']) self.assertEqual( execute("array(dateTime([2011,4,8,12,0]))"), [2011, 4, 8, 12, 0, 0, 0] ) self.assertEqual(execute("array(date([2011,4,8]))"), [2011, 4, 8]) self.assertEqual(execute("array(time([12,12,30]))"), [12, 12, 30, 0]) def test_builtin_arithmetic(self): self.assertEqual(execute("sum([1,2,3,4])"), sum([1, 2, 3, 4])) self.assertEqual(execute("sum([2,3,4,'333',[]])"), 9) self.assertEqual(execute("sum(1)"), 1) self.assertEqual(execute("min([1,2,3,4])"), min([1, 2, 3, 4])) self.assertEqual(execute("min([2,3,4,'333',[]])"), 2) self.assertEqual(execute("min(1)"), 1) self.assertEqual(execute("max([1,2,3,4])"), max([1, 2, 3, 4])) self.assertEqual(execute("max([2,3,4,'333',[]])"), 4) self.assertEqual(execute("max(1)"), 1) self.assertEqual(execute("avg([1,2,3,4])"), 2.5) self.assertEqual(execute("avg([1,3,3,1])"), 2.0) self.assertEqual(execute("avg([1.1,1.3,1.3,1.1])"), 1.2000000000000002) self.assertEqual(execute("avg([2,3,4,'333',[]])"), 3) self.assertEqual(execute("avg(1)"), 1) self.assertEqual(execute("round(2/3)"), round(2.0/3)) self.assertEqual(execute("round(2/3,3)"), round(2.0/3, 3)) # edge cases self.assertEqual(execute("avg(1)"), 1) # should ommit 'sss' self.assertEqual(execute("avg([1,'sss',3,3,1])"), 2.0) def test_builtin_string(self): self.assertEqual(execute("replace('foobar','oob','baz')"), 'fbazar') self.assertEqual(execute("""escape('<')"""), "&lt;") self.assertEqual(execute("""escape('<"&>')"""), "<"&>") self.assertEqual(execute("""unescape('<"&>')"""), "<\"&>") self.assertEqual(execute("upper('aaa')"), "AAA") self.assertEqual(execute("lower('AAA')"), "aaa") self.assertEqual(execute("title('AAA aaa')"), "Aaa Aaa") self.assertEqual(execute("capitalize('AAA Aaa')"), "Aaa aaa") self.assertEqual(execute("split('aaa aaa')"), ["aaa", "aaa"]) self.assertEqual(execute("split('aaaxaaa','x')"), ["aaa", "aaa"]) self.assertEqual(execute("join(['aaą','aaę'],'ć')"), "aaąćaaę") self.assertEqual(execute("join(['aaa','aaa'])"), "aaaaaa") self.assertEqual(execute("join(['aaa','aaa',3,55])"), "aaaaaa355") self.assertEqual(execute('slice("Hello world!", [6, 11])'), "world") self.assertEqual(execute('slice("Hello world!", [6, -1])'), "world") self.assertEqual( execute('slice("Hello world!", [[0,5], [6, 11]])'), ["Hello", "world"] ) self.assertRaises(ProgrammingError, lambda: execute('slice()')) self.assertRaises(ExecutionError, lambda: execute('slice("", {})')) self.assertEqual(execute('map(upper, ["a", "b", "c"])'), ["A", "B", "C"]) def test_builtin_arrays(self): self.assertEqual(execute("sort([1,2,3,4]+[2,4])"), [1, 2, 2, 3, 4, 4]) self.assertEqual(execute("sort($.._id)"), [1, 2, 3, 4]) self.assertEqual( execute("sort($..l.*, _id)"), [{ '_id': 3, 'aaa': 'ddd', 'false': 2 }, { '_id': 4 }] ) self.assertEqual(execute("reverse([1,2,3,4]+[2,4])"), [4, 2, 4, 3, 2, 1]) self.assertEqual(execute("reverse(sort($.._id))"), [4, 3, 2, 1]) self.assertEqual(execute("len([1,2,3,4]+[2,4])"), 6) self.assertEqual(execute("unique([1,1,3,3])"), [1, 3]) # edge cases self.assertEqual(execute("len(True)"), True) self.assertEqual(execute("len('aaa')"), 3) def test_builtin_time(self): import datetime self.assertIsInstance(execute("now()"), datetime.datetime) self.assertIsInstance(execute("date()"), datetime.date) self.assertIsInstance(execute("date(now())"), datetime.date) self.assertIsInstance(execute("date([2001,12,30])"), datetime.date) self.assertIsInstance(execute("time()"), datetime.time) self.assertIsInstance(execute("time(now())"), datetime.time) self.assertIsInstance(execute("time([12,23])"), datetime.time) self.assertIsInstance(execute("time([12,23,21,777777])"), datetime.time) self.assertIsInstance(execute("dateTime(now())"), datetime.datetime) self.assertIsInstance( execute("dateTime([2001,12,30,12,23])"), datetime.datetime ) self.assertIsInstance( execute("dateTime([2001,12,30,12,23,21,777777])"), datetime.datetime ) self.assertIsInstance(execute('dateTime("1980-05-11 04:22:33", "%Y-%m-%d %H:%M:%S")'), datetime.datetime) self.assertEqual(str(execute('dateTime("1980-05-11 04:22:33", "%Y-%m-%d %H:%M:%S")')), "1980-05-11 04:22:33") self.assertEqual( execute("toMillis(dateTime([2001,12,30,12,23,21,777777]))"), 1009715001777 ) self.assertIsInstance( execute("dateTime(date(),time())"), datetime.datetime ) self.assertIsInstance( execute("dateTime(date(),[12,23])"), datetime.datetime ) self.assertIsInstance( execute("dateTime(date(),[12,23,21,777777])"), datetime.datetime ) self.assertIsInstance( execute("dateTime([2001,12,30],time())"), datetime.datetime ) self.assertEqual( execute("array(time([12,30])-time([8,00]))"), [4, 30, 0, 0] ) self.assertEqual( execute("array(time([12,12,12,12])-time([8,8,8,8]))"), [4, 4, 4, 4] ) self.assertEqual( execute("array(time([12,12,12,12])-time([1,2,3,4]))"), [11, 10, 9, 8] ) self.assertEqual( execute("array(time([12,00])-time([1,10]))"), [10, 50, 0, 0] ) self.assertEqual( execute("array(time([1,00])-time([1,10]))"), [23, 50, 0, 0] ) self.assertEqual( execute("array(time([0,00])-time([0,0,0,1]))"), [23, 59, 59, 999999] ) self.assertEqual( execute("array(time([0,0])+time([1,1,1,1]))"), [1, 1, 1, 1] ) self.assertEqual( execute("array(time([0,0])+time([1,2,3,4]))"), [1, 2, 3, 4] ) self.assertEqual( execute("array(time([23,59,59,999999])+time([0,0,0,1]))"), [0, 0, 0, 0] ) # age tests self.assertEqual(execute("age(now())"), [0, "seconds"]) self.assertEqual( execute("age(dateTime([2000,1,1,1,1]),dateTime([2001,1,1,1,1]))"), [1, "year"] ) self.assertEqual( execute("age(dateTime([2000,1,1,1,1]),dateTime([2000,2,1,1,1]))"), [1, "month"] ) self.assertEqual( execute("age(dateTime([2000,1,1,1,1]),dateTime([2000,1,2,1,1]))"), [1, "day"] ) self.assertEqual( execute("age(dateTime([2000,1,1,1,1]),dateTime([2000,1,1,2,1]))"), [1, "hour"] ) self.assertEqual( execute("age(dateTime([2000,1,1,1,1]),dateTime([2000,1,1,1,2]))"), [1, "minute"] ) self.assertEqual( execute("age(dateTime([2000,1,1,1,1,1]),dateTime([2000,1,1,1,1,2]))"), [1, "second"] ) self.assertEqual( execute("""array(time([0,0]) - time([0,0,0,999999]))"""), [23, 59, 59, 1] ) self.assertEqual( execute("""array(time([0,0]) + time([0,0,0,999999]))"""), [0, 0, 0, 999999] ) def test_localize(self): pass #these tests are passing on computers with timezone set to UTC - not the case of TravisCI #test of non-DST time #if sys.version < "3": #self.assertEqual(execute("array(localize(dateTime([2000,1,1,10,10,1,0]),'Europe/Warsaw'))"), [2000,1,1,11,10,1,0]) #test of DST time #self.assertEqual(execute("array(localize(dateTime([2000,7,1,10,10,1,0]),'Europe/Warsaw'))"), [2000,7,1,12,10,1,0]) def test_builtin_type(self): self.assertEqual(execute("type([1,2,3,4]+[2,4])"), "array") self.assertEqual(execute("type({})"), "object") self.assertEqual(execute("type('')"), "str") def test_selector_with_empty_result(self): self.assertEqual(execute("$.missing is None"), True) self.assertEqual(execute("$.missing is not None"), False) def test_misc(self): self.assertEqual(execute(2), 2) self.assertEqual(execute('{"@aaa":1}.@aaa'), 1) self.assertEqual(execute('$ss.a'), None) self.assertEqual(execute("$..*[10]"), None) self.assertEqual(sorted(execute("keys({'a':1,'b':2})")), ['a', 'b']) self.assertRaises(ExecutionError, lambda: execute('keys([])')) self.assertRaises(ProgrammingError, lambda: execute('blah([])')) def test_optimizations(self): self.assertEqual(execute("$.*[@]"), execute("$.*")) self.assertIsInstance(execute_raw("$..*"), generator) self.assertIsInstance(execute_raw("$..* + $..*"), chain) self.assertIsInstance(execute_raw("$..* + 2"), chain) self.assertIsInstance(execute_raw("2 + $..*"), chain) self.assertEqual(execute("$.._id[0]"), 1) self.assertEqual(execute("sort($.._id + $.._id)[2]"), 2) self.assertIsInstance(execute("$.._id[2]"), int) self.assertEqual( execute2("$.store.book.(price)[0].price"), execute2("$.store.book[0].price") ) class ObjectPath_Paths(unittest.TestCase): def assertItemsEqual(self, a, b, m=None): try: return self.assertCountEqual(a, b, m) except: pass return unittest.TestCase.assertItemsEqual(self, a, b, m) def test_simple_paths(self): self.assertEqual(execute("$.*"), object1) self.assertEqual(execute("$.a.b.c"), None) self.assertEqual(execute("$.a.b.c[0]"), None) self.assertEqual(execute("$.__lang__"), "en") self.assertEqual(execute("$.test.o._id"), 2) self.assertEqual(execute("$.test.l._id"), [3, 4]) self.assertEqual(execute("$.*[test].o._id"), 2) self.assertEqual(execute("$.*['test'].o._id"), 2) self.assertEqual( execute('[1,"aa",{"a":2,"c":3},{"c":3},{"a":1,"b":2}].(a,b)'), [{ "a": 2 }, { "a": 1, "b": 2 }] ) self.assertEqual( execute2("$.store.book.(price,title)[0]"), { "price": 8.95, "title": "Sayings of the Century" } ) self.assertEqual(len(execute2("$..*['Lord' in @.title]")), 1) self.assertEqual( execute2("$..book.(price,title)"), [{ 'price': 8.95, 'title': 'Sayings of the Century' }, { 'price': 12.99, 'title': 'Sword of Honour' }, { 'price': 8.99, 'title': 'Moby Dick' }, { 'price': 22.99, 'title': 'The Lord of the Rings' }] ) self.assertEqual( execute2("sort($..(price,title),'price')"), [{ 'price': 8.95, 'title': 'Sayings of the Century' }, { 'price': 8.99, 'title': 'Moby Dick' }, { 'price': 12.99, 'title': 'Sword of Honour' }, { 'price': 19.95 }, { 'price': 22.99, 'title': 'The Lord of the Rings' }] ) self.assertIsInstance(execute("now().year"), int) def test_complex_paths(self): self.assertEqual(sorted(execute("$.._id")), [1, 2, 3, 4]) self.assertEqual(execute("$..l"), object1["test"]["l"]) self.assertEqual(execute("$..l.._id"), [3, 4]) self.assertEqual(execute2("$.store.*"), object2["store"]) self.assertEqual( execute2("$.store.book.author"), ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien'] ) #print() #print(execute2("$.store.book.(author,aaa)")) self.assertEqual( execute2("$.store.book.(author,aaa)"), [{ "author": "Nigel Rees" }, { "author": "Evelyn Waugh" }, { "author": "Herman Melville" }, { "author": "J. R. R. Tolkien" }] ) self.assertEqual( execute2("$.store.book.(author,price)"), [{ 'price': 8.95, 'author': 'Nigel Rees' }, { 'price': 12.99, 'author': 'Evelyn Waugh' }, { 'price': 8.99, 'author': 'Herman Melville' }, { 'price': 22.99, 'author': 'J. R. R. Tolkien' }] ) self.assertEqual( execute2("$.store.book.*[author]"), ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien'] ) self.assertEqual( execute2("$.store.book.*['author']"), ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien'] ) self.assertEqual(execute2("$.store.book"), object2["store"]["book"]) self.assertEqual( list(execute2("$..author")), ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien'] ) def test_selectors(self): self.assertEqual( execute2("$.store.book[@.id is not null]"), [{ 'category': 'reference', 'price': 8.95, 'title': 'Sayings of the Century', 'id': 1, 'author': 'Nigel Rees' }] ) self.assertEqual(len(execute2("$.store.book[@.id is null]")), 3) self.assertEqual(len(execute("$..*[@._id>2]")), 2) self.assertEqual(execute("$..*[3 in @.l._id]")[0], object1['test']) self.assertEqual(execute2("$.store..*[4 in @.k._id]")[0], object2['store']) self.assertEqual(execute("$..*[@._id>1 and @._id<3][0]"), {'_id': 2}) # very bad syntax!!! self.assertEqual( sorted(execute2("$.store.book[@.price]")), sorted([8.95, 12.99, 8.99, 22.99]) ) self.assertEqual( execute3("$..*[@.x is 5.6 and @.y is 9.891].value"), ['bar'] ) def test_object_list(self): self.assertItemsEqual(execute3('values($.*).value'), ['foo', 'bar', 'foobar']) self.assertItemsEqual(execute3('keys($.*)'), ['item_1', 'item_2', 'item_3']) self.assertItemsEqual( execute4('map(values, $..root..response).value'), [5, 4, 0] ) #testcase2=unittest.FunctionTestCase(test_efficiency(2)) testcase1 = unittest.TestLoader().loadTestsFromTestCase(ObjectPath) testcase2 = unittest.TestLoader().loadTestsFromTestCase(ObjectPath_Paths) op_test = unittest.TestSuite([testcase1, testcase2]) #utils_interpreter=unittest.TestSuite([testcase2]) ================================================ FILE: tests/test_utils.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- from objectpath.utils import * from objectpath.utils.json_ext import * import sys, unittest, os sys.setrecursionlimit(20000) class Utils_test(unittest.TestCase): def test_Utils_JSON_compat(self): self.assertEqual(loads("['ddd']"), ['ddd']) if sys.version_info.major < 3: self.assertEqual(loads("[u'ddd']"), ['ddd']) self.assertRaises(Exception, lambda: loads(['ddd}'])) self.assertEqual(dumps(['ddd']), '["ddd"]') self.assertEqual(py2JSON(False), 'false') self.assertEqual(py2JSON(None), 'null') self.assertEqual(py2JSON((2, 3, 4)), [2, 3, 4]) if sys.version_info.major < 3: self.assertEqual(py2JSON(unicode('')), '') self.assertEqual(py2JSON(2), 2) self.assertEqual( printJSON([1, 2, 3, 4, 5, 6]), "[\n \x1b[36m\x1b[1m1\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m2\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m3\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m4\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m5\x1b[0m\x1b[0m,\n ... (1 more items)\n]" ) self.assertEqual( printJSON([{}, 1]), '[\n {},\n \x1b[36m\x1b[1m1\x1b[0m\x1b[0m\n]' ) self.assertEqual( printJSON({ "aaa": 1 }), '{\x1b[33m\x1b[1m"aaa"\x1b[0m\x1b[0m: \x1b[36m\x1b[1m1\x1b[0m\x1b[0m}' ) self.assertEqual( printJSON({ "a": [1, 2, 3] }), '{\x1b[33m\x1b[1m"a"\x1b[0m\x1b[0m: [\n \x1b[36m\x1b[1m1\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m2\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m3\x1b[0m\x1b[0m\n]}' ) self.assertEqual( printJSON([[1], { "aa": 2 }]), '[\n [\x1b[36m\x1b[1m1\x1b[0m\x1b[0m],\n {\x1b[33m\x1b[1m"aa"\x1b[0m\x1b[0m: \x1b[36m\x1b[1m2\x1b[0m\x1b[0m}\n]' ) self.assertEqual( printJSON({ "aaa": { "bbb": { "ccc": { "ddd": [1, 2, 3, 4, 5] } } } }), '{\x1b[33m\x1b[1m"aaa"\x1b[0m\x1b[0m: {\x1b[33m\x1b[1m"bbb"\x1b[0m\x1b[0m: {\x1b[33m\x1b[1m"ccc"\x1b[0m\x1b[0m: {\x1b[33m\x1b[1m"ddd"\x1b[0m\x1b[0m: [\n \x1b[36m\x1b[1m1\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m2\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m3\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m4\x1b[0m\x1b[0m,\n \x1b[36m\x1b[1m5\x1b[0m\x1b[0m\n]}}}}' ) if str(sys.version_info.major) + str(sys.version_info.minor) < '33': self.assertEqual( printJSON({ "aaa": { "bbb": { "ccc": { "ddd": { "eee": [1, 2, 3, 4, 5], "ddd": {} } } } } }), '{\x1b[33m\x1b[1m"aaa"\x1b[0m\x1b[0m: {\x1b[33m\x1b[1m"bbb"\x1b[0m\x1b[0m: {\x1b[33m\x1b[1m"ccc"\x1b[0m\x1b[0m: {\x1b[33m\x1b[1m"ddd"\x1b[0m\x1b[0m: {\n \x1b[33m\x1b[1m"eee"\x1b[0m\x1b[0m: ,\n \x1b[33m\x1b[1m"ddd"\x1b[0m\x1b[0m: {...}\n}}}}}' ) testcase1 = unittest.TestLoader().loadTestsFromTestCase(Utils_test) utils_test = unittest.TestSuite([testcase1])