Repository: Blacksmoke16/oq
Branch: master
Commit: 5d2f04797a49
Files: 32
Total size: 103.1 KB
Directory structure:
gitextract_301sijum/
├── .editorconfig
├── .github/
│ ├── dependabot.yml
│ └── workflows/
│ ├── ci.yml
│ └── deployment.yml
├── .gitignore
├── LICENSE
├── README.md
├── shard.yml
├── snap/
│ └── snapcraft.yaml
├── spec/
│ ├── assets/
│ │ ├── data1.json
│ │ ├── data1.yml
│ │ ├── data2.json
│ │ ├── data2.yml
│ │ ├── raw.json
│ │ ├── stream-data.json
│ │ ├── stream-filter
│ │ ├── test.jq
│ │ └── test_filter
│ ├── converters/
│ │ ├── simple_yaml_spec.cr
│ │ ├── xml_spec.cr
│ │ └── yaml_spec.cr
│ ├── format_spec.cr
│ ├── oq_spec.cr
│ ├── processor_spec.cr
│ └── spec_helper.cr
└── src/
├── converters/
│ ├── json.cr
│ ├── processor_aware.cr
│ ├── simple_yaml.cr
│ ├── xml.cr
│ └── yaml.cr
├── oq.cr
└── oq_cli.cr
================================================
FILE CONTENTS
================================================
================================================
FILE: .editorconfig
================================================
root = true
[*.cr]
charset = utf-8
end_of_line = lf
insert_final_newline = true
indent_style = space
indent_size = 2
trim_trailing_whitespace = true
================================================
FILE: .github/dependabot.yml
================================================
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
labels:
- "kind:infrastructure"
- "kind:enhancement"
================================================
FILE: .github/workflows/ci.yml
================================================
name: CI
on:
pull_request:
branches:
- 'master'
schedule:
- cron: '0 21 * * *'
jobs:
check_format:
runs-on: ubuntu-latest
container:
image: crystallang/crystal:latest-alpine
steps:
- uses: actions/checkout@v6
- name: Format
run: crystal tool format --check
coding_standards:
runs-on: ubuntu-latest
container:
image: crystallang/crystal:latest
steps:
- uses: actions/checkout@v6
- name: Install Dependencies
run: shards install
- name: Ameba
run: ./bin/ameba
test:
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
crystal:
- latest
- nightly
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
- name: Install Crystal
uses: crystal-lang/install-crystal@v1
with:
crystal: ${{ matrix.crystal }}
- name: Build
run: shards build --production
- name: Specs
run: crystal spec --order=random --error-on-warnings
================================================
FILE: .github/workflows/deployment.yml
================================================
name: Deployment
on:
release:
types:
- created
jobs:
dist_linux:
runs-on: ubuntu-latest
container:
image: crystallang/crystal:latest-alpine
steps:
- uses: actions/checkout@v6
- name: Update Libs
run: apk add --update --upgrade --no-cache --force-overwrite libxml2-dev yaml-dev yaml-static
- name: Build
run: shards build --production --release --static --no-debug --link-flags="-s -Wl,-z,relro,-z,now"
- name: Upload
uses: actions/upload-release-asset@v1.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ github.event.release.upload_url }}
asset_path: ./bin/oq
asset_name: oq-${{ github.event.release.tag_name }}-linux-x86_64
asset_content_type: binary/octet-stream
dist_snap:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Build Snap
uses: snapcore/action-build@v1
id: build
- name: Publish Snap
uses: snapcore/action-publish@v1
env:
SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
with:
snap: ${{ steps.build.outputs.snap }}
release: stable
dist_homebrew:
runs-on: macos-latest
steps:
- run: git config --global user.email "george@dietrich.app"
- run: git config --global user.name "George Dietrich"
- name: Bump Formula
uses: Homebrew/actions/bump-formulae@b5d9170bc1edf1103e40226592b5842b783dd1e0
with:
token: ${{ secrets.HOMEBREW_GITHUB_API_TOKEN }}
formulae: oq
deploy_docs:
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
permissions:
contents: read
pages: write
id-token: write
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Install Crystal
uses: crystal-lang/install-crystal@v1
- name: Build
run: crystal docs
- name: Setup Pages
uses: actions/configure-pages@v6
- name: Upload artifact
uses: actions/upload-pages-artifact@v4
with:
path: 'docs/'
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v5
================================================
FILE: .gitignore
================================================
*.dwarf
*.snap
/.shards/
/bin/
/docs/
/lib/
# Libraries don't need dependency lock
# Dependencies will be locked in application that uses them
/shard.lock
================================================
FILE: LICENSE
================================================
The MIT License (MIT)
Copyright (c) 2021 George Dietrich
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================
FILE: README.md
================================================
# oq
[](https://crystal-lang.org/)
[](https://github.com/blacksmoke16/oq/actions?query=workflow%3ACI)
[](https://github.com/blacksmoke16/oq/releases)
[](https://snapcraft.io/oq)
[](https://aur.archlinux.org/packages/oq/)
[](https://aur.archlinux.org/packages/oq-bin/)
A performant, portable [jq](https://github.com/stedolan/jq/) wrapper that facilitates the consumption and output of formats other than JSON; using `jq` filters to transform the data.
* Compiles to a single binary for easy portability.
* Performant, similar performance with JSON data compared to `jq`. Slightly longer execution time when going to/from a non-JSON format.
* Supports various other input/output [formats](https://blacksmoke16.github.io/oq/OQ/Format.html), such as `XML` and `YAML`.
* Can be used as a dependency within other Crystal projects.
## Installation
### Linux
A statically linked binary for Linux `x86_64` as available on the [Releases](https://github.com/Blacksmoke16/oq/releases) tab. Additionally it can also be installed via various package managers.
#### Snapcraft
For more on installing & using `snap` with your Linux distribution, see the [official documentation](https://docs.snapcraft.io/installing-snapd).
```sh
sudo snap install oq
```
#### Arch Linux
Using [yay](https://github.com/Jguer/yay):
```sh
yay -S oq
```
A pre-compiled version is also available:
```sh
yay -S oq-bin
```
### macOS
```sh
brew install oq
```
### From Source
If building from source, `jq` will need to be installed separately. Installation instructions can be found in the [official documentation](https://stedolan.github.io/jq/).
Requires Crystal to be installed, see the [installation documentation](https://crystal-lang.org/install).
```sh
git clone https://github.com/Blacksmoke16/oq.git
cd oq/
shards build --production --release
```
The built binary will be available as `./bin/oq`. This can be relocated elsewhere on your machine; be sure it is in your `PATH` to access it as `oq`.
### Docker
`oq` can easily be included into a Docker image by fetching the static binary from Github for the version of `oq` that you want.
```dockerfile
# Set an arg to store the oq version that should be installed.
ARG OQ_VERSION=1.3.5
# Grab the binary from the latest Github release and make it executable; placing it within /usr/local/bin. Can also put it elsewhere if you so desire.
RUN wget https://github.com/Blacksmoke16/oq/releases/download/v${OQ_VERSION}/oq-v${OQ_VERSION}-linux-x86_64 -O /usr/local/bin/oq && chmod +x /usr/local/bin/oq
# Or using curl (needs to follow Github's redirect):
RUN curl -L -o /usr/local/bin/oq https://github.com/Blacksmoke16/oq/releases/download/v${OQ_VERSION}/oq-v${OQ_VERSION}-linux-x86_64 && chmod +x /usr/local/bin/oq
# Also be sure to install jq if it is not already!
```
### Existing Crystal Project
Add the following to your `shard.yml` and run `shards install`.
```yaml
dependencies:
oq:
github: blacksmoke16/oq
version: ~> 1.3.0
```
## Usage
### CLI
Use the `oq` binary, with a few optional custom arguments, see `oq --help`. All other arguments get passed to `jq`. See [jq manual](https://stedolan.github.io/jq/manual/) for details.
### Library
Checkout the [API Documentation](https://blacksmoke16.github.io/oq/OQ/Processor.html) for using `oq` within an existing Crystal project.
### Examples
Consume JSON and output XML
```sh
$ echo '{"name": "Jim"}' | oq -o xml .
Jim
```
Consume YAML from a file and output XML
data.yaml
```yaml
---
name: Jim
numbers:
- 1
- 2
- 3
```
```sh
$ oq -i yaml -o xml . data.yaml
Jim
1
2
3
```
Use `oq` as a library, consuming some raw `JSON` input, convert it to `YAML`, and write the transformed data to a file.
```crystal
require "oq"
# This could be any `IO`, e.g. an `HTTP` request body, etc.
input_io = IO::Memory.new %({"name":"Jim"})
# Create a processor, specifying that we want the output format to be `YAML`.
processor = OQ::Processor.new output_format: :yaml
File.open("./out.yml", "w") do |file|
# Process the data using our custom input and output IOs.
# The first argument represents the input arguments;
# i.e. the filter and/or any other arguments that should be passed to `jq`.
processor.process ["."], input: input_io, output: file
end
```
## Contributing
1. Fork it ()
2. Create your feature branch (`git checkout -b my-new-feature`)
3. Commit your changes (`git commit -am 'Add some feature'`)
4. Push to the branch (`git push origin my-new-feature`)
5. Create a new Pull Request
## Contributors
- [George Dietrich](https://github.com/Blacksmoke16) - creator, maintainer
- [Michael Springer](https://github.com/sprngr) - contributor
================================================
FILE: shard.yml
================================================
name: oq
description: |
A performant, and portable jq wrapper that facilitates the consumption and output of formats other than JSON; using jq filters to transform the data.
version: 1.3.5
authors:
- George Dietrich
crystal: ~> 1.4
license: MIT
targets:
oq:
main: src/oq_cli.cr
development_dependencies:
ameba:
github: crystal-ameba/ameba
version: ~> 1.5.0
================================================
FILE: snap/snapcraft.yaml
================================================
name: oq
version: '1.3.5'
summary: A performant, and portable jq wrapper to support formats other than JSON
description: |
A performant, and portable jq wrapper that facilitates the consumption and output of formats other than JSON; using jq filters to transform the data.
contact: george@dietrich.app
issues: https://github.com/Blacksmoke16/oq/issues
website: https://github.com/Blacksmoke16/oq
source-code: https://github.com/Blacksmoke16/oq.git
license: MIT
grade: stable
confinement: strict
base: core20
type: app
apps:
oq:
command: bin/oq
plugs:
- home
- removable-media
parts:
oq:
plugin: crystal
crystal-build-options:
- --release
- --no-debug
- '--link-flags=-s -Wl,-z,relro,-z,now'
source: ./
stage-packages:
- jq
override-pull: |
snapcraftctl pull
rm -rf $SNAPCRAFT_PART_SRC/lib $SNAPCRAFT_PART_SRC/bin
================================================
FILE: spec/assets/data1.json
================================================
{"name": "Jim"}
================================================
FILE: spec/assets/data1.yml
================================================
---
name: Jim
================================================
FILE: spec/assets/data2.json
================================================
{"name": "Bob"}
================================================
FILE: spec/assets/data2.yml
================================================
age: 17
name: Fred
================================================
FILE: spec/assets/raw.json
================================================
1
2
3
================================================
FILE: spec/assets/stream-data.json
================================================
{"machine": "possible_victim01", "domain": "evil.com", "timestamp":1435071870}
{"machine": "possible_victim01", "domain": "evil.com", "timestamp":1435071875}
{"machine": "possible_victim01", "domain": "soevil.com", "timestamp":1435071877}
{"machine": "possible_victim02", "domain": "bad.com", "timestamp":1435071877}
{"machine": "possible_victim03", "domain": "soevil.com", "timestamp":1435071879}
================================================
FILE: spec/assets/stream-filter
================================================
reduce inputs as $line
({};
$line.machine as $machine
| $line.domain as $domain
| .[$machine].total as $total
| .[$machine].evildoers as $evildoers
| . + { ($machine): {"total": (1 + $total),
"evildoers": ($evildoers | (.[$domain] += 1)) }} )
================================================
FILE: spec/assets/test.jq
================================================
def increment: . + 1;
================================================
FILE: spec/assets/test_filter
================================================
.name
================================================
FILE: spec/converters/simple_yaml_spec.cr
================================================
require "../spec_helper"
# Essentially copied from the `YAML` spec, minus the `with anchors` test.
#
# TODO: Allow the test code to be more easily shared.
describe OQ::Converters::SimpleYAML do
describe ".deserialize" do
describe String do
describe "not blank" do
it "should output correctly" do
run_binary(%(--- Jim), args: ["-i", "simpleyaml", "."]) do |output|
output.should eq %("Jim"\n)
end
end
end
describe "blank" do
it "should output correctly" do
run_binary(%(--- ), args: ["-i", "simpleyaml", "."]) do |output|
output.should eq "null\n"
end
end
end
describe "with a tag" do
it "should output correctly" do
run_binary(%(--- !!str 0.5), args: ["-i", "simpleyaml", "."]) do |output|
output.should eq %("0.5"\n)
end
end
end
describe "that is single quoted" do
it "should output correctly" do
run_binary(%(---\nhowever: 'foobar'), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"however":"foobar"}\n)
end
end
end
describe "that is double quoted" do
it "should output correctly" do
run_binary(%(---\nhowever: "foobar"), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"however":"foobar"}\n)
end
end
end
describe "literal block" do
it "should output correctly" do
run_binary(LITERAL_BLOCK, args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"literal_block":"This entire block of text will be the value of the 'literal_block' key,\\nwith line breaks being preserved.\\n\\nThe literal continues until de-dented, and the leading indentation is\\nstripped.\\n\\n Any lines that are 'more-indented' keep the rest of their indentation -\\n these lines will be indented by 4 spaces."}\n)
end
end
end
describe "folded block" do
it "should output correctly" do
run_binary(FOLDED_BLOCK, args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"folded_style":"This entire block of text will be the value of 'folded_style', but this time, all newlines will be replaced with a single space.\\nBlank lines, like above, are converted to a newline character.\\n\\n 'More-indented' lines keep their newlines, too -\\n this text will appear over two lines."}\n)
end
end
end
end
describe Bool do
it "should output correctly" do
run_binary(%(--- true), args: ["-i", "simpleyaml", "."]) do |output|
output.should eq "true\n"
end
end
end
describe Float do
it "should output correctly" do
run_binary(%(--- 10.50), args: ["-i", "simpleyaml", "."]) do |output|
output.should eq "10.5\n"
end
end
end
describe Nil do
it "should output correctly" do
run_binary(%(--- ), args: ["-i", "simpleyaml", "."]) do |output|
output.should eq "null\n"
end
end
end
describe Object do
describe "a simple object" do
it "should output correctly" do
run_binary(%(---\nname: Jim), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"name":"Jim"}\n)
end
end
end
describe "with spaces in the key" do
it "should output correctly" do
run_binary(%(---\nkey with spaces: value), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"key with spaces":"value"}\n)
end
end
end
describe "with a quoted key key" do
it "should output correctly" do
run_binary(%(---\n'Keys can be quoted too.': value), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"Keys can be quoted too.":"value"}\n)
end
end
end
describe "with nested object" do
it "should output correctly" do
run_binary(NESTED_OBJECT, args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"a_nested_map":{"key":"value","another_key":"Another Value","another_nested_map":{"hello":"hello"}}}\n)
end
end
end
describe "with a non string key" do
it "should output correctly" do
run_binary(%(---\n0.25: a float key), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"0.25":"a float key"}\n)
end
end
end
describe "with JSON syntax" do
describe "with quotes" do
it "should output correctly" do
run_binary(%(---\njson_seq: {"key": "value"}), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"json_seq":{"key":"value"}}\n)
end
end
end
describe "without quotes" do
it "should output correctly" do
run_binary(%(---\njson_seq: {key: value}), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"json_seq":{"key":"value"}}\n)
end
end
end
end
describe "with a complex mapping key" do
it "should output correctly" do
run_binary(COMPLEX_MAPPING_KEY, args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"This is a key\\nthat has multiple lines\\n":"and this is its value"}\n)
end
end
end
describe "with set notation" do
it "should output correctly" do
run_binary(%(---\nset:\n ? item1\n ? item2), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"set":{"item1":null,"item2":null}}\n)
end
end
end
pending "with a complex sequence key" do
it "should output correctly" do
run_binary(COMPLEX_SEQUENCE_KEY, args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"["Manchester United", "Real Madrid"]":["2001-01-01T00:00:00Z","2002-02-02T00:00:00Z"]}\n)
end
end
end
end
describe Array do
describe "with mixed/nested array values" do
it "should output correctly" do
run_binary(NESTED_ARRAY, args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"a_sequence":["Item 1","Item 2",0.5,"Item 4",{"key":"value","another_key":"another_value"},["This is a sequence","inside another sequence"],[["Nested sequence indicators","can be collapsed"]]]}\n)
end
end
end
describe "with JSON syntax" do
describe "with quotes" do
it "should output correctly" do
run_binary(%(---\njson_seq: [3, 2, 1, "takeoff"]), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"json_seq":[3,2,1,"takeoff"]}\n)
end
end
end
describe "without quotes" do
it "should output correctly" do
run_binary(%(---\njson_seq: [3, 2, 1, takeoff]), args: ["-i", "simpleyaml", "-c", "."]) do |output|
output.should eq %({"json_seq":[3,2,1,"takeoff"]}\n)
end
end
end
end
end
end
describe ".serialize" do
describe String do
describe "not blank" do
it "should output correctly" do
run_binary(%("Jim"), args: ["-o", "simpleyaml", "."]) do |output|
output.should start_with <<-YAML
--- Jim
YAML
end
end
end
describe "blank" do
it "should output correctly" do
run_binary(%(""), args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
--- ""\n
YAML
)
end
end
end
end
describe Bool do
it "should output correctly" do
run_binary(%(true), args: ["-o", "simpleyaml", "."]) do |output|
output.should start_with <<-YAML
--- true
YAML
end
end
end
describe Float do
it "should output correctly" do
run_binary(%("1.5"), args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
--- "1.5"\n
YAML
)
end
end
end
describe Nil do
it "should output correctly" do
run_binary("null", args: ["-o", "simpleyaml", "."]) do |output|
output.should start_with "---"
end
end
end
describe Array do
describe "empty array on root" do
it "should emit a self closing root tag" do
run_binary("[]", args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
--- []\n
YAML
)
end
end
end
describe "array with values on root" do
it "should emit item tags for non empty values" do
run_binary(%(["x",{}]), args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
---
- x
- {}\n
YAML
)
end
end
end
describe "object with empty array/values" do
it "should emit self closing tags for each" do
run_binary(%({"a":[],"b":{},"c":null}), args: ["-o", "simpleyaml", "."]) do |output|
output.should start_with <<-YAML
---
a: []
b: {}
c:
YAML
end
end
end
describe "2D array object value" do
it "should emit key name tag then self closing item tag" do
run_binary(%({"a":[[]]}), args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
---
a:
- []\n
YAML
)
end
end
end
describe "object value mixed/nested array values" do
it "should emit correctly" do
run_binary(%({"x":[1,[2,[3]]]}), args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
---
x:
- 1
- - 2
- - 3\n
YAML
)
end
end
end
describe "object value array primitive values" do
it "should emit correctly" do
run_binary(%({"x":[1,2,3]}), args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
---
x:
- 1
- 2
- 3\n
YAML
)
end
end
end
describe "when the jq filter doesn't return data" do
it "should return an empty string" do
run_binary(%([{"name":"foo"}]), args: ["-i", "simpleyaml", "-o", "simpleyaml", %<.[] | select(.name != "foo")>]) do |output|
output.should be_empty
end
end
end
end
describe Object do
describe "simple key/value" do
it "should output correctly" do
run_binary(%({"name":"Jim"}), args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
---
name: Jim\n
YAML
)
end
end
end
describe "nested object" do
it "should output correctly" do
run_binary(%({"name":"Jim", "city": {"street":"forbs"}}), args: ["-o", "simpleyaml", "."]) do |output|
output.should eq(<<-YAML
---
name: Jim
city:
street: forbs\n
YAML
)
end
end
end
end
end
end
================================================
FILE: spec/converters/xml_spec.cr
================================================
require "../spec_helper"
WITH_WHITESPACE = <<-XML
-
0
0
0
0
-1
0
XML
XML_SCALAR_ARRAY = <<-XML
1
2
3
XML
XML_SCALAR_ARRAY_WITH_ATTRIBUTE = <<-XML
1
2
3
XML
XML_CDATA = <<-XML
Some Description]]>
XML
XML_OBJECT_ARRAY = <<-XML
-
0
0
0
0
-1
0
-
0
1
0
0
-1
0
XML
XML_NESTED_OBJECT_ARRAY = <<-XML
cubsfantony
848
Visa/MasterCard, Money Order/Cashiers Checks, Personal Checks, See item description for payment methods accepted
ct-inc
403
Visa/MasterCard, Discover, Money Order/Cashiers Checks, Personal Checks, See item description for payment methods accepted
XML
XML_INLINE_ARRAY = <<-XML
E. F. Codd
Robert S. Arnold
Jean-Marc Cadiou
Chin-Liang Chang
Nick Roussopoulos
RENDEZVOUS Version 1: An Experimental English Language Query Formulation System for Casual Users of Relational Data Bases.
IBM Research Report
RJ2144
January
1978
db/labs/ibm/RJ2144.html
ibmTR/rj2144.pdf
XML
XML_INLINE_ARRAY_WITHIN_ARRAY = <<-XML
1997
db/labs/dec/SRC1997-018.html
http://www.mcjones.org/System_R/SQL_Reunion_95/
db/labs/gte/TR-0263-08-94-165.html
1994
XML
XML_DOCTYPE = <<-XML
Kurt P. Brown
PRPL: A Database Workload Specification Language, v1.3.
1992
Univ. of Wisconsin-Madison
XML
XML_ATTRIBUTE_IN_ARRAY = <<-XML
80000
full-time
full-time
XML
XML_ATTRIBUTE_IN_ARRAY_ROOT_ELEMENT = <<-XML
Kurt P. Brown
Tolga Yurek
XML
XML_ALL_EMPTY = <<-XML
XML
XML_NAMESPACE_ARRAY = <<-XML
1
2
3
XML
XML_NAMESPACE_ARRAY_SCALAR_VALUE_PREFIX = <<-XML
1
2
3
XML
XML_NAMESPACE_PREFIXES = <<-XML
foo
bar
XML
XML_NESTED_NAMESPACES = <<-XML
herp
XML
describe OQ::Converters::XML do
describe ".deserialize" do
# See https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html
describe "conventions" do
describe "an empty element" do
it "self closing" do
run_binary("", args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"e":null}\n)
end
end
it "non self closing" do
run_binary("", args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"e":null}\n)
end
end
end
it "an element with pure text content" do
run_binary("text", args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"e":"text"}\n)
end
end
it "an empty element with attributes" do
run_binary(%(), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"e":{"@name":"value"}}\n)
end
end
it "an element with pure text content and attributes" do
run_binary(%(text), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"e":{"@name":"value","#text":"text"}}\n)
end
end
it "an element containing elements with different names" do
run_binary(%( text text ), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"e":{"a":"text","b":"text"}}\n)
end
end
it "an element containing elements with identical names" do
run_binary(%( text text ), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"e":{"a":["text","text"]}}\n)
end
end
it "an element containing elements and contiguous text" do
run_binary(%(texttext), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"e":{"#text":"text","a":"text"}}\n)
end
end
end
describe "should raise if invalid" do
it "should output correctly" do
run_binary(%(Fred), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"person":"Fred"}\n)
end
end
describe "that has only empty children elements" do
it "should output an object with null values" do
run_binary(XML_ALL_EMPTY, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"root":{"one":" ","two":"\\n ","three":null,"four":null}}\n)
end
end
end
it "with whitespace" do
run_binary(WITH_WHITESPACE, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"item":{"flagID":"0","itemID":"0","locationID":"0","ownerID":"0","quantity":"-1","typeID":"0"}}\n)
end
end
it "with the prolog" do
run_binary(%(- 0
), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"item":{"typeID":"0"}}\n)
end
end
it "a simple object" do
run_binary(%(JaneDoe), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"person":{"firstname":"Jane","lastname":"Doe"}}\n)
end
end
it "attributes" do
run_binary(%(JaneDoe), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"person":{"@id":"1","@foo":"bar","firstname":"Jane","lastname":"Doe"}}\n)
end
end
it "nested objects" do
run_binary(%(JaneDoe15061123 Foo Street), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"person":{"firstname":"Jane","lastname":"Doe","location":{"zip":"15061","address":"123 Foo Street"}}}\n)
end
end
it "complex object" do
run_binary(%(24), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"root":{"x":{"@a":"1","a":"2"},"y":{"@b":"3","#text":"4"}}}\n)
end
end
it "with mixed content" do
run_binary(%(xz), args: ["-i", "xml", "-c", ".root"]) do |output|
output.should eq %({"#text":"x","y":"z"}\n)
end
end
it "with an inline array" do
run_binary(XML_INLINE_ARRAY, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"article":{"@key":"tr/ibm/RJ2144","author":["E. F. Codd","Robert S. Arnold","Jean-Marc Cadiou","Chin-Liang Chang","Nick Roussopoulos"],"title":"RENDEZVOUS Version 1: An Experimental English Language Query Formulation System for Casual Users of Relational Data Bases.","journal":"IBM Research Report","volume":"RJ2144","month":"January","year":"1978","ee":"db/labs/ibm/RJ2144.html","cdrom":"ibmTR/rj2144.pdf"}}\n)
end
end
it "with a doctype" do
run_binary(XML_DOCTYPE, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"dblp":{"mastersthesis":{"@key":"ms/Brown92","author":"Kurt P. Brown","title":"PRPL: A Database Workload Specification Language, v1.3.","year":"1992","school":"Univ. of Wisconsin-Madison"}}}\n)
end
end
it "with CDATA" do
run_binary(XML_CDATA, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"desc":"Some Description"}\n)
end
end
it "with a prefixed key" do
run_binary(%(bar), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"a:foo":"bar"}\n)
end
end
describe "with namespaces" do
describe "without --xmlns" do
it "retains prefixes but strips namespace declarations of a prefixed namespace" do
run_binary(%(bar), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"a:foo":"bar"}\n)
end
end
it "does not add pefix if none was already present but strips namespace declarations" do
run_binary(%(bar), args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"foo":"bar"}\n)
end
end
it "adds namespace attribute properties only to declaring element and handles differentiating prefixed elements" do
run_binary(XML_NESTED_NAMESPACES, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"root":{"a:foo":"herp","foo":{"bar":{"baz":null}}}}\n)
end
end
it "retains prefixes of scalar value elements" do
run_binary(XML_NAMESPACE_PREFIXES, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"root":{"foo":"foo","a:bar":"bar"}}\n)
end
end
describe "with --xml-namespace-alias" do
it "should error" do
run_binary(%(bar), args: ["-i", "xml", "-c", "--xml-namespace-alias", "aa=https://a-namespace", "."], success: false) do |_, _, error|
error.should start_with "oq error:"
end
end
end
end
describe "with --xmlns" do
it "creates a namespace attribute property" do
run_binary(%(bar), args: ["-i", "xml", "-c", "--xmlns", "."]) do |output|
output.should eq %({"a:foo":{"@xmlns:a":"http://www.w3.org/1999/xhtml","#text":"bar"}}\n)
end
end
it "does not add pefix if none was already present and creates multiple namespace attribute properties" do
run_binary(%(bar), args: ["-i", "xml", "-c", "--xmlns", "."]) do |output|
output.should eq %({"foo":{"@xmlns":"urn:oasis:names:tc:SAML:2.0:metadata","@xmlns:a":"http://www.w3.org/1999/xhtml","#text":"bar"}}\n)
end
end
it "treats prefixed & unprefixed elements as unique elements" do
run_binary(XML_NESTED_NAMESPACES, args: ["-i", "xml", "-c", "--xmlns", "."]) do |output|
output.should eq %({"root":{"@xmlns:a":"https://a","@xmlns":"https://b","a:foo":"herp","foo":{"bar":{"@xmlns":"https://c","baz":{"@xmlns":"https://d"}}}}}\n)
end
end
it "retains prefixes of scalar value elements and adds a namespace attribute property" do
run_binary(XML_NAMESPACE_PREFIXES, args: ["-i", "xml", "-c", "--xmlns", "."]) do |output|
output.should eq %({"root":{"@xmlns:a":"https://a","foo":"foo","a:bar":"bar"}}\n)
end
end
describe "with --xml-namespace-alias" do
it "normalizes the provided namespace" do
run_binary(%(bar), args: ["-i", "xml", "-c", "--xmlns", "--xml-namespace-alias", "aa=https://a-namespace", "."]) do |output|
output.should eq %({"aa:foo":{"@xmlns:aa":"https://a-namespace","#text":"bar"}}\n)
end
end
it "normalizes the default namespace" do
run_binary(%(bar), args: ["-i", "xml", "-c", "--xmlns", "--xml-namespace-alias", "aa=https://a-namespace", "."]) do |output|
output.should eq %({"aa:foo":{"@xmlns:aa":"https://a-namespace","#text":"bar"}}\n)
end
end
it "normalizes multiple namespaces" do
run_binary(XML_NESTED_NAMESPACES, args: ["-i", "xml", "-c", "--xmlns", "--xml-namespace-alias", "=https://a", "--xml-namespace-alias", "bb=https://b", "."]) do |output|
output.should eq %({"bb:root":{"@xmlns":"https://a","@xmlns:bb":"https://b","foo":"herp","bb:foo":{"bar":{"@xmlns":"https://c","baz":{"@xmlns":"https://d"}}}}}\n)
end
end
end
end
end
end
describe Array do
it "of scalar values" do
run_binary(XML_SCALAR_ARRAY, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"items":{"number":["1","2","3"]}}\n)
end
end
it "of scalar values with attribute" do
run_binary(XML_SCALAR_ARRAY_WITH_ATTRIBUTE, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"items":{"number":["1","2",{"@foo":"bar","#text":"3"}]}}\n)
end
end
describe "of objects" do
describe "with no nested objects" do
it "should output correctly" do
run_binary(XML_OBJECT_ARRAY, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"items":{"item":[{"flagID":"0","itemID":"0","locationID":"0","ownerID":"0","quantity":"-1","typeID":"0"},{"flagID":"0","itemID":"1","locationID":"0","ownerID":"0","quantity":"-1","typeID":"0"}]}}\n)
end
end
end
describe "with an inline array" do
it "should output correctly" do
run_binary(XML_INLINE_ARRAY_WITHIN_ARRAY, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"articles":{"article":[{"@key":"tr/dec/SRC1997-018","year":"1997","ee":["db/labs/dec/SRC1997-018.html","http://www.mcjones.org/System_R/SQL_Reunion_95/"]},{"@key":"tr/gte/TR-0263-08-94-165","ee":"db/labs/gte/TR-0263-08-94-165.html","year":"1994"}]}}\n)
end
end
end
describe "with nested objects" do
it "should output correctly" do
run_binary(XML_NESTED_OBJECT_ARRAY, args: ["-i", "xml", "-c", ".root.listing"]) do |output|
output.should eq %([{"seller_info":{"seller_name":" cubsfantony","seller_rating":" 848"},"payment_types":"Visa/MasterCard, Money Order/Cashiers Checks, Personal Checks, See item description for payment methods accepted"},{"seller_info":{"seller_name":" ct-inc","seller_rating":" 403"},"payment_types":"Visa/MasterCard, Discover, Money Order/Cashiers Checks, Personal Checks, See item description for payment methods accepted"}]\n)
end
end
end
end
describe "with object that has an attribute" do
it "should output correctly" do
run_binary(XML_ATTRIBUTE_IN_ARRAY, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"jobs":{"ad":[{"salary":{"@currency":"CAD","#text":"80000"},"working_hours":"full-time"},{"working_hours":"full-time"}]}}\n)
end
end
end
describe "where array object element has an attribute" do
it "should output correctly" do
run_binary(XML_ATTRIBUTE_IN_ARRAY_ROOT_ELEMENT, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"dblp":{"mastersthesis":[{"@key":"ms/Brown92","author":"Kurt P. Brown"},{"@key":"ms/Yurek97","author":"Tolga Yurek"}]}}\n)
end
end
end
describe "with namespaces" do
describe "without --xmlns" do
it "treats prefixed & unprefixed elements as unique elements" do
run_binary(XML_NAMESPACE_ARRAY, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"items":{"n:number":["1","2"],"number":"3"}}\n)
end
end
it "ignores the namespace declaration" do
run_binary(XML_NAMESPACE_ARRAY_SCALAR_VALUE_PREFIX, args: ["-i", "xml", "-c", "."]) do |output|
output.should eq %({"items":{"n:number":["1","2","3"]}}\n)
end
end
end
describe "with --xmlns" do
it "treats prefixed & unprefixed elements as unique elements, adding namespace attribute property as needed" do
run_binary(XML_NAMESPACE_ARRAY, args: ["-i", "xml", "-c", "--xmlns", "."]) do |output|
output.should eq %({"items":{"@xmlns:n":"http://n","n:number":["1","2"],"number":{"@xmlns":"http://default","#text":"3"}}}\n)
end
end
it "expands the scalar value to include a namespace attribute property" do
run_binary(XML_NAMESPACE_ARRAY_SCALAR_VALUE_PREFIX, args: ["-i", "xml", "-c", "--xmlns", "."]) do |output|
output.should eq %({"items":{"@xmlns:n":"http://n","n:number":["1","2",{"@xmlns":"http://default","#text":"3"}]}}\n)
end
end
describe "with --xml-namespace-alias" do
it do
run_binary(XML_NAMESPACE_ARRAY, args: ["-i", "xml", "-c", "--xmlns", "--xml-namespace-alias", "num=http://n", "."]) do |output|
output.should eq %({"items":{"@xmlns:num":"http://n","num:number":["1","2"],"number":{"@xmlns":"http://default","#text":"3"}}}\n)
end
end
it do
run_binary(XML_NAMESPACE_ARRAY, args: ["-i", "xml", "-c", "--xmlns", "--xml-namespace-alias", "=http://n", "--xml-namespace-alias", "d=http://default", "."]) do |output|
output.should eq %({"items":{"@xmlns":"http://n","number":["1","2"],"d:number":{"@xmlns:d":"http://default","#text":"3"}}}\n)
end
end
end
end
end
describe "with a single element" do
it "without --xml-force-array" do
run_binary(%(Hello World!
"}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
<p>Hello World!</p>
\n
XML
)
end
end
it "should be wrapped in CDATA if the json key starts with '!'" do
run_binary(%({"!x":"Hello World!
"}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
Hello World!]]>
\n
XML
)
end
end
it "should produce an empty CDATA if the json key starts with '!' and the value is null" do
run_binary(%({"!x":null}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
\n
XML
)
end
end
end
end
describe Bool do
it "should output correctly" do
run_binary(%(true), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
true\n
XML
)
end
end
end
describe Float do
it "should output correctly" do
run_binary(%("1.5"), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
1.5\n
XML
)
end
end
end
describe Nil do
it "should output correctly" do
run_binary("null", args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
\n
XML
)
end
end
end
describe Array do
describe "empty array on root" do
it "should emit a self closing root tag" do
run_binary("[]", args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
\n
XML
)
end
end
end
describe "array with values on root" do
it "should emit item tags for non empty values" do
run_binary(%(["x",{}]), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
- x
- 2
-
- 3
\n
XML
)
end
end
it "object value array primitive values" do
run_binary(%({"x":[1,2,3]}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
1
2
3
\n
XML
)
end
end
end
describe Object do
it "simple key/value" do
run_binary(%({"name":"Jim"}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
Jim
\n
XML
)
end
end
it "nested object" do
run_binary(%({"name":"Jim", "city": {"street":"forbs"}}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
Jim
forbs
\n
XML
)
end
end
it "with an attribute" do
run_binary(%({"name":"Jim", "city": {"@street":"forbs"}}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
Jim
\n
XML
)
end
end
it "with an attribute and #text" do
run_binary(%({"name":"Jim", "city": {"@street":"forbs", "#text": "Atlantic"}}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
Jim
Atlantic
\n
XML
)
end
end
it "with attributes" do
run_binary(%({"name":"Jim", "city": {"@street":"forbs", "@post": 123}}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
Jim
\n
XML
)
end
end
it "with attributes and #text" do
run_binary(%({"name":"Jim", "city": {"@street":"forbs", "@post": 123, "#text": "Atlantic"}}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
Jim
Atlantic
\n
XML
)
end
end
it "with a prefixed key" do
run_binary(%({"foo:name":"Jim"}), args: ["-o", "xml", "."]) do |output|
output.should eq(<<-XML
Jim
\n
XML
)
end
end
end
end
end
================================================
FILE: spec/converters/yaml_spec.cr
================================================
require "../spec_helper"
LITERAL_BLOCK = <<-YAML
---
literal_block: |
This entire block of text will be the value of the 'literal_block' key,
with line breaks being preserved.
The literal continues until de-dented, and the leading indentation is
stripped.
Any lines that are 'more-indented' keep the rest of their indentation -
these lines will be indented by 4 spaces.
YAML
FOLDED_BLOCK = <<-YAML
folded_style: >
This entire block of text will be the value of 'folded_style', but this
time, all newlines will be replaced with a single space.
Blank lines, like above, are converted to a newline character.
'More-indented' lines keep their newlines, too -
this text will appear over two lines.
YAML
NESTED_OBJECT = <<-YAML
a_nested_map:
key: value
another_key: Another Value
another_nested_map:
hello: hello
YAML
COMPLEX_MAPPING_KEY = <<-YAML
? |
This is a key
that has multiple lines
: and this is its value
YAML
COMPLEX_SEQUENCE_KEY = <<-YAML
? - Manchester United
- Real Madrid
: [2001-01-01, 2002-02-02]
YAML
NESTED_ARRAY = <<-YAML
a_sequence:
- Item 1
- Item 2
- 0.5 # sequences can contain disparate types.
- Item 4
- key: value
another_key: another_value
-
- This is a sequence
- inside another sequence
- - - Nested sequence indicators
- can be collapsed
YAML
ANCHORS = <<-YAML
base: &base
name: Everyone has same name
foo: &foo
<<: *base
age: 10
bar: &bar
<<: *base
age: 20
YAML
describe OQ::Converters::YAML do
describe ".deserialize" do
describe String do
describe "not blank" do
it "should output correctly" do
run_binary(%(--- Jim), args: ["-i", "yaml", "."]) do |output|
output.should eq %("Jim"\n)
end
end
end
describe "blank" do
it "should output correctly" do
run_binary(%(--- ), args: ["-i", "yaml", "."]) do |output|
output.should eq "null\n"
end
end
end
describe "with a tag" do
it "should output correctly" do
run_binary(%(--- !!str 0.5), args: ["-i", "yaml", "."]) do |output|
output.should eq %("0.5"\n)
end
end
end
describe "that is single quoted" do
it "should output correctly" do
run_binary(%(---\nhowever: 'foobar'), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"however":"foobar"}\n)
end
end
end
describe "that is double quoted" do
it "should output correctly" do
run_binary(%(---\nhowever: "foobar"), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"however":"foobar"}\n)
end
end
end
describe "literal block" do
it "should output correctly" do
run_binary(LITERAL_BLOCK, args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"literal_block":"This entire block of text will be the value of the 'literal_block' key,\\nwith line breaks being preserved.\\n\\nThe literal continues until de-dented, and the leading indentation is\\nstripped.\\n\\n Any lines that are 'more-indented' keep the rest of their indentation -\\n these lines will be indented by 4 spaces."}\n)
end
end
end
describe "folded block" do
it "should output correctly" do
run_binary(FOLDED_BLOCK, args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"folded_style":"This entire block of text will be the value of 'folded_style', but this time, all newlines will be replaced with a single space.\\nBlank lines, like above, are converted to a newline character.\\n\\n 'More-indented' lines keep their newlines, too -\\n this text will appear over two lines."}\n)
end
end
end
end
describe Bool do
it "should output correctly" do
run_binary(%(--- true), args: ["-i", "yaml", "."]) do |output|
output.should eq "true\n"
end
end
end
describe Float do
it "should output correctly" do
run_binary(%(--- 10.50), args: ["-i", "yaml", "."]) do |output|
output.should eq "10.5\n"
end
end
end
describe Nil do
it "should output correctly" do
run_binary(%(--- ), args: ["-i", "yaml", "."]) do |output|
output.should eq "null\n"
end
end
end
describe Object do
describe "a simple object" do
it "should output correctly" do
run_binary(%(---\nname: Jim), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"name":"Jim"}\n)
end
end
end
describe "with spaces in the key" do
it "should output correctly" do
run_binary(%(---\nkey with spaces: value), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"key with spaces":"value"}\n)
end
end
end
describe "with a quoted key key" do
it "should output correctly" do
run_binary(%(---\n'Keys can be quoted too.': value), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"Keys can be quoted too.":"value"}\n)
end
end
end
describe "with nested object" do
it "should output correctly" do
run_binary(NESTED_OBJECT, args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"a_nested_map":{"key":"value","another_key":"Another Value","another_nested_map":{"hello":"hello"}}}\n)
end
end
end
describe "with a non string key" do
it "should output correctly" do
run_binary(%(---\n0.25: a float key), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"0.25":"a float key"}\n)
end
end
end
describe "with JSON syntax" do
describe "with quotes" do
it "should output correctly" do
run_binary(%(---\njson_seq: {"key": "value"}), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"json_seq":{"key":"value"}}\n)
end
end
end
describe "without quotes" do
it "should output correctly" do
run_binary(%(---\njson_seq: {key: value}), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"json_seq":{"key":"value"}}\n)
end
end
end
end
describe "with a complex mapping key" do
it "should output correctly" do
run_binary(COMPLEX_MAPPING_KEY, args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"This is a key\\nthat has multiple lines\\n":"and this is its value"}\n)
end
end
end
describe "with set notation" do
it "should output correctly" do
run_binary(%(---\nset:\n ? item1\n ? item2), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"set":{"item1":null,"item2":null}}\n)
end
end
end
pending "with a complex sequence key" do
it "should output correctly" do
run_binary(COMPLEX_SEQUENCE_KEY, args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"["Manchester United", "Real Madrid"]":["2001-01-01T00:00:00Z","2002-02-02T00:00:00Z"]}\n)
end
end
end
describe "with anchors" do
it "should output correctly" do
run_binary(ANCHORS, args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"base":{"name":"Everyone has same name"},"foo":{"name":"Everyone has same name","age":10},"bar":{"name":"Everyone has same name","age":20}}\n)
end
end
end
end
describe Array do
describe "with mixed/nested array values" do
it "should output correctly" do
run_binary(NESTED_ARRAY, args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"a_sequence":["Item 1","Item 2",0.5,"Item 4",{"key":"value","another_key":"another_value"},["This is a sequence","inside another sequence"],[["Nested sequence indicators","can be collapsed"]]]}\n)
end
end
end
describe "with JSON syntax" do
describe "with quotes" do
it "should output correctly" do
run_binary(%(---\njson_seq: [3, 2, 1, "takeoff"]), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"json_seq":[3,2,1,"takeoff"]}\n)
end
end
end
describe "without quotes" do
it "should output correctly" do
run_binary(%(---\njson_seq: [3, 2, 1, takeoff]), args: ["-i", "yaml", "-c", "."]) do |output|
output.should eq %({"json_seq":[3,2,1,"takeoff"]}\n)
end
end
end
end
end
end
describe ".serialize" do
describe String do
describe "not blank" do
it "should output correctly" do
run_binary(%("Jim"), args: ["-o", "yaml", "."]) do |output|
output.should start_with <<-YAML
--- Jim
YAML
end
end
end
describe "blank" do
it "should output correctly" do
run_binary(%(""), args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
--- ""\n
YAML
)
end
end
end
end
describe Bool do
it "should output correctly" do
run_binary(%(true), args: ["-o", "yaml", "."]) do |output|
output.should start_with <<-YAML
--- true
YAML
end
end
end
describe Float do
it "should output correctly" do
run_binary(%("1.5"), args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
--- "1.5"\n
YAML
)
end
end
end
describe Nil do
it "should output correctly" do
run_binary("null", args: ["-o", "yaml", "."]) do |output|
output.should start_with "---"
end
end
end
describe Array do
describe "empty array on root" do
it "should emit a self closing root tag" do
run_binary("[]", args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
--- []\n
YAML
)
end
end
end
describe "array with values on root" do
it "should emit item tags for non empty values" do
run_binary(%(["x",{}]), args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
---
- x
- {}\n
YAML
)
end
end
end
describe "object with empty array/values" do
it "should emit self closing tags for each" do
run_binary(%({"a":[],"b":{},"c":null}), args: ["-o", "yaml", "."]) do |output|
output.should start_with <<-YAML
---
a: []
b: {}
c:
YAML
end
end
end
describe "2D array object value" do
it "should emit key name tag then self closing item tag" do
run_binary(%({"a":[[]]}), args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
---
a:
- []\n
YAML
)
end
end
end
describe "object value mixed/nested array values" do
it "should emit correctly" do
run_binary(%({"x":[1,[2,[3]]]}), args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
---
x:
- 1
- - 2
- - 3\n
YAML
)
end
end
end
describe "object value array primitive values" do
it "should emit correctly" do
run_binary(%({"x":[1,2,3]}), args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
---
x:
- 1
- 2
- 3\n
YAML
)
end
end
end
describe "when the jq filter doesn't return data" do
it "should return an empty string" do
run_binary(%([{"name":"foo"}]), args: ["-i", "yaml", "-o", "yaml", %<.[] | select(.name != "foo")>]) do |output|
output.should be_empty
end
end
end
end
describe Object do
describe "simple key/value" do
it "should output correctly" do
run_binary(%({"name":"Jim"}), args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
---
name: Jim\n
YAML
)
end
end
end
describe "nested object" do
it "should output correctly" do
run_binary(%({"name":"Jim", "city": {"street":"forbs"}}), args: ["-o", "yaml", "."]) do |output|
output.should eq(<<-YAML
---
name: Jim
city:
street: forbs\n
YAML
)
end
end
end
end
end
end
================================================
FILE: spec/format_spec.cr
================================================
require "./spec_helper"
describe OQ::Format do
describe ".to_s" do
it "returns a comma separated list of the formats" do
OQ::Format.to_s.should eq "json, simpleyaml, xml, yaml"
end
end
end
================================================
FILE: spec/oq_spec.cr
================================================
require "./spec_helper"
private SIMPLE_JSON_OBJECT = <<-JSON
{
"name": "Jim"
}
JSON
private NESTED_JSON_OBJECT = <<-JSON
{"foo":{"bar":{"baz":5}}}
JSON
private ARRAY_JSON_OBJECT = <<-JSON
{"names":[1,2,3]}
JSON
describe OQ do
describe "when given a filter file" do
it "should return the correct output" do
run_binary(input: SIMPLE_JSON_OBJECT, args: ["-f", "spec/assets/test_filter"]) do |output|
output.should eq %("Jim"\n)
end
end
end
it "with a simple filter" do
run_binary(input: SIMPLE_JSON_OBJECT, args: [".name"]) do |output|
output.should eq %("Jim"\n)
end
end
it "with a filter to get nested values" do
run_binary(input: NESTED_JSON_OBJECT, args: [".foo.bar.baz"]) do |output|
output.should eq "5\n"
end
end
it "should colorize the output with the -C option" do
run_binary(input: SIMPLE_JSON_OBJECT, args: ["-c", "-C", "."]) do |output|
output.should start_with "\e"
output.should contain %("name")
output.should contain %("Jim")
output.should end_with "\e[0m\n"
end
end
describe "with a non-JSON output format" do
it "should convert the JSON to that format" do
run_binary(input: SIMPLE_JSON_OBJECT, args: ["-o", "yaml", "."]) do |output|
output.should eq "---\nname: Jim\n"
end
end
describe "with the -C option" do
it "should remove the -C option" do
run_binary(input: SIMPLE_JSON_OBJECT, args: ["-o", "yaml", "-C", "."]) do |output|
output.should eq "---\nname: Jim\n"
end
end
end
end
describe "files" do
describe "with a file input" do
it "should return the correct output" do
run_binary(args: [".", "spec/assets/data1.json"]) do |output|
output.should eq "#{SIMPLE_JSON_OBJECT}\n"
end
end
end
describe "with multiple JSON file input" do
it "raw data" do
run_binary(args: ["-c", ".", "spec/assets/data1.json", "spec/assets/data2.json"]) do |output|
output.should eq %({"name":"Jim"}\n{"name":"Bob"}\n)
end
end
it "--slurp" do
run_binary(args: ["-c", "--slurp", ".", "spec/assets/data1.json", "spec/assets/data2.json"]) do |output|
output.should eq %([{"name":"Jim"},{"name":"Bob"}]\n)
end
end
end
describe "with multiple non JSON file input" do
it "raw data" do
run_binary(args: ["-i", "yaml", "-c", ".", "spec/assets/data1.yml", "spec/assets/data2.yml"]) do |output|
output.should eq %({"name":"Jim"}\n{"age":17,"name":"Fred"}\n)
end
end
it "--slurp" do
run_binary(args: ["-i", "yaml", "-c", "--slurp", ".", "spec/assets/data1.yml", "spec/assets/data2.yml"]) do |output|
output.should eq %([{"name":"Jim"},{"age":17,"name":"Fred"}]\n)
end
end
end
it "with multiple --arg" do
run_binary(args: ["-c", "-r", "--arg", "chart", "stolon", "--arg", "version", "1.5.10", "$version", "spec/assets/data1.json"]) do |output|
output.should eq %(1.5.10\n)
end
end
end
it "should minify the output with the -c option" do
run_binary(input: NESTED_JSON_OBJECT, args: ["-c", "."]) do |output|
output.should eq %({"foo":{"bar":{"baz":5}}}\n)
end
end
it "should format the output without the -c option" do
run_binary(input: NESTED_JSON_OBJECT, args: ["."]) do |output|
output.should eq(<<-JSON
{
"foo": {
"bar": {
"baz": 5
}
}
}\n
JSON
)
end
end
describe "with null input option" do
describe "with a scalar value" do
it "should return the correct output" do
run_binary(args: ["-n", "0"]) do |output|
output.should eq "0\n"
end
end
it "should return the correct output" do
run_binary(args: ["--null-input", "0"]) do |output|
output.should eq "0\n"
end
end
end
describe "with a JSON object string" do
it "should return the correct output" do
run_binary(args: ["-cn", %([{"foo":"bar"},{"foo":"baz"}])]) do |output|
output.should eq %([{"foo":"bar"},{"foo":"baz"}]\n)
end
end
end
describe "with input from STDIN" do
it "should return the correct output" do
run_binary(input: "foo", args: ["-n", "."]) do |output|
output.should eq "null\n"
end
end
end
it "should not block waiting for input" do
run_binary(input: Process::Redirect::Inherit, args: ["-n", "."]) do |output|
output.should eq "null\n"
end
end
end
describe "with a custom indent value with JSON" do
it "should return the correct output" do
run_binary(input: SIMPLE_JSON_OBJECT, args: ["--indent", "1", "."]) do |output|
output.should eq %({\n "name": "Jim"\n}\n)
end
end
end
describe "when streaming input" do
it "should return the correct output" do
run_binary(input: %({"a": [1, 2.2, true, "abc", null]}), args: ["-nc", "--stream", "fromstream( 1|truncate_stream(inputs) | select(length>1) | .[0] |= .[1:] )"]) do |output|
output.should eq %(1\n2.2\ntrue\n"abc"\nnull\n)
end
end
end
describe "when using 'input'" do
it "should return the correct output" do
run_binary(args: ["-cn", "-f", "spec/assets/stream-filter", "spec/assets/stream-data.json"]) do |output|
output.should eq %({"possible_victim01":{"total":3,"evildoers":{"evil.com":2,"soevil.com":1}},"possible_victim02":{"total":1,"evildoers":{"bad.com":1}},"possible_victim03":{"total":1,"evildoers":{"soevil.com":1}}}\n)
end
end
end
describe "with the -L option" do
it "should be passed correctly without a space" do
run_binary(args: ["-n", "-L#{__DIR__}/assets", %(import "test" as test; 9 | test::increment)]) do |output|
output.should eq %(10\n)
end
end
it "should be passed correctly with a space" do
run_binary(args: ["-n", "-L", "#{__DIR__}/assets", %(import "test" as test; 9 | test::increment)]) do |output|
output.should eq %(10\n)
end
end
end
describe "--arg" do
it "single arg" do
run_binary(args: ["-cn", "--arg", "foo", "bar", %({"name":$foo})]) do |output|
output.should eq %({"name":"bar"}\n)
end
end
it "multiple arg" do
run_binary(args: ["-rcn", "-r", "--arg", "chart", "stolon", "--arg", "version", "1.5.10", "$version"]) do |output|
output.should eq %(1.5.10\n)
end
end
it "different option in between args" do
run_binary(args: ["-rcn", "--arg", "chart", "stolon", "--arg", "version", "1.5.10", "$version"]) do |output|
output.should eq %(1.5.10\n)
end
end
it "when the arg name matches a directory name" do
run_binary(args: ["-rcn", "--arg", "spec", "dir", "$spec"]) do |output|
output.should eq %(dir\n)
end
end
end
describe "with the --argjson option" do
it "should be passed correctly" do
run_binary(args: ["-rcn", "--argjson", "foo", "123", %({"id":$foo})]) do |output|
output.should eq %({"id":123}\n)
end
end
it "when the arg name matches a directory name" do
run_binary(args: ["-rcn", "--argjson", "spec", "123", "$spec"]) do |output|
output.should eq %(123\n)
end
end
end
describe "with the --slurpfile option" do
it "should be passed correctly" do
run_binary(args: ["-rcn", "--slurpfile", "ids", "spec/assets/raw.json", %({"ids":$ids})]) do |output|
output.should eq %({"ids":[1,2,3]}\n)
end
end
end
describe "with the --rawfile option" do
it "should be passed correctly" do
run_binary(args: ["-rcn", "--rawfile", "ids", "spec/assets/raw.json", %({"ids":$ids})]) do |output|
output.should eq %({"ids":"1\\n2\\n3\\n"}\n)
end
end
end
describe "with the --args option" do
it "should be passed correctly" do
run_binary(args: ["-rcn", %({"ids":$ARGS.positional}), "--args", "1", "2", "3"]) do |output|
output.should eq %({"ids":["1","2","3"]}\n)
end
end
end
describe "with the --jsonargs option" do
it "should be passed correctly" do
run_binary(args: ["-rcn", %({"ids":$ARGS.positional}), "--jsonargs", "1", "2", "3"]) do |output|
output.should eq %({"ids":[1,2,3]}\n)
end
end
end
describe "when there is a jq error" do
it "should return the error and correct exit code" do
run_binary(input: ARRAY_JSON_OBJECT, args: [".names | .[] | .name"], success: false) do |_, _, error|
error.should eq %(jq: error (at :0): Cannot index number with string "name"\n)
end
end
end
describe "with an invalid input format" do
it "should return the error and correct exit code" do
run_binary(input: SIMPLE_JSON_OBJECT, args: ["-i", "foo"], success: false) do |_, _, error|
error.should eq %(Invalid input format: 'foo'\n)
end
end
end
describe "with an invalid output format" do
it "should return the error and correct exit code" do
run_binary(input: SIMPLE_JSON_OBJECT, args: ["-o", "foo"], success: false) do |_, _, error|
error.should eq %(Invalid output format: 'foo'\n)
end
end
end
end
================================================
FILE: spec/processor_spec.cr
================================================
require "./spec_helper"
describe OQ::Processor do
describe "custom IOs" do
it "works with \"STDIN\" input" do
input_io = IO::Memory.new %({"name":"Jim"})
output_io = IO::Memory.new
OQ::Processor.new.process input_args: [".name"], input: input_io, output: output_io
output_io.to_s.should eq %("Jim"\n)
end
it "works with custom error output" do
input_io = IO::Memory.new %({"name:"Jim"})
output_io = IO::Memory.new
error_io = IO::Memory.new
expect_raises RuntimeError do
OQ::Processor.new.process input_args: [".name"], input: input_io, output: output_io, error: error_io
end
output_io.to_s.should be_empty
error_io.to_s.should contain "parse error: Invalid numeric literal at line 1, column 12\n"
end
describe "file input" do
it "single file" do
output_io = IO::Memory.new
OQ::Processor.new.process input_args: [".", "-c", "spec/assets/data1.json"], output: output_io
output_io.to_s.should eq %({"name":"Jim"}\n)
end
it "single file, standard input IO" do
input_io = IO::Memory.new
output_io = IO::Memory.new
OQ::Processor.new.process input_args: [".", "-c", "spec/assets/data1.json"], input: input_io, output: output_io
output_io.to_s.should eq %({"name":"Jim"}\n)
end
it "multiple file" do
output_io = IO::Memory.new
OQ::Processor.new.process input_args: [".", "-c", "spec/assets/data1.json", "spec/assets/data2.json"], output: output_io
output_io.to_s.should eq %({"name":"Jim"}\n{"name":"Bob"}\n)
end
it "multiple files and --slurp" do
output_io = IO::Memory.new
OQ::Processor.new.process input_args: [".", "-c", "-s", "spec/assets/data1.json", "spec/assets/data2.json"], output: output_io
output_io.to_s.should eq %([{"name":"Jim"},{"name":"Bob"}]\n)
end
end
end
end
================================================
FILE: spec/spec_helper.cr
================================================
require "spec"
require "../src/oq"
# Runs the binary with the given *name* and *args*.
def run_binary(input : String | Process::Redirect | Nil = nil, name : String = "bin/oq", args : Array(String) = [] of String, *, success : Bool = true, file = __FILE__, line = __LINE__, & : String, Process::Status, String -> Nil)
buffer_io = IO::Memory.new
error_io = IO::Memory.new
input_io = IO::Memory.new
if input.is_a? Process::Redirect
input_io = input
else
input_io << input if input
input_io = input_io.rewind
end
status = Process.run(name, args, output: buffer_io, input: input_io, error: error_io)
if success
status.success?.should be_true, file: file, line: line, failure_message: error_io.to_s
else
status.success?.should_not be_true, file: file, line: line, failure_message: error_io.to_s
end
yield buffer_io.to_s, status, error_io.to_s
end
================================================
FILE: src/converters/json.cr
================================================
# Converter for the `OQ::Format::JSON` format.
module OQ::Converters::JSON
def self.deserialize(input : IO, output : IO) : Nil
IO.copy input, output
end
def self.serialize(input : IO, output : IO) : Nil
IO.copy input, output
end
end
================================================
FILE: src/converters/processor_aware.cr
================================================
# :nodoc:
#
# Denotes a converter exposes the related `OQ::Processor`
# instance in order to read configuration options off of it.
module OQ::Converters::ProcessorAware
macro extended
class_property! processor : OQ::Processor
end
end
================================================
FILE: src/converters/simple_yaml.cr
================================================
require "./yaml"
# Converter for the `OQ::Format::SimpleYAML` format.
module OQ::Converters::SimpleYAML
extend OQ::Converters::YAML
extend self
# ameba:disable Metrics/CyclomaticComplexity
def deserialize(input : IO, output : IO) : Nil
yaml = ::YAML::PullParser.new(input)
json = ::JSON::Builder.new(output)
yaml.read_stream do
loop do
case yaml.kind
when .document_start?
json.start_document
when .document_end?
json.end_document
yaml.read_next
break
when .scalar?
string = yaml.value
if json.next_is_object_key?
json.scalar(string)
else
scalar = ::YAML::Schema::Core.parse_scalar(yaml)
case scalar
when Nil
json.scalar(scalar)
when Bool
json.scalar(scalar)
when Int64
json.scalar(scalar)
when Float64
json.scalar(scalar)
else
json.scalar(string)
end
end
when .sequence_start?
json.start_array
when .sequence_end?
json.end_array
when .mapping_start?
json.start_object
when .mapping_end?
json.end_object
end
yaml.read_next
end
end
end
end
================================================
FILE: src/converters/xml.cr
================================================
# Converter for the `OQ::Format::XML` format.
module OQ::Converters::XML
extend OQ::Converters::ProcessorAware
def self.deserialize(input : IO, output : IO) : Nil
builder = ::JSON::Builder.new output
xml = ::XML::Reader.new input
# Set reader to first element
xml.read
# Raise an error if the document is invalid and could not be read
raise ::XML::Error.new LibXML.xmlGetLastError if xml.node_type.none?
builder.document do
builder.object do
# Skip non element nodes, i.e. the prolog or DOCTYPE, etc.
until xml.node_type.element?
xml.read
end
process_element_node xml.expand, builder
end
end
end
private def self.process_element_node(node : ::XML::Node, builder : ::JSON::Builder) : Nil
# If the node doesn't have nested elements nor attributes nor a namespace (with --xmlns); just emit a scalar value
if self.scalar_node? node
return builder.field self.normalize_node_name(node), get_node_value node
end
# Otherwise process the node as a key/value pair
builder.field self.normalize_node_name node do
builder.object do
process_children node, builder
end
end
end
private def self.process_array_node(name : String, children : Array(::XML::Node), builder : ::JSON::Builder) : Nil
builder.field name do
builder.array do
children.each do |node|
# If the node doesn't have nested elements nor attributes nor a namespace (with --xmlns); just emit a scalar value
if self.scalar_node? node
builder.scalar get_node_value node
else
# Otherwise process the node within an object
builder.object do
process_children node, builder
end
end
end
end
end
end
private def self.process_children(node : ::XML::Node, builder : ::JSON::Builder) : Nil
# Process node attributes
node.attributes.each do |attr|
builder.field "@#{attr.name}", attr.content
end
# Include attributes for namespaces defined on this node
# TODO: Make this the default behavior in oq 2.x
if self.processor.xmlns?
node.namespace_definitions.each do |ns|
builder.field "@#{self.normalize_namespace_prefix ns}", ns.href
end
end
# Determine how to process a node's children
node.children.group_by(&->normalize_node_name(::XML::Node)).each do |name, children|
# Skip non significant whitespace; Skip mixed character input
if children.first.text? && has_nested_elements?(node)
# Only emit text content if there is only one child
if children.size == 1
builder.field "#text", children.first.content
end
next
end
# Array
if children.size > 1 || self.processor.xml_forced_arrays.includes? name
process_array_node name, children, builder
else
if children.first.text?
# node content in attribute object
builder.field "#text", children.first.content
else
# Element
process_element_node children.first, builder
end
end
end
end
private def self.has_nested_elements?(node : ::XML::Node) : Bool
node.children.any? { |child| !child.text? && !child.cdata? }
end
# TODO: Make checking for namespaces the default behavior in oq 2.x
private def self.scalar_node?(node : ::XML::Node) : Bool
!self.has_nested_elements?(node) && node.attributes.empty? && ((self.processor.xmlns? && node.namespace_definitions.empty?) || !self.processor.xmlns?)
end
private def self.get_node_value(node : ::XML::Node) : String?
node.children.empty? ? nil : node.children.first.content
end
private def self.normalize_node_name(node : ::XML::Node) : String
return node.name unless namespace = node.namespace
(prefix = (self.processor.xml_namespaces[namespace.href]? || namespace.prefix).presence) ? "#{prefix}:#{node.name}" : node.name
end
private def self.normalize_namespace_prefix(namespace : ::XML::Namespace) : String
(prefix = (self.processor.xml_namespaces[namespace.href]? || namespace.prefix).presence) ? "xmlns:#{prefix}" : "xmlns"
end
def self.serialize(input : IO, output : IO) : Nil
json = ::JSON::PullParser.new input
builder = ::XML::Builder.new output
builder.indent = ((self.processor.tab? ? "\t" : " ")*self.processor.indent)
builder.start_document "1.0", "UTF-8" if self.processor.xml_prolog?
if root = self.processor.xml_root.presence
builder.start_element root
end
loop do
emit builder, json
break if json.kind.eof?
end
if self.processor.xml_root.presence
builder.end_element
end
builder.end_document if self.processor.xml_prolog?
builder.flush unless self.processor.xml_prolog?
end
private def self.emit(builder : ::XML::Builder, json : ::JSON::PullParser, key : String? = nil, array_key : String? = nil) : Nil
case json.kind
when .null? then json.read_null
when .string?, .int?, .float?, .bool? then builder.text get_value json
when .begin_object? then handle_object builder, json, key, array_key
when .begin_array? then handle_array builder, json, key, array_key
else
nil
end
end
private def self.handle_object(builder : ::XML::Builder, json : ::JSON::PullParser, key : String? = nil, array_key : String? = nil) : Nil
json.read_object do |k|
if k.starts_with?('@')
builder.attribute k.lchop('@'), get_value json
elsif k.starts_with?('!')
builder.element k.lchop('!') do
builder.cdata get_value json
end
elsif json.kind.begin_array? || k == "#text"
emit builder, json, k, k
else
builder.element k do
emit builder, json, k
end
end
end
end
private def self.handle_array(builder : ::XML::Builder, json : ::JSON::PullParser, key : String? = nil, array_key : String? = nil) : Nil
json.read_begin_array
array_key = array_key || self.processor.xml_item
if json.kind.end_array?
# If the array is empty don't emit anything
else
until json.kind.end_array?
builder.element array_key do
emit builder, json, key
end
end
end
json.read_end_array
end
private def self.get_value(json : ::JSON::PullParser) : String
case json.kind
when .string? then json.read_string
when .int? then json.read_int
when .float? then json.read_float
when .bool? then json.read_bool
when .null? then json.read_null
else
""
end.to_s
end
end
================================================
FILE: src/converters/yaml.cr
================================================
# Converter for the `OQ::Format::YAML` format.
module OQ::Converters::YAML
extend self
def deserialize(input : IO, output : IO) : Nil
::YAML.parse(input).to_json output
end
# ameba:disable Metrics/CyclomaticComplexity
def serialize(input : IO, output : IO) : Nil
json = ::JSON::PullParser.new input
yaml = ::YAML::Builder.new output
# Return early is there is no JSON to be read.
return if json.kind.eof?
yaml.stream do
yaml.document do
loop do
case json.kind
when .null?
yaml.scalar nil
when .bool?
yaml.scalar json.bool_value
when .int?
yaml.scalar json.int_value
when .float?
yaml.scalar json.float_value
when .string?
if ::YAML::Schema::Core.reserved_string? json.string_value
yaml.scalar json.string_value, style: :double_quoted
else
yaml.scalar json.string_value
end
when .begin_array?
yaml.start_sequence
when .end_array?
yaml.end_sequence
when .begin_object?
yaml.start_mapping
when .end_object?
yaml.end_mapping
when .eof?
break
end
json.read_next
end
end
end
end
end
================================================
FILE: src/oq.cr
================================================
require "json"
require "xml"
require "yaml"
require "./converters/*"
# A performant, and portable jq wrapper that facilitates the consumption and output of formats other than JSON; using jq filters to transform the data.
module OQ
VERSION = "1.3.5"
# The support formats that can be converted to/from.
enum Format
# The [JSON](https://www.json.org/) format.
JSON
# Same as `YAML`, but does not support [anchors or aliases](https://yaml.org/spec/1.2/spec.html#id2765878);
# thus allowing for the input conversion to be streamed, reducing the memory usage for large inputs.
SimpleYAML
# The [XML](https://en.wikipedia.org/wiki/XML) format.
#
# NOTE: Conversion to and from `JSON` uses [this](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html) spec.
XML
# The [YAML](https://yaml.org/) format.
YAML
# Returns the list of supported formats.
def self.to_s(io : IO) : Nil
self.names.join(io, ", ") { |str, join_io| str.downcase join_io }
end
# Maps a given format to its converter.
def converter(processor : OQ::Processor)
case self
in .json? then OQ::Converters::JSON
in .simple_yaml? then OQ::Converters::SimpleYAML
in .xml? then OQ::Converters::XML
in .yaml? then OQ::Converters::YAML
end.tap { |converter| converter.processor = processor if converter.is_a? OQ::Converters::ProcessorAware }
end
end
# Handles the logic of converting the input format (if needed),
# processing it via [jq](https://stedolan.github.io/jq/),
# and converting the output format (if needed).
#
# ```
# require "oq"
#
# # This could be any `IO`, e.g. an `HTTP` request body, etc.
# input_io = IO::Memory.new %({"name":"Jim"})
#
# # Create a processor, specifying that we want the output format to be `YAML`.
# processor = OQ::Processor.new output_format: :yaml
#
# File.open("./out.yml", "w") do |file|
# # Process the data using our custom input and output IOs.
# # The first argument represents the input arguments;
# # i.e. the filter and/or any other arguments that should be passed to `jq`.
# processor.process ["."], input: input_io, output: file
# end
# ```
class Processor
# The format that the input data is in.
property input_format : Format
# The format that the output should be transcoded into.
property output_format : Format
# The root of the XML document when transcoding to XML.
property xml_root : String
# If the XML prolog should be emitted.
property? xml_prolog : Bool
# The name for XML array elements without keys.
property xml_item : String
# The number of spaces to use for indentation.
property indent : Int32
# If a tab for each indentation level instead of spaces.
property? tab : Bool
# Do not read any input, using `null` as the singular input value.
property? null : Bool
# If XML namespaces should be parsed as well.
# TODO: Remove this in oq 2.0 as it'll becomethe default.
property? xmlns : Bool
# Mapping to namespace aliases to their related namespace.
protected getter xml_namespaces = Hash(String, String).new
# Set of elements who should be force expanded to an array.
protected getter xml_forced_arrays = Set(String).new
# The args that'll be passed to `jq`.
@args : Array(String) = [] of String
# Keep a reference to the created temp files in order to delete them later.
@tmp_files = Set(File).new
def initialize(
@input_format : Format = Format::JSON,
@output_format : Format = Format::JSON,
@xml_root : String = "root",
@xml_prolog : Bool = true,
@xml_item : String = "item",
@indent : Int32 = 2,
@tab : Bool = false,
@null : Bool = false,
@xmlns : Bool = false,
)
end
@[Deprecated("Use `Processor#tab?` instead.")]
def tab : Bool
self.tab?
end
@[Deprecated("Use `Processor#xml_prolog?` instead.")]
def xml_prolog : Bool
self.xml_prolog?
end
# Adds the provided *value* to the internal args array.
def add_arg(value : String) : Nil
@args << value
end
def add_xml_namespace(prefix : String, href : String) : Nil
@xml_namespaces[href] = prefix
end
def add_forced_array(name : String) : Nil
xml_forced_arrays << name
end
# Consumes `#input_format` data from the provided *input* `IO`, along with any *input_args*.
# The data is then converted to `JSON`, passed to `jq`, and then converted to `#output_format` while being written to the *output* `IO`.
# Any errors are written to the *error* `IO`.
def process(input_args : Array(String) = ARGV, input : IO = ARGF, output : IO = STDOUT, error : IO = STDERR) : Nil
# Register an at_exit handler to cleanup temp files.
at_exit { @tmp_files.each &.delete }
# Parse out --rawfile, --argfile, --slurpfile,-f/--from-file, and -L before processing additional args
# since these options use a file that should not be used as input.
self.consume_file_args input_args, "--rawfile", "--argfile", "--slurpfile"
self.consume_file_args input_args, "-f", "--from-file", "-L", count: 1
# Also parse out --arg, and --argjson as they may include identifiers that also exist as a directory/file
# which would result in incorrect arg extraction.
self.consume_file_args input_args, "--arg", "--argjson"
# Extract `jq` arguments from `ARGV`.
self.extract_args input_args, output
# The --xml-namespace-alias option must be used with the --xmlns option.
# TODO: Remove this in oq 2.x
raise ArgumentError.new "The `--xml-namespace-alias` option must be used with the `--xmlns` option." if !@xmlns && !@xml_namespaces.empty?
# Replace the *input* with a fake `ARGF` `IO` to handle both file and `IO` inputs in case `ARGV` is not being used for the input arguments.
#
# If using `null` input, set the input to an empty memory `IO` to essentially consume nothing.
input = @null ? IO::Memory.new : IO::ARGF.new input_args, input
input_read, input_write = IO.pipe
output_read, output_write = IO.pipe
channel = Channel(Bool | Exception).new
# If the input format is not JSON and there is more than 1 file in ARGV,
# convert each file to JSON from the `#input_format` and save it to a temp file.
# Then replace ARGV with the temp files.
if !@input_format.json? && input_args.size > 1
input_args.replace(input_args.map do |file_name|
File.tempfile ".#{File.basename file_name}" do |tmp_file|
File.open file_name do |file|
@input_format.converter(self).deserialize file, tmp_file
end
end
.tap { |tf| @tmp_files << tf }
.path
end)
# Conversion has already been completed by this point, so reset input format back to JSON.
@input_format = :json
end
spawn do
@input_format.converter(self).deserialize input, input_write
input_write.close
channel.send true
rescue ex
input_write.close
channel.send ex
end
spawn do
output_write.close
@output_format.converter(self).serialize output_read, output
channel.send true
rescue ex
channel.send ex
end
run = Process.run(
"jq",
@args,
input: input_read,
output: output_write,
error: error
)
unless run.success?
# Raise this to represent a jq error.
# jq writes its errors directly to the *error* IO so no need to include a message.
raise RuntimeError.new
end
2.times do
case v = channel.receive
when Exception then raise v
end
end
end
# Parses the *input_args*, extracting `jq` arguments while leaving files
private def extract_args(input_args : Array(String), output : IO) : Nil
# Add color option if *output* is a tty
# and the output format is JSON
# (Since it will go straight to *output* and not converted)
input_args.unshift "-C" if output.tty? && @output_format.json? && !input_args.includes? "-C"
# If the -C option was explicitly included
# and the output format is not JSON;
# remove it from *input_args* to prevent
# conversion errors
input_args.delete("-C") if !@output_format.json?
# If there are any files within the *input_args*, ignore "." as it's both a valid file and filter
idx = if first_file_idx = input_args.index { |a| a != "." && File.exists? a }
# extract everything else
first_file_idx - 1
else
# otherwise just take it all
-1
end
@args.concat input_args.delete_at 0..idx
end
# Extracts *arg_name* from the provided *input_args* if it exists;
# concatenating the result to the internal arg array.
private def consume_file_arg(input_args : Array(String), arg_name : String, count : Int32 = 2) : Nil
input_args.index(arg_name).try { |idx| @args.concat input_args.delete_at idx..(idx + count) }
end
private def consume_file_args(input_args : Array(String), *arg_names : String, count : Int32 = 2) : Nil
arg_names.each { |name| consume_file_arg input_args, name, count }
end
end
end
================================================
FILE: src/oq_cli.cr
================================================
require "option_parser"
require "./oq"
processor = OQ::Processor.new
OptionParser.parse do |parser|
parser.banner = "Usage: oq [--help] [oq-arguments] [jq-arguments] jq_filter [file [files...]]"
parser.on("-h", "--help", "Show this help message.") do
output = IO::Memory.new
version = IO::Memory.new
Process.run("jq", ["-h"], output: output)
Process.run("jq", ["--version"], output: version)
puts "oq version: #{OQ::VERSION}, jq version: #{version}", parser, output.to_s.lines.map(&.gsub('\t', " ")).tap(&.delete_at(0..1)).join('\n')
exit
end
parser.on("-V", "--version", "Returns the current versions of oq and jq.") do
output = IO::Memory.new
Process.run("jq", ["--version"], output: output)
puts "jq: #{output}", "oq: #{OQ::VERSION}"
exit
end
parser.on("-i FORMAT", "--input FORMAT", "Format of the input data. Supported formats: #{OQ::Format}") { |format| (f = OQ::Format.parse?(format)) ? processor.input_format = f : abort "Invalid input format: '#{format}'" }
parser.on("-o FORMAT", "--output FORMAT", "Format of the output data. Supported formats: #{OQ::Format}") { |format| (f = OQ::Format.parse?(format)) ? processor.output_format = f : abort "Invalid output format: '#{format}'" }
parser.on("--indent NUMBER", "Use the given number of spaces for indentation (JSON/XML only).") { |n| processor.indent = n.to_i; processor.add_arg "--indent"; processor.add_arg n }
parser.on("--tab", "Use a tab for each indentation level instead of two spaces.") { processor.tab = true; processor.add_arg "--tab" }
parser.on("-n", "--null-input", "Don't read any input at all, running the filter once using `null` as the input.") { processor.null = true; processor.add_arg "--null-input" }
parser.on("--no-prolog", "Whether the XML prolog should be emitted if converting to XML.") { processor.xml_prolog = false }
parser.on("--xml-item NAME", "The name for XML array elements without keys.") { |i| processor.xml_item = i }
parser.on("--xmlns", "If XML namespaces should be parsed. NOTE: This will become the default in oq 2.x.") { processor.xmlns = true }
parser.on("--xml-force-array NAME", "Forces an element with the provided name to be parsed as an array even if it only contains one item.") { |n| processor.add_forced_array n }
parser.on("--xml-namespace-alias ALIAS", "Value should be in the form of: `key=namespace`. Elements within the provided namespace are normalized to the provided key. NOTE: Requires the `--xmlns` option to be passed as well.") { |a| k, v = a.split('=', 2); processor.add_xml_namespace k, v }
parser.on("--xml-root ROOT", "Name of the root XML element if converting to XML.") { |r| processor.xml_root = r }
parser.invalid_option { }
end
begin
processor.process
rescue ex : RuntimeError
# ignore jq errors as it writes directly to error output.
exit 1
rescue ex
abort "oq error: #{ex.message}"
end