[ { "path": ".coveralls.yml", "content": "service_name: travis-ci" }, { "path": ".gitattributes", "content": "# Don't mess with my CSV files\n*.csv binary" }, { "path": ".github/ISSUE_TEMPLATE.md", "content": "> Please provide a general summary of the issue in the Issue Title above\n> fill out the headings below as applicable to the issue you are reporting,\n> deleting as appropriate but offering us as much detail as you can to help us resolve the issue\n\n### Expected Behaviour\n> What should happen?\n\n### Desired Behaviour (for improvement suggestions only)\n> if relevant include images or hyperlinks to other resources that clarify the enhancement you're seeking\n\n### Current Behaviour (for problems)\n> What currently happens that isn't expected behaviour?\n\n### Steps to Reproduce (for problems)\n> Provide a link to a live example, or an unambiguous set of steps to reproduce this bug. Include code to reproduce, if relevant\n1.\n2.\n3.\n4.\n\n### Your Environment\n> Include as many relevant details about the environment you experienced the bug in - this will help us resolve the bug more expediently\n* Environment name and version (e.g. Chrome 39, node.js 5.4):\n* Operating System and version (desktop or mobile):\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "This PR fixes #\n\nChanges proposed in this pull request:\n\n-\n-\n-\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n- package-ecosystem: bundler\n directory: \"/\"\n schedule:\n interval: daily\n open-pull-requests-limit: 10\n- package-ecosystem: github-actions\n directory: \"/\"\n schedule:\n interval: weekly\n" }, { "path": ".github/workflows/push.yml", "content": "name: CI\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\njobs:\n appraisal:\n name: Ruby ${{ matrix.ruby-version }} / Rails ${{ matrix.activesupport-version }}\n runs-on: ubuntu-latest\n strategy:\n matrix:\n ruby-version: ['2.5', '2.6', '2.7', '3.0', '3.1', '3.2', '3.3', '3.4', '4.0']\n activesupport-version:\n - activesupport_5.2\n - activesupport_6.0\n - activesupport_6.1\n - activesupport_7.0\n - activesupport_7.1\n - activesupport_7.2\n exclude:\n - ruby-version: '2.5'\n activesupport-version: activesupport_7.0\n - ruby-version: '2.6'\n activesupport-version: activesupport_7.0\n - ruby-version: '2.5'\n activesupport-version: activesupport_7.1\n - ruby-version: '2.6'\n activesupport-version: activesupport_7.1\n - ruby-version: '2.5'\n activesupport-version: activesupport_7.2\n - ruby-version: '2.6'\n activesupport-version: activesupport_7.2\n - ruby-version: '2.7'\n activesupport-version: activesupport_7.2\n - ruby-version: '3.0'\n activesupport-version: activesupport_7.2\n fail-fast: false\n\n env:\n BUNDLE_GEMFILE: gemfiles/${{ matrix.activesupport-version }}.gemfile\n\n steps:\n - uses: actions/checkout@v4\n - uses: ruby/setup-ruby@v1\n with:\n bundler-cache: true\n ruby-version: ${{ matrix.ruby-version }}\n - name: Install dependencies\n run: bundle install\n - name: Run the tests\n run: bundle exec rake\n\n lint:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - uses: ruby/setup-ruby@v1\n with:\n bundler-cache: true\n ruby-version: \"4.0\"\n - name: Install dependencies\n run: bundle install\n - name: Run the tests\n run: bundle exec standardrb\n" }, { "path": ".gitignore", "content": "*.gem\n*.rbc\n.bundle\n.config\n.yardoc\nGemfile.lock\nInstalledFiles\n_yardoc\ncoverage\ndoc/\nlib/bundler/man\npkg\nrdoc\nspec/reports\ntest/tmp\ntest/version_tmp\ntmp\ncoverage/\n/.rspec\n\n.idea\n.DS_Store\nfeatures/csvw_validation_tests.feature\nfeatures/fixtures/csvw\n\nbin/run-csvw-tests\n\ncsvlint-earl.ttl\n.byebug_history\n\ngemfiles/*.lock\n" }, { "path": ".pre-commit-hooks.yaml", "content": "- id: csvlint\n name: csvlint\n entry: csvlint\n language: ruby\n files: \\.csv$\n" }, { "path": ".ruby-version", "content": "4.0.1\n" }, { "path": ".standard_todo.yml", "content": "# Auto generated files with errors to ignore.\n# Remove from this list as you refactor files.\n---\nignore:\n- features/support/load_tests.rb:\n - Security/Open\n- lib/csvlint/csvw/column.rb:\n - Style/TernaryParentheses\n- lib/csvlint/csvw/date_format.rb:\n - Lint/MixedRegexpCaptureTypes\n- lib/csvlint/csvw/number_format.rb:\n - Style/SlicingWithRange\n - Style/IdenticalConditionalBranches\n- lib/csvlint/csvw/property_checker.rb:\n - Performance/InefficientHashSearch\n - Naming/VariableName\n - Style/SlicingWithRange\n - Security/Open\n - Lint/BooleanSymbol\n- lib/csvlint/csvw/table_group.rb:\n - Style/OptionalArguments\n- lib/csvlint/field.rb:\n - Naming/VariableName\n- lib/csvlint/schema.rb:\n - Security/Open\n - Style/SlicingWithRange\n- lib/csvlint/validate.rb:\n - Performance/Count\n - Lint/BooleanSymbol\n - Naming/VariableName\n - Security/Open\n - Lint/NonLocalExitFromIterator\n- lib/csvlint/schema.rb:\n - Lint/UselessRescue\n- lib/csvlint/validate.rb:\n - Lint/UselessRescue\n- lib/csvlint/cli.rb:\n - Style/SafeNavigation\n" }, { "path": "Appraisals", "content": "# After a new entry: `bundle exec appraisal install`\n# Add an entry in `.github/workflows/push.yml`'s file\n\nappraise \"activesupport_5.2\" do\n gem \"activesupport\", \"~> 5.2.0\"\nend\n\nappraise \"activesupport_6.0\" do\n gem \"activesupport\", \"~> 6.0.0\"\nend\n\nappraise \"activesupport_6.1\" do\n gem \"activesupport\", \"~> 6.1.0\"\nend\n\nappraise \"activesupport_7.0\" do\n gem \"activesupport\", \"~> 7.0.0\"\nend\n\nappraise \"activesupport_7.1\" do\n gem \"activesupport\", \"~> 7.1.0\"\nend\n\nappraise \"activesupport_7.2\" do\n gem \"activesupport\", \"~> 7.2.0\"\nend\n" }, { "path": "CHANGELOG.md", "content": "# Change Log\n\n## [v1.2.0](https://github.com/data-liberation-front/csvlint.rb/tree/v1.2.0) (2023-02-27)\n\n[Full Changelog](https://github.com/data-liberation-front/csvlint.rb/compare/v1.1.0...v1.2.0)\n\n**Closed issues:**\n\n- Pre-commit integration [\\#275](https://github.com/Data-Liberation-Front/csvlint.rb/issues/275)\n\n**Merged pull requests:**\n\n- Pre commit hook [\\#276](https://github.com/Data-Liberation-Front/csvlint.rb/pull/276) ([jrottenberg](https://github.com/jrottenberg))\n\n## [v1.1.0](https://github.com/data-liberation-front/csvlint.rb/tree/v1.1.0) (2022-12-28)\n\n[Full Changelog](https://github.com/data-liberation-front/csvlint.rb/compare/v1.0.0...v1.1.0)\n\n**Closed issues:**\n\n- Requires ruby \\< 3.2 [\\#272](https://github.com/Data-Liberation-Front/csvlint.rb/issues/272)\n- Release a new version [\\#244](https://github.com/Data-Liberation-Front/csvlint.rb/issues/244)\n\n**Merged pull requests:**\n\n- bump version to 1.1.0 [\\#274](https://github.com/Data-Liberation-Front/csvlint.rb/pull/274) ([Floppy](https://github.com/Floppy))\n- Add support for Ruby 3.2 [\\#273](https://github.com/Data-Liberation-Front/csvlint.rb/pull/273) ([Floppy](https://github.com/Floppy))\n- fix lint error [\\#271](https://github.com/Data-Liberation-Front/csvlint.rb/pull/271) ([youpy](https://github.com/youpy))\n- optimize validation with regular expression [\\#270](https://github.com/Data-Liberation-Front/csvlint.rb/pull/270) ([youpy](https://github.com/youpy))\n- Bump actions/checkout from 2 to 3 [\\#269](https://github.com/Data-Liberation-Front/csvlint.rb/pull/269) ([dependabot[bot]](https://github.com/apps/dependabot))\n- Add GitHub Actions to Dependabot [\\#267](https://github.com/Data-Liberation-Front/csvlint.rb/pull/267) ([petergoldstein](https://github.com/petergoldstein))\n- Lint with standardrb [\\#266](https://github.com/Data-Liberation-Front/csvlint.rb/pull/266) ([Floppy](https://github.com/Floppy))\n- Add Dockerfile and notes for usage on MS Windows. [\\#243](https://github.com/Data-Liberation-Front/csvlint.rb/pull/243) ([jespertp-systematic](https://github.com/jespertp-systematic))\n\n## [v1.0.0](https://github.com/Data-Liberation-Front/csvlint.rb/tree/v1.0.0) (2022-07-13)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.4.0...v1.0.0)\n\nSupport Ruby 3.x, and DROPPED support for Ruby 2.4 - that's why the major version bump. That and this has been around long enough that it really shouldn't be on a zero version any more :)\n\n## What's Changed\n\n- Don't patch CSV#init_converters for ruby 2.5 compatibility by @rbmrclo in \n\n- correct typos in README by @erikj in \n- add info about your PATH by @ftrotter in \n- Remove tests on deprecated ruby versions < 2.3 by @Floppy in \n- Drop mime-types gem dependency by @ohbarye in \n- remove specific version of net-http-persistent in gemspec by @kotaro0522 in \n- Replace colorize with rainbow to make licensing consistent. by @cobbr2 in \n- Update rdf requirement from < 2.0 to < 4.0 by @dependabot-preview in \n- Test on Ruby 2.5 and 2.6 by @Domon in \n- Fix load_from_json deprecation warnings. by @jezhiggins in \n- Fix csvw tests by @Floppy in \n- Test on Ruby 2.6 and 2.7 by @Floppy in \n- Create Dependabot config file by @dependabot-preview in \n- Include active_support/object to ensure this works in ruby 2.6 by @mseverini in \n- add CI workflow for github actions by @Floppy in \n- Enable and fix tests for Ruby 2.5 by @Floppy in \n- Support Ruby 2.6 by @Floppy in \n- Ruby 2.7 support by @Floppy in \n- Drop support for Ruby 2.4 by @Floppy in \n- Ruby 3.0 by @Floppy in \n\n## New Contributors\n\n- @rbmrclo made their first contribution in \n\n- @erikj made their first contribution in \n- @ftrotter made their first contribution in \n- @ohbarye made their first contribution in \n- @kotaro0522 made their first contribution in \n- @cobbr2 made their first contribution in \n- @dependabot-preview made their first contribution in \n- @Domon made their first contribution in \n- @mseverini made their first contribution in \n\n## [0.4.0](https://github.com/theodi/csvlint.rb/tree/0.4.0) (2017-xx-xx)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.3.3...0.4.0)\n\n- Support for Ruby 2.4\n - Ruby 2.4 improves detections of unclosed quotes\n- Support Rails ~> 5.0\n- Added `--werror` flag to command line, to treat warnings as errors\n- Deprecated `Schema#load_from_json` and replaced with `Schema#load_from_uri`. Method will be removed in 1.0.0.\n- Added `Schema#load_from_string` to load from a string instead of reading a URI\n\n**Closed issues:**\n\n- CLI doesn't handle filenames with spaces [\\#182](https://github.com/theodi/csvlint.rb/issues/182)\n\n## [0.3.3](https://github.com/theodi/csvlint.rb/tree/0.3.3) (2016-11-10)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.3.2...0.3.3)\n\n**Closed issues:**\n\n- testing issue alerts, sorry [\\#186](https://github.com/theodi/csvlint.rb/issues/186)\n\n**Merged pull requests:**\n\n- Add row + col to foreign key & duplicate key errors [\\#188](https://github.com/theodi/csvlint.rb/pull/188) ([nickzoic](https://github.com/nickzoic))\n- Trap-and-bin this [\\#185](https://github.com/theodi/csvlint.rb/pull/185) ([pikesley](https://github.com/pikesley))\n- csvw: common property names can be URLs [\\#181](https://github.com/theodi/csvlint.rb/pull/181) ([JeniT](https://github.com/JeniT))\n- force UTF-8 if encoding is ASCII-8BIT [\\#180](https://github.com/theodi/csvlint.rb/pull/180) ([JeniT](https://github.com/JeniT))\n\n## [0.3.2](https://github.com/theodi/csvlint.rb/tree/0.3.2) (2016-05-24)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.3.1...0.3.2)\n\n**Merged pull requests:**\n\n- Add schema errors to cli json [\\#184](https://github.com/theodi/csvlint.rb/pull/184) ([pezholio](https://github.com/pezholio))\n\n## [0.3.1](https://github.com/theodi/csvlint.rb/tree/0.3.1) (2016-05-23)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.3.0...0.3.1)\n\n**Closed issues:**\n\n- Error installing on Windows because of \\*escape\\_utils\\* dependency [\\#175](https://github.com/theodi/csvlint.rb/issues/175)\n\n**Merged pull requests:**\n\n- Add CLI option to output JSON [\\#183](https://github.com/theodi/csvlint.rb/pull/183) ([pezholio](https://github.com/pezholio))\n\n## [0.3.0](https://github.com/theodi/csvlint.rb/tree/0.3.0) (2016-01-12)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.2.6...0.3.0)\n\n**Merged pull requests:**\n\n- still increment current\\_line after invalid\\_encoding error [\\#174](https://github.com/theodi/csvlint.rb/pull/174) ([wjordan213](https://github.com/wjordan213))\n- Support for CSV on the Web transformations [\\#173](https://github.com/theodi/csvlint.rb/pull/173) ([JeniT](https://github.com/JeniT))\n\n## [0.2.6](https://github.com/theodi/csvlint.rb/tree/0.2.6) (2015-11-16)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.2.5...0.2.6)\n\n## [0.2.5](https://github.com/theodi/csvlint.rb/tree/0.2.5) (2015-11-16)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.2.4...0.2.5)\n\n**Merged pull requests:**\n\n- Use STDIN instead of ARGF [\\#169](https://github.com/theodi/csvlint.rb/pull/169) ([pezholio](https://github.com/pezholio))\n\n## [0.2.4](https://github.com/theodi/csvlint.rb/tree/0.2.4) (2015-10-20)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.2.3...0.2.4)\n\n**Merged pull requests:**\n\n- Fixes for CLI [\\#164](https://github.com/theodi/csvlint.rb/pull/164) ([pezholio](https://github.com/pezholio))\n\n## [0.2.3](https://github.com/theodi/csvlint.rb/tree/0.2.3) (2015-10-20)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.2.2...0.2.3)\n\n**Closed issues:**\n\n- Include field name with error [\\#161](https://github.com/theodi/csvlint.rb/issues/161)\n- Refactor the binary [\\#150](https://github.com/theodi/csvlint.rb/issues/150)\n\n**Merged pull requests:**\n\n- Refactor CLI [\\#163](https://github.com/theodi/csvlint.rb/pull/163) ([pezholio](https://github.com/pezholio))\n- Update schema file example to clarify type [\\#162](https://github.com/theodi/csvlint.rb/pull/162) ([wachunga](https://github.com/wachunga))\n\n## [0.2.2](https://github.com/theodi/csvlint.rb/tree/0.2.2) (2015-10-09)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.2.1...0.2.2)\n\n**Closed issues:**\n\n- Eliminate some date and time formats \\(for speed\\) [\\#105](https://github.com/theodi/csvlint.rb/issues/105)\n\n**Merged pull requests:**\n\n- Check characters in validate\\_line method [\\#160](https://github.com/theodi/csvlint.rb/pull/160) ([pezholio](https://github.com/pezholio))\n- Further optimisations [\\#159](https://github.com/theodi/csvlint.rb/pull/159) ([pezholio](https://github.com/pezholio))\n- More optimizations after \\#157 [\\#158](https://github.com/theodi/csvlint.rb/pull/158) ([jpmckinney](https://github.com/jpmckinney))\n- Memoize the result of CSV\\#encode\\_re [\\#157](https://github.com/theodi/csvlint.rb/pull/157) ([jpmckinney](https://github.com/jpmckinney))\n- Don't pass leading string to parse\\_line [\\#155](https://github.com/theodi/csvlint.rb/pull/155) ([pezholio](https://github.com/pezholio))\n\n## [0.2.1](https://github.com/theodi/csvlint.rb/tree/0.2.1) (2015-10-07)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.2.0...0.2.1)\n\n**Implemented enhancements:**\n\n- Get total rows number about the CSV file that was validated [\\#143](https://github.com/theodi/csvlint.rb/issues/143)\n\n**Closed issues:**\n\n- Optimization: Stream CSV [\\#122](https://github.com/theodi/csvlint.rb/issues/122)\n\n**Merged pull requests:**\n\n- Add `row\\_count` method [\\#153](https://github.com/theodi/csvlint.rb/pull/153) ([pezholio](https://github.com/pezholio))\n- Streaming validation [\\#146](https://github.com/theodi/csvlint.rb/pull/146) ([pezholio](https://github.com/pezholio))\n\n## [0.2.0](https://github.com/theodi/csvlint.rb/tree/0.2.0) (2015-10-05)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.1.4...0.2.0)\n\n**Closed issues:**\n\n- CSV on the web support [\\#141](https://github.com/theodi/csvlint.rb/issues/141)\n\n**Merged pull requests:**\n\n- Recover from `ArgumentError`s when attempting to locate a schema and detect bad schema when JSON is malformed [\\#152](https://github.com/theodi/csvlint.rb/pull/152) ([pezholio](https://github.com/pezholio))\n- Catch errors if link headers are don't have particular values [\\#151](https://github.com/theodi/csvlint.rb/pull/151) ([pezholio](https://github.com/pezholio))\n- Rescue excel warning [\\#149](https://github.com/theodi/csvlint.rb/pull/149) ([quadrophobiac](https://github.com/quadrophobiac))\n- CSVW-based validation! [\\#142](https://github.com/theodi/csvlint.rb/pull/142) ([JeniT](https://github.com/JeniT))\n\n## [0.1.4](https://github.com/theodi/csvlint.rb/tree/0.1.4) (2015-08-06)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.1.3...0.1.4)\n\n**Merged pull requests:**\n\n- change made to the constraint parameter in order that it is more cons… [\\#140](https://github.com/theodi/csvlint.rb/pull/140) ([quadrophobiac](https://github.com/quadrophobiac))\n\n## [0.1.3](https://github.com/theodi/csvlint.rb/tree/0.1.3) (2015-07-24)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.1.2...0.1.3)\n\n**Merged pull requests:**\n\n- Error reporting schema expanded test suite [\\#138](https://github.com/theodi/csvlint.rb/pull/138) ([quadrophobiac](https://github.com/quadrophobiac))\n- Validate header size improvement [\\#137](https://github.com/theodi/csvlint.rb/pull/137) ([adamc00](https://github.com/adamc00))\n- Invalid schema [\\#132](https://github.com/theodi/csvlint.rb/pull/132) ([bcouston](https://github.com/bcouston))\n\n## [0.1.2](https://github.com/theodi/csvlint.rb/tree/0.1.2) (2015-07-15)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.1.1...0.1.2)\n\n**Closed issues:**\n\n- When an encoding error is thrown the line content is put into the column field in the error object [\\#131](https://github.com/theodi/csvlint.rb/issues/131)\n\n**Merged pull requests:**\n\n- Catch invalid URIs [\\#133](https://github.com/theodi/csvlint.rb/pull/133) ([pezholio](https://github.com/pezholio))\n- Emit a warning when the CSV header does not match the supplied schema [\\#127](https://github.com/theodi/csvlint.rb/pull/127) ([adamc00](https://github.com/adamc00))\n\n## [0.1.1](https://github.com/theodi/csvlint.rb/tree/0.1.1) (2015-07-13)\n\n[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.1.0...0.1.1)\n\n**Closed issues:**\n\n- Add Command Line Support [\\#128](https://github.com/theodi/csvlint.rb/issues/128)\n- BUG: Incorrect inconsistent\\_values error on numeric columns [\\#106](https://github.com/theodi/csvlint.rb/issues/106)\n\n**Merged pull requests:**\n\n- Fixes line content incorrectly being put into the row column field when there is an encoding error. [\\#130](https://github.com/theodi/csvlint.rb/pull/130) ([glacier](https://github.com/glacier))\n- Add command line help [\\#129](https://github.com/theodi/csvlint.rb/pull/129) ([pezholio](https://github.com/pezholio))\n- Remove stray q character. [\\#125](https://github.com/theodi/csvlint.rb/pull/125) ([adamc00](https://github.com/adamc00))\n- csvlint utility can take arguments to specify a schema and pp errors [\\#124](https://github.com/theodi/csvlint.rb/pull/124) ([adamc00](https://github.com/adamc00))\n- Fixed warning - use expect\\( \\) rather than .should [\\#123](https://github.com/theodi/csvlint.rb/pull/123) ([jezhiggins](https://github.com/jezhiggins))\n- Fixed spelling mistake [\\#121](https://github.com/theodi/csvlint.rb/pull/121) ([jezhiggins](https://github.com/jezhiggins))\n- Avoid using \\#blank? if unnecessary [\\#120](https://github.com/theodi/csvlint.rb/pull/120) ([jpmckinney](https://github.com/jpmckinney))\n- eliminate some date and time formats, related \\#105 [\\#119](https://github.com/theodi/csvlint.rb/pull/119) ([jpmckinney](https://github.com/jpmckinney))\n- Match another CSV error about line endings [\\#118](https://github.com/theodi/csvlint.rb/pull/118) ([jpmckinney](https://github.com/jpmckinney))\n- fixed typo mistake in README [\\#117](https://github.com/theodi/csvlint.rb/pull/117) ([railsfactory-kumaresan](https://github.com/railsfactory-kumaresan))\n- Integrate @jpmickinney's build\\_formats improvements [\\#112](https://github.com/theodi/csvlint.rb/pull/112) ([Floppy](https://github.com/Floppy))\n- make limit\\_lines into a non-dialect option [\\#110](https://github.com/theodi/csvlint.rb/pull/110) ([Floppy](https://github.com/Floppy))\n- fix coveralls stats [\\#109](https://github.com/theodi/csvlint.rb/pull/109) ([Floppy](https://github.com/Floppy))\n- Limit lines [\\#101](https://github.com/theodi/csvlint.rb/pull/101) ([Hoedic](https://github.com/Hoedic))\n\n## [0.1.0](https://github.com/theodi/csvlint.rb/tree/0.1.0) (2014-11-27)\n\n**Implemented enhancements:**\n\n- Blank values shouldn't count as inconsistencies [\\#90](https://github.com/theodi/csvlint.rb/issues/90)\n- Make sure we don't check schema column count and ragged row count together [\\#66](https://github.com/theodi/csvlint.rb/issues/66)\n- Include the failed constraints in error message when doing field validation [\\#64](https://github.com/theodi/csvlint.rb/issues/64)\n- Include the column value in error message when field validation fails [\\#63](https://github.com/theodi/csvlint.rb/issues/63)\n- Expose optional JSON table schema fields [\\#55](https://github.com/theodi/csvlint.rb/issues/55)\n- Ensure header rows are properly handled and validated [\\#48](https://github.com/theodi/csvlint.rb/issues/48)\n- Support zipped CSV? [\\#30](https://github.com/theodi/csvlint.rb/issues/30)\n- Improve feedback on inconsistent values [\\#29](https://github.com/theodi/csvlint.rb/issues/29)\n- Reported error positions are not massively useful [\\#15](https://github.com/theodi/csvlint.rb/issues/15)\n\n**Fixed bugs:**\n\n- undefined method `\\[\\]' for nil:NilClass from fetch\\_error [\\#71](https://github.com/theodi/csvlint.rb/issues/71)\n- Inconsistent column bases [\\#69](https://github.com/theodi/csvlint.rb/issues/69)\n- Improve error handling in Schema loading [\\#42](https://github.com/theodi/csvlint.rb/issues/42)\n- Recover from some line ending problems [\\#41](https://github.com/theodi/csvlint.rb/issues/41)\n- Inconsistent values due to number format differences [\\#32](https://github.com/theodi/csvlint.rb/issues/32)\n- New lines in quoted fields are valid [\\#31](https://github.com/theodi/csvlint.rb/issues/31)\n- Wrongly reporting incorrect file extension [\\#23](https://github.com/theodi/csvlint.rb/issues/23)\n- Incorrect extension reported when URL has query options at the end [\\#14](https://github.com/theodi/csvlint.rb/issues/14)\n\n**Closed issues:**\n\n- Get gem continuously deploying [\\#93](https://github.com/theodi/csvlint.rb/issues/93)\n- Publish on rubygems.org [\\#92](https://github.com/theodi/csvlint.rb/issues/92)\n- Duplicate column names [\\#87](https://github.com/theodi/csvlint.rb/issues/87)\n- Return code is always 0 \\(except when it isn't\\) [\\#85](https://github.com/theodi/csvlint.rb/issues/85)\n- Can't pipe data to csvlint [\\#84](https://github.com/theodi/csvlint.rb/issues/84)\n- They have some validator running if someone wants to inspect it for \"inspiration\" [\\#27](https://github.com/theodi/csvlint.rb/issues/27)\n- Allow CSV parsing options to be configured as a parameter [\\#6](https://github.com/theodi/csvlint.rb/issues/6)\n- Use explicit CSV parsing options [\\#5](https://github.com/theodi/csvlint.rb/issues/5)\n- Improving encoding detection [\\#2](https://github.com/theodi/csvlint.rb/issues/2)\n\n**Merged pull requests:**\n\n- Speed up \\#build\\_formats \\(changes its API\\) [\\#103](https://github.com/theodi/csvlint.rb/pull/103) ([jpmckinney](https://github.com/jpmckinney))\n- Continuously deploy gem [\\#102](https://github.com/theodi/csvlint.rb/pull/102) ([pezholio](https://github.com/pezholio))\n- Make csvlint way faster [\\#99](https://github.com/theodi/csvlint.rb/pull/99) ([jpmckinney](https://github.com/jpmckinney))\n- Update README.md [\\#98](https://github.com/theodi/csvlint.rb/pull/98) ([rmalecky](https://github.com/rmalecky))\n- Undeclared header error [\\#95](https://github.com/theodi/csvlint.rb/pull/95) ([Floppy](https://github.com/Floppy))\n- Blank values shouldn't count as inconsistencies [\\#91](https://github.com/theodi/csvlint.rb/pull/91) ([pezholio](https://github.com/pezholio))\n- Use `reject` instead of `delete\\_if` [\\#89](https://github.com/theodi/csvlint.rb/pull/89) ([pezholio](https://github.com/pezholio))\n- Raise a warning if a title row is found [\\#88](https://github.com/theodi/csvlint.rb/pull/88) ([pezholio](https://github.com/pezholio))\n- Improve executable [\\#86](https://github.com/theodi/csvlint.rb/pull/86) ([pezholio](https://github.com/pezholio))\n- Feature undeclared header [\\#83](https://github.com/theodi/csvlint.rb/pull/83) ([ldodds](https://github.com/ldodds))\n- Support xsd:integer [\\#82](https://github.com/theodi/csvlint.rb/pull/82) ([ldodds](https://github.com/ldodds))\n- Downgrade header errors [\\#81](https://github.com/theodi/csvlint.rb/pull/81) ([ldodds](https://github.com/ldodds))\n- Go home, pry [\\#78](https://github.com/theodi/csvlint.rb/pull/78) ([pikesley](https://github.com/pikesley))\n- Use type validations to check consistency [\\#77](https://github.com/theodi/csvlint.rb/pull/77) ([pezholio](https://github.com/pezholio))\n- Add data accessor [\\#76](https://github.com/theodi/csvlint.rb/pull/76) ([Floppy](https://github.com/Floppy))\n- Add failed constraints to schema errors [\\#75](https://github.com/theodi/csvlint.rb/pull/75) ([ldodds](https://github.com/ldodds))\n- Only perform ragged row check if there's no schema [\\#74](https://github.com/theodi/csvlint.rb/pull/74) ([ldodds](https://github.com/ldodds))\n- Handle tempfiles [\\#73](https://github.com/theodi/csvlint.rb/pull/73) ([pezholio](https://github.com/pezholio))\n- Catch errors if regex doesn't match [\\#72](https://github.com/theodi/csvlint.rb/pull/72) ([pezholio](https://github.com/pezholio))\n- Inconsistent column base [\\#70](https://github.com/theodi/csvlint.rb/pull/70) ([ldodds](https://github.com/ldodds))\n- include column name in :header\\_name message [\\#68](https://github.com/theodi/csvlint.rb/pull/68) ([Floppy](https://github.com/Floppy))\n- Record default dialect [\\#67](https://github.com/theodi/csvlint.rb/pull/67) ([pezholio](https://github.com/pezholio))\n- Schema validation message improvements [\\#65](https://github.com/theodi/csvlint.rb/pull/65) ([Floppy](https://github.com/Floppy))\n- Fix ignore empty fields [\\#62](https://github.com/theodi/csvlint.rb/pull/62) ([ldodds](https://github.com/ldodds))\n- Create stub schema from existing CSV file [\\#61](https://github.com/theodi/csvlint.rb/pull/61) ([ldodds](https://github.com/ldodds))\n- Validate dates [\\#59](https://github.com/theodi/csvlint.rb/pull/59) ([ldodds](https://github.com/ldodds))\n- add schema access from validator [\\#58](https://github.com/theodi/csvlint.rb/pull/58) ([Floppy](https://github.com/Floppy))\n- Allow schema and fields to have title and description [\\#57](https://github.com/theodi/csvlint.rb/pull/57) ([ldodds](https://github.com/ldodds))\n- Feature min max ranges [\\#56](https://github.com/theodi/csvlint.rb/pull/56) ([ldodds](https://github.com/ldodds))\n- Check header without schema [\\#54](https://github.com/theodi/csvlint.rb/pull/54) ([ldodds](https://github.com/ldodds))\n- Validate types [\\#53](https://github.com/theodi/csvlint.rb/pull/53) ([pikesley](https://github.com/pikesley))\n- Added open\\_uri\\_redirections to allow HTTP/HTTPS transfers [\\#52](https://github.com/theodi/csvlint.rb/pull/52) ([ldodds](https://github.com/ldodds))\n- Added docs on CSV options and header error/warning messages [\\#51](https://github.com/theodi/csvlint.rb/pull/51) ([ldodds](https://github.com/ldodds))\n- Feature header validation [\\#50](https://github.com/theodi/csvlint.rb/pull/50) ([ldodds](https://github.com/ldodds))\n- Handle unique columns [\\#49](https://github.com/theodi/csvlint.rb/pull/49) ([pikesley](https://github.com/pikesley))\n- Validate all the fields [\\#47](https://github.com/theodi/csvlint.rb/pull/47) ([ldodds](https://github.com/ldodds))\n- Tolerate incomplete schemas [\\#46](https://github.com/theodi/csvlint.rb/pull/46) ([ldodds](https://github.com/ldodds))\n- Add accessor for line breaks [\\#45](https://github.com/theodi/csvlint.rb/pull/45) ([Floppy](https://github.com/Floppy))\n- update README for info messages and new error types [\\#44](https://github.com/theodi/csvlint.rb/pull/44) ([Floppy](https://github.com/Floppy))\n- Info messages for line breaks [\\#43](https://github.com/theodi/csvlint.rb/pull/43) ([Floppy](https://github.com/Floppy))\n- Add category to messages [\\#40](https://github.com/theodi/csvlint.rb/pull/40) ([ldodds](https://github.com/ldodds))\n- Badges [\\#39](https://github.com/theodi/csvlint.rb/pull/39) ([pikesley](https://github.com/pikesley))\n- Generic field validation using JSON Table Schema [\\#38](https://github.com/theodi/csvlint.rb/pull/38) ([ldodds](https://github.com/ldodds))\n- Feature validate strings and files [\\#37](https://github.com/theodi/csvlint.rb/pull/37) ([ldodds](https://github.com/ldodds))\n- Support reporting of column number in errors [\\#36](https://github.com/theodi/csvlint.rb/pull/36) ([ldodds](https://github.com/ldodds))\n- Fix up casing of keys in CSV DDF options [\\#35](https://github.com/theodi/csvlint.rb/pull/35) ([ldodds](https://github.com/ldodds))\n- Add errors for incorrect newlines [\\#34](https://github.com/theodi/csvlint.rb/pull/34) ([pezholio](https://github.com/pezholio))\n- Change from parsing CSV line by line to using CSV.new and trapping errors [\\#33](https://github.com/theodi/csvlint.rb/pull/33) ([ldodds](https://github.com/ldodds))\n- Improved the README, tweaked LICENSE [\\#28](https://github.com/theodi/csvlint.rb/pull/28) ([ldodds](https://github.com/ldodds))\n- Handle 404s [\\#26](https://github.com/theodi/csvlint.rb/pull/26) ([pezholio](https://github.com/pezholio))\n- Create more fine-grained errors and warnings for content type issues [\\#25](https://github.com/theodi/csvlint.rb/pull/25) ([ldodds](https://github.com/ldodds))\n- Report trailing empty rows as an error. Previously threw exception [\\#24](https://github.com/theodi/csvlint.rb/pull/24) ([ldodds](https://github.com/ldodds))\n- Simplify the guessing of column types [\\#22](https://github.com/theodi/csvlint.rb/pull/22) ([ldodds](https://github.com/ldodds))\n- Class-ify error messages [\\#21](https://github.com/theodi/csvlint.rb/pull/21) ([pezholio](https://github.com/pezholio))\n- Error extracts [\\#20](https://github.com/theodi/csvlint.rb/pull/20) ([Floppy](https://github.com/Floppy))\n- Return headers [\\#19](https://github.com/theodi/csvlint.rb/pull/19) ([pezholio](https://github.com/pezholio))\n- Return a warning if no character set specified [\\#18](https://github.com/theodi/csvlint.rb/pull/18) ([pezholio](https://github.com/pezholio))\n- Ignore query params [\\#17](https://github.com/theodi/csvlint.rb/pull/17) ([Floppy](https://github.com/Floppy))\n- Add invalid\\_encoding error for invalid byte sequences [\\#16](https://github.com/theodi/csvlint.rb/pull/16) ([ldodds](https://github.com/ldodds))\n- Check inconsistent values [\\#13](https://github.com/theodi/csvlint.rb/pull/13) ([pezholio](https://github.com/pezholio))\n- Add CSV dialect options [\\#11](https://github.com/theodi/csvlint.rb/pull/11) ([pezholio](https://github.com/pezholio))\n- Return warning if extension doesn't match content type [\\#10](https://github.com/theodi/csvlint.rb/pull/10) ([pezholio](https://github.com/pezholio))\n- Return warnings for file extension [\\#8](https://github.com/theodi/csvlint.rb/pull/8) ([pezholio](https://github.com/pezholio))\n- Detect blank rows [\\#7](https://github.com/theodi/csvlint.rb/pull/7) ([pezholio](https://github.com/pezholio))\n- Detect bad content type [\\#3](https://github.com/theodi/csvlint.rb/pull/3) ([pezholio](https://github.com/pezholio))\n- Return information about CSV [\\#1](https://github.com/theodi/csvlint.rb/pull/1) ([pezholio](https://github.com/pezholio))\n\n\\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "## Code of Conduct\n\n### Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to making participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, gender identity and expression, level of experience,\nnationality, personal appearance, race, religion, or sexual identity and\norientation.\n\n### Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\nadvances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\n address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n### Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviors that they deem inappropriate,\nthreatening, offensive, or harmful.\n\n### Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces\nwhen an individual is representing the project or its community. Examples of\nrepresenting a project or community include using an official project e-mail\naddress, posting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event. Representation of a project may be\nfurther defined and clarified by project maintainers.\n\n### Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting the project team at [labs@theodi.org](mailto:labs@theodi.org). All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n### Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,\navailable at [http://contributor-covenant.org/version/1/4][version]\n\n[homepage]: http://contributor-covenant.org\n[version]: http://contributor-covenant.org/version/1/4/\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to CSVlint.rb\n\nThe CSVlint library is open source, and contributions are gratefully accepted!\nDetails on how to contribute are below. By participating in this project, you agree to abide by our [Code of Conduct](https://github.com/theodi/csvlint.rb/blob/CODE_OF_CONDUCT.md).\n\nBefore you start coding, please reach out to us either on our [gitter channel](https://gitter.im/theodi/toolbox) or by tagging a repository administrator on the issue ticket you are interested in contributing towards to indicate your interest in helping.\n\nIf this is your first time contributing to the ODI’s codebase you will need to [create a fork of this repository](https://help.github.com/articles/fork-a-repo/).\n\nConsult our [Getting Started Guide](https://github.com/theodi/toolbox/wiki/Developers-Guide:-Getting-Started) (if necessary) and then follow the [readme instructions](https://github.com/theodi/csvlint.rb/blob/master/README.md#development) to get your Development environment running locally\n\nEnsure that the [tests](https://github.com/theodi/csvlint.rb/blob/master/README.md#tests) pass before working on your contribution\n\n## Code Review Process\n\nAll contributions to the codebase - whether fork or pull request - will be reviewed per the below criteria.\nTo increase your chances of your push being accepted please be aware of the following\n- Write [well formed commit messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)\n- Follow our [style guide recommendations](https://github.com/theodi/toolbox/blob/README.md#code-style-guide)\n- Write tests for all changes (additions or refactors of existing code).\n- Of the github integrations we use two will be utilised to check appraise your contribution. In order of priority these are\n - Travis ensures that all tests (existing and additions) pass\n - Travis/Coveralls ensures that overall test coverage for lines of code meets a certain threshold. If this metric dips below what it previously was for the repository you’re pushing to then your PR will be rejected\n - Gemnasium ensures dependencies are up to date\n- Once your PR is published and passes the above checks a repository administrator will review your contribution. Where appropriate comments may be provided and amendments suggested before your PR is merged into Master.\n- Once your PR is accepted you will be granted push access to the repository you have contributed to! Congratulations on joining our community, you’ll no longer need to work from forks.\n\nIf you make a contribution to another repository in the Toolbox you will be expected to repeat this process. Read more about that [here](https://github.com/theodi/toolbox/blob/master/README.md#push-access).\n\n## Code Style Guide\n\nWe follow the same code style conventions as detailed in Github’s [Ruby Style Guide](https://github.com/github/rubocop-github/blob/master/STYLEGUIDE.md)\n" }, { "path": "Dockerfile", "content": "FROM ruby:2.5.8-buster\r\n\r\n# throw errors if Gemfile has been modified since Gemfile.lock\r\nRUN bundle config --global frozen 1\r\n\r\nWORKDIR /usr/src/app\r\n\r\nENV LANG C.UTF-8\r\n\r\nCOPY ./lib/csvlint/version.rb ./lib/csvlint/\r\nCOPY csvlint.gemspec Gemfile Gemfile.lock ./\r\nRUN bundle install\r\n\r\nCOPY ./ ./\r\n\r\nCMD [\"./bin/csvlint\"]\r\n" }, { "path": "Gemfile", "content": "source \"https://rubygems.org\"\n\n# Specify your gem's dependencies in csvlint.rb.gemspec\ngemspec\n" }, { "path": "LICENSE.md", "content": "##Copyright (c) 2014 The Open Data Institute\n\n#MIT License\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." }, { "path": "README.md", "content": "[![Build Status](https://img.shields.io/github/workflow/status/Data-Liberation-Front/csvlint.rb/CI/main)](https://travis-ci.org/theodi/csvlint.rb)\n[![Dependency Status](https://img.shields.io/librariesio/github/Data-Liberation-Front/csvlint.rb)](https://libraries.io/github/Data-Liberation-Front/csvlint.rb)\n[![Coverage Status](http://img.shields.io/coveralls/Data-Liberation-Front/csvlint.rb.svg)](https://coveralls.io/r/Data-Liberation-Front/csvlint.rb)\n[![License](http://img.shields.io/:license-mit-blue.svg)](http://theodi.mit-license.org)\n[![Badges](http://img.shields.io/:badges-5/5-ff6799.svg)](https://github.com/pikesley/badger)\n\n# CSV Lint\n\nA ruby gem to support validating CSV files to check their syntax and contents. You can either use this gem within your own Ruby code, or as a standalone command line application\n\n## Summary of features\n\n* Validation that checks the structural formatting of a CSV file\n* Validation of a delimiter-separated values (dsv) file accesible via URL, File, or an IO-style object (e.g. StringIO)\n* Validation against [CSV dialects](http://dataprotocols.org/csv-dialect/)\n* Validation against multiple schema standards; [JSON Table Schema](https://github.com/theodi/csvlint.rb/blob/master/README.md#json-table-schema-support) and [CSV on the Web](https://github.com/theodi/csvlint.rb/blob/master/README.md#csv-on-the-web-validation-support)\n\n## Development\n\n`ruby version 4.0`\n\n### Tests\n\nThe codebase includes both rspec and cucumber tests, which can be run together using:\n\n $ rake\n\nor separately:\n\n $ rake spec\n $ rake features\n\nWhen the cucumber tests are first run, a script will create tests based on the latest version of the [CSV on the Web test suite](http://w3c.github.io/csvw/tests/), including creating a local cache of the test files. This requires an internet connection and some patience. Following that download, the tests will run locally; there's also a batch script:\n\n $ bin/run-csvw-tests\n\nwhich will run the tests from the command line.\n\nIf you need to refresh the CSV on the Web tests:\n\n $ rm bin/run-csvw-tests\n $ rm features/csvw_validation_tests.feature\n $ rm -r features/fixtures/csvw\n\nand then run the cucumber tests again or:\n\n $ ruby features/support/load_tests.rb\n\n\n## Installation\n\nAdd this line to your application's Gemfile:\n\n gem 'csvlint'\n\nAnd then execute:\n\n $ bundle\n\nOr install it yourself as:\n\n $ gem install csvlint\n\n## Usage\n\nYou can either use this gem within your own Ruby code, or as a standalone command line application\n\n## On the command line\n\nAfter installing the gem, you can validate a CSV on the command line like so:\n\n\tcsvlint myfile.csv\n\nYou may need to add the gem exectuable directory to your path, by adding '/usr/local/lib/ruby/gems/2.6.0/bin'\nor whatever your version is, to your .bash_profile PATH entry. [like so](https://stackoverflow.com/questions/2392293/ruby-gems-returns-command-not-found)\n\nYou will then see the validation result, together with any warnings or errors e.g.\n\n```\nmyfile.csv is INVALID\n1. blank_rows. Row: 3\n1. title_row.\n2. inconsistent_values. Column: 14\n```\n\nYou can also optionally pass a schema file like so:\n\n\tcsvlint myfile.csv --schema=schema.json\n\n## Via pre-commit\n\nAdd to your .pre-commit-config.yaml file :\n\n```\nrepos: # `pre-commit autoupdate` to get latest available tags\n\n - repo: https://github.com/Data-Liberation-Front/csvlint.rb\n rev: v1.2.0\n hooks:\n - id: csvlint\n```\n\n`pre-commit install` to enable it on your repository.\n\nTo force a manual run of [pre-commit](https://pre-commit.com/) use the command :\n\n```\npre-commit run -a\n```\n\n## In your own Ruby code\n\nCurrently the gem supports retrieving a CSV accessible from a URL, File, or an IO-style object (e.g. StringIO)\n\n\trequire 'csvlint'\n\n\tvalidator = Csvlint::Validator.new( \"http://example.org/data.csv\" )\n\tvalidator = Csvlint::Validator.new( File.new(\"/path/to/my/data.csv\" ))\n\tvalidator = Csvlint::Validator.new( StringIO.new( my_data_in_a_string ) )\n\nWhen validating from a URL the range of errors and warnings is wider as the library will also check HTTP headers for\nbest practices\n\n\t#invoke the validation\n\tvalidator.validate\n\n\t#check validation status\n\tvalidator.valid?\n\n\t#access array of errors, each is an Csvlint::ErrorMessage object\n\tvalidator.errors\n\n\t#access array of warnings\n\tvalidator.warnings\n\n\t#access array of information messages\n\tvalidator.info_messages\n\n\t#get some information about the CSV file that was validated\n\tvalidator.encoding\n\tvalidator.content_type\n\tvalidator.extension\n\tvalidator.row_count\n\n\t#retrieve HTTP headers from request\n\tvalidator.headers\n\n## Controlling CSV Parsing\n\nThe validator supports configuration of the [CSV Dialect](http://dataprotocols.org/csv-dialect/) used in a data file. This is specified by\npassing a dialect hash to the constructor:\n\n dialect = {\n \t\"header\" => true,\n \t\"delimiter\" => \",\"\n }\n\tvalidator = Csvlint::Validator.new( \"http://example.org/data.csv\", dialect )\n\nThe options should be a Hash that conforms to the [CSV Dialect](http://dataprotocols.org/csv-dialect/) JSON structure.\n\nWhile these options configure the parser to correctly process the file, the validator will still raise errors or warnings for CSV\nstructure that it considers to be invalid, e.g. a missing header or different delimiters.\n\nNote that the parser will also check for a `header` parameter on the `Content-Type` header returned when fetching a remote CSV file. As\nspecified in [RFC 4180](http://www.ietf.org/rfc/rfc4180.txt) the values for this can be `present` and `absent`, e.g:\n\n\tContent-Type: text/csv; header=present\n\n## Error Reporting\n\nThe validator provides feedback on a validation result using instances of `Csvlint::ErrorMessage`. Errors are divided into errors, warnings and information\nmessages. A validation attempt is successful if there are no errors.\n\nMessages provide context including:\n\n* `category` has a symbol that indicates the category or error/warning: `:structure` (well-formedness issues), `:schema` (schema validation), `:context` (publishing metadata, e.g. content type)\n* `type` has a symbol that indicates the type of error or warning being reported\n* `row` holds the line number of the problem\n* `column` holds the column number of the issue\n* `content` holds the contents of the row that generated the error or warning\n\n## Errors\n\nThe following types of error can be reported:\n\n* `:wrong_content_type` -- content type is not `text/csv`\n* `:ragged_rows` -- row has a different number of columns (than the first row in the file)\n* `:blank_rows` -- completely empty row, e.g. blank line or a line where all column values are empty\n* `:invalid_encoding` -- encoding error when parsing row, e.g. because of invalid characters\n* `:not_found` -- HTTP 404 error when retrieving the data\n* `:stray_quote` -- missing or stray quote\n* `:unclosed_quote` -- unclosed quoted field\n* `:whitespace` -- a quoted column has leading or trailing whitespace\n* `:line_breaks` -- line breaks were inconsistent or incorrectly specified\n\n## Warnings\n\nThe following types of warning can be reported:\n\n* `:no_encoding` -- the `Content-Type` header returned in the HTTP request does not have a `charset` parameter\n* `:encoding` -- the character set is not UTF-8\n* `:no_content_type` -- file is being served without a `Content-Type` header\n* `:excel` -- no `Content-Type` header and the file extension is `.xls`\n* `:check_options` -- CSV file appears to contain only a single column\n* `:inconsistent_values` -- inconsistent values in the same column. Reported if <90% of values seem to have same data type (either numeric or alphanumeric including punctuation)\n* `:empty_column_name` -- a column in the CSV header has an empty name\n* `:duplicate_column_name` -- a column in the CSV header has a duplicate name\n* `:title_row` -- if there appears to be a title field in the first row of the CSV\n\n## Information Messages\n\nThere are also information messages available:\n\n* `:nonrfc_line_breaks` -- uses non-CRLF line breaks, so doesn't conform to RFC4180.\n* `:assumed_header` -- the validator has assumed that a header is present\n\n## Schema Validation\n\nThe library supports validating data against a schema. A schema configuration can be provided as a Hash or parsed from JSON. The structure currently\nfollows JSON Table Schema with some extensions and rudinmentary [CSV on the Web Metadata](http://www.w3.org/TR/tabular-metadata/).\n\nAn example JSON Table Schema schema file is:\n\n\t{\n\t\t\"fields\": [\n\t\t\t{\n\t\t\t\t\"name\": \"id\",\n\t\t\t\t\"constraints\": {\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"type\": \"http://www.w3.org/TR/xmlschema-2/#integer\"\n\t\t\t\t}\n\t\t\t},\n\t\t\t{\n\t\t\t\t\"name\": \"price\",\n\t\t\t\t\"constraints\": {\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"minLength\": 1\n\t\t\t\t}\n\t\t\t},\n\t\t\t{\n\t\t\t\t\"name\": \"postcode\",\n\t\t\t\t\"constraints\": {\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"pattern\": \"[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}\"\n\t\t\t\t}\n\t\t\t}\n\t\t]\n\t}\n\nAn equivalent CSV on the Web Metadata file is:\n\n\t{\n\t\t\"@context\": \"http://www.w3.org/ns/csvw\",\n\t\t\"url\": \"http://example.com/example1.csv\",\n\t\t\"tableSchema\": {\n\t\t\t\"columns\": [\n\t\t\t\t{\n\t\t\t\t\t\"name\": \"id\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"datatype\": { \"base\": \"integer\" }\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"name\": \"price\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"datatype\": { \"base\": \"string\", \"minLength\": 1 }\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"name\": \"postcode\",\n\t\t\t\t\t\"required\": true\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}\n\nParsing and validating with a schema (of either kind):\n\n\tschema = Csvlint::Schema.load_from_json(uri)\n\tvalidator = Csvlint::Validator.new( \"http://example.org/data.csv\", nil, schema )\n\n### CSV on the Web Validation Support\n\nThis gem passes all the validation tests in the [official CSV on the Web test suite](http://w3c.github.io/csvw/tests/) (though there might still be errors or parts of the [CSV on the Web standard](http://www.w3.org/TR/tabular-metadata/) that aren't tested by that test suite).\n\n### JSON Table Schema Support\n\nSupported constraints:\n\n* `required` -- there must be a value for this field in every row\n* `unique` -- the values in every row should be unique\n* `minLength` -- minimum number of characters in the value\n* `maxLength` -- maximum number of characters in the value\n* `pattern` -- values must match the provided regular expression\n* `type` -- specifies an XML Schema data type. Values of the column must be a valid value for that type\n* `minimum` -- specify a minimum range for values, the value will be parsed as specified by `type`\n* `maximum` -- specify a maximum range for values, the value will be parsed as specified by `type`\n* `datePattern` -- specify a `strftime` compatible date pattern to be used when parsing date values and min/max constraints\n\nSupported data types (this is still a work in progress):\n\n* String -- `http://www.w3.org/2001/XMLSchema#string` (effectively a no-op)\n* Integer -- `http://www.w3.org/2001/XMLSchema#integer` or `http://www.w3.org/2001/XMLSchema#int`\n* Float -- `http://www.w3.org/2001/XMLSchema#float`\n* Double -- `http://www.w3.org/2001/XMLSchema#double`\n* URI -- `http://www.w3.org/2001/XMLSchema#anyURI`\n* Boolean -- `http://www.w3.org/2001/XMLSchema#boolean`\n* Non Positive Integer -- `http://www.w3.org/2001/XMLSchema#nonPositiveInteger`\n* Positive Integer -- `http://www.w3.org/2001/XMLSchema#positiveInteger`\n* Non Negative Integer -- `http://www.w3.org/2001/XMLSchema#nonNegativeInteger`\n* Negative Integer -- `http://www.w3.org/2001/XMLSchema#negativeInteger`\n* Date -- `http://www.w3.org/2001/XMLSchema#date`\n* Date Time -- `http://www.w3.org/2001/XMLSchema#dateTime`\n* Year -- `http://www.w3.org/2001/XMLSchema#gYear`\n* Year Month -- `http://www.w3.org/2001/XMLSchema#gYearMonth`\n* Time -- `http://www.w3.org/2001/XMLSchema#time`\n\nUse of an unknown data type will result in the column failing to validate.\n\nSchema validation provides some additional types of error and warning messages:\n\n* `:missing_value` (error) -- a column marked as `required` in the schema has no value\n* `:min_length` (error) -- a column with a `minLength` constraint has a value that is too short\n* `:max_length` (error) -- a column with a `maxLength` constraint has a value that is too long\n* `:pattern` (error) -- a column with a `pattern` constraint has a value that doesn't match the regular expression\n* `:malformed_header` (warning) -- the header in the CSV doesn't match the schema\n* `:missing_column` (warning) -- a row in the CSV file has a missing column, that is specified in the schema. This is a warning only, as it may be legitimate\n* `:extra_column` (warning) -- a row in the CSV file has extra column.\n* `:unique` (error) -- a column with a `unique` constraint contains non-unique values\n* `:below_minimum` (error) -- a column with a `minimum` constraint contains a value that is below the minimum\n* `:above_maximum` (error) -- a column with a `maximum` constraint contains a value that is above the maximum\n\n### Other validation options\n\nYou can also provide an optional options hash as the fourth argument to Validator#new. Supported options are:\n\n* :limit_lines -- only check this number of lines of the CSV file. Good for a quick check on huge files.\n\n```\noptions = {\n limit_lines: 100\n}\nvalidator = Csvlint::Validator.new( \"http://example.org/data.csv\", nil, nil, options )\n```\n\n* :lambda -- Pass a block of code to be called when each line is validated, this will give you access to the `Validator` object. For example, this will return the current line number for every line validated:\n\n```\n options = {\n lambda: ->(validator) { puts validator.current_line }\n }\n validator = Csvlint::Validator.new( \"http://example.org/data.csv\", nil, nil, options )\n => 1\n 2\n 3\n 4\n .....\n```\n" }, { "path": "Rakefile", "content": "require \"bundler/gem_tasks\"\n\n$:.unshift File.join(File.dirname(__FILE__), \"lib\")\n\nrequire \"rubygems\"\nrequire \"cucumber\"\nrequire \"cucumber/rake/task\"\nrequire \"coveralls/rake/task\"\nrequire \"rspec/core/rake_task\"\n\nRSpec::Core::RakeTask.new(:spec)\nCoveralls::RakeTask.new\nCucumber::Rake::Task.new(:features) do |t|\n t.cucumber_opts = \"features --format pretty\"\nend\n\ntask default: [:spec, :features, \"coveralls:push\"]\n" }, { "path": "bin/create_schema", "content": "#!/usr/bin/env ruby\n$:.unshift File.join( File.dirname(__FILE__), \"..\", \"lib\")\n\nrequire 'csvlint'\n\nbegin\n puts ARGV[0]\n csv = CSV.new( URI.open(ARGV[0]) )\n\theaders = csv.shift\n\t\n\tname = File.basename( ARGV[0] )\n\tschema = {\n\t \"title\" => name,\n\t \"description\" => \"Auto generated schema for #{name}\",\n\t \"fields\" => []\n\t}\n\t\n\theaders.each do |name|\n\t schema[\"fields\"] << {\n\t \"name\" => name,\n\t \"title\" => \"\",\n\t \"description\" => \"\",\n\t \"constraints\" => {}\n\t }\n\tend\n\t\n\t$stdout.puts JSON.pretty_generate(schema)\nrescue => e\n puts e\n puts e.backtrace\n\tputs \"Unable to parse CSV file\"\nend\n" }, { "path": "bin/csvlint", "content": "#!/usr/bin/env ruby\n$:.unshift File.join( File.dirname(__FILE__), \"..\", \"lib\")\n\nrequire 'csvlint/cli'\n\nif ARGV == [\"help\"]\n Csvlint::Cli.start([\"help\"])\nelse\n Csvlint::Cli.start(ARGV.unshift(\"validate\"))\nend\n" }, { "path": "csvlint.gemspec", "content": "lib = File.expand_path(\"../lib\", __FILE__)\n$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)\nrequire \"csvlint/version\"\n\nGem::Specification.new do |spec|\n spec.name = \"csvlint\"\n spec.version = Csvlint::VERSION\n spec.authors = [\"pezholio\"]\n spec.email = [\"pezholio@gmail.com\"]\n spec.description = \"CSV Validator\"\n spec.summary = \"CSV Validator\"\n spec.homepage = \"https://github.com/theodi/csvlint.rb\"\n spec.license = \"MIT\"\n\n spec.files = `git ls-files`.split($/)\n spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }\n spec.require_paths = [\"lib\"]\n\n spec.required_ruby_version = [\">= 2.5\", \"< 4.1\"]\n\n spec.add_dependency \"csv\"\n spec.add_dependency \"rainbow\"\n spec.add_dependency \"open_uri_redirections\"\n spec.add_dependency \"activesupport\"\n spec.add_dependency \"addressable\"\n spec.add_dependency \"typhoeus\"\n spec.add_dependency \"escape_utils\"\n spec.add_dependency \"uri_template\"\n spec.add_dependency \"thor\"\n spec.add_dependency \"rack\"\n spec.add_dependency \"net-http-persistent\"\n spec.add_dependency \"mutex_m\" # For Ruby 3.4+\n\n spec.add_development_dependency \"bundler\", \">= 1.3\"\n spec.add_development_dependency \"rake\"\n spec.add_development_dependency \"cucumber\"\n spec.add_development_dependency \"simplecov\"\n spec.add_development_dependency \"simplecov-rcov\"\n spec.add_development_dependency \"spork\"\n spec.add_development_dependency \"webmock\"\n spec.add_development_dependency \"rspec\"\n spec.add_development_dependency \"rspec-pride\"\n spec.add_development_dependency \"rspec-expectations\"\n spec.add_development_dependency \"coveralls_reborn\"\n spec.add_development_dependency \"byebug\"\n spec.add_development_dependency \"github_changelog_generator\"\n spec.add_development_dependency \"aruba\"\n spec.add_development_dependency \"rdf\", \"< 4.0\"\n spec.add_development_dependency \"rdf-turtle\"\n spec.add_development_dependency \"standardrb\"\n spec.add_development_dependency \"appraisal\"\n spec.add_development_dependency \"benchmark\"\nend\n" }, { "path": "docker_notes_for_windows.txt", "content": "# Note that these commands are specific for a docker environment on MS Windows.\r\n\r\n# to generate Gemfile.lock file\r\ndocker run --rm -v %CD%:/usr/src/app -w /usr/src/app ruby:2.5 bundle install\r\n\r\n# to build docker image from source (the ending dot is significant)\r\ndocker build -t csvlint .\r\n\r\n# to run tests\r\ndocker run -it --rm csvlint rake\r\n\r\n# to run csvlint command line with a CSV file.\r\n# cd to the directory with the CSV file then\r\ndocker run -it --rm -v %CD%:/tmp csvlint ./bin/csvlint --dump-errors /tmp/file-to-lint.csv\r\n\r\n# to enter the linux container\r\ndocker run -it --rm -v %CD%:/tmp csvlint bash\r\n\r\n# to enter the ruby REPL\r\ndocker run -it --rm -v %CD%:/tmp csvlint irb\r\n" }, { "path": "features/check_format.feature", "content": "Feature: Check inconsistent formatting\n\n Scenario: Inconsistent formatting for integers\n Given I have a CSV with the following content:\n \"\"\"\n\"1\",\"2\",\"3\"\n\"Foo\",\"5\",\"6\"\n\"3\",\"2\",\"1\"\n\"3\",\"2\",\"1\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"inconsistent_values\"\n And that warning should have the column \"1\"\n \n Scenario: Inconsistent formatting for alpha fields\n Given I have a CSV with the following content:\n \"\"\"\n\"Foo\",\"Bar\",\"Baz\"\n\"Biz\",\"1\",\"Baff\"\n\"Boff\",\"Giff\",\"Goff\"\n\"Boff\",\"Giff\",\"Goff\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"inconsistent_values\"\n And that warning should have the column \"2\"\n\n Scenario: Inconsistent formatting for alphanumeric fields\n Given I have a CSV with the following content:\n \"\"\"\n\"Foo 123\",\"Bar\",\"Baz\"\n\"1\",\"Bar\",\"Baff\"\n\"Boff 432423\",\"Giff\",\"Goff\"\n\"Boff444\",\"Giff\",\"Goff\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"inconsistent_values\"\n And that warning should have the column \"1\"\n\n\n \n" }, { "path": "features/cli.feature", "content": "Feature: CSVlint CLI\n\n Scenario: Valid CSV from url\n Given I have a CSV with the following content:\n \"\"\"\n\"Foo\",\"Bar\",\"Baz\"\n\"1\",\"2\",\"3\"\n\"3\",\"2\",\"1\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I run `csvlint http://example.com/example1.csv`\n Then the output should contain \"http://example.com/example1.csv is VALID\"\n\n Scenario: Valid CSV from file\n When I run `csvlint ../../features/fixtures/valid.csv`\n Then the output should contain \"valid.csv is VALID\"\n\n # This is a hacky way of saying to run `cat features/fixtures/valid.csv | csvlint`\n Scenario: Valid CSV from pipe\n Given I have stubbed stdin to contain \"features/fixtures/valid.csv\"\n When I run `csvlint`\n Then the output should contain \"CSV is VALID\"\n\n Scenario: URL that 404s\n Given there is no file at the url \"http://example.com/example1.csv\"\n And there is no file at the url \"http://example.com/.well-known/csvm\"\n And there is no file at the url \"http://example.com/example1.csv-metadata.json\"\n And there is no file at the url \"http://example.com/csv-metadata.json\"\n When I run `csvlint http://example.com/example1.csv`\n Then the output should contain \"http://example.com/example1.csv is INVALID\"\n And the output should contain \"not_found\"\n\n Scenario: File doesn't exist\n When I run `csvlint ../../features/fixtures/non-existent-file.csv`\n Then the output should contain \"non-existent-file.csv not found\"\n\n Scenario: No file or URL specified\n Given I have stubbed stdin to contain nothing\n When I run `csvlint`\n Then the output should contain \"No CSV data to validate\"\n\n Scenario: No file or URL specified, but schema specified\n Given I have stubbed stdin to contain nothing\n And I have a schema with the following content:\n \"\"\"\n{\n \"fields\": [\n { \"name\": \"Name\", \"constraints\": { \"required\": true } },\n { \"name\": \"Id\", \"constraints\": { \"required\": true, \"minLength\": 1 } },\n { \"name\": \"Email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n And the schema is stored at the url \"http://example.com/schema.json\"\n When I run `csvlint --schema http://example.com/schema.json`\n Then the output should contain \"No CSV data to validate\"\n\n Scenario: Invalid CSV from url\n Given I have a CSV with the following content:\n \"\"\"\n \"Foo\",\t\"Bar\"\t,\t\"Baz\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I run `csvlint http://example.com/example1.csv`\n Then the output should contain \"http://example.com/example1.csv is INVALID\"\n And the output should contain \"whitespace\"\n\n Scenario: Invalid CSV from url with JSON\n Given I have a CSV with the following content:\n \"\"\"\n \"Foo\",\t\"Bar\"\t,\t\"Baz\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I run `csvlint http://example.com/example1.csv --json`\n Then the output should contain JSON\n And the JSON should have a state of \"invalid\"\n And the JSON should have 1 error\n And that error should have the \"type\" \"whitespace\"\n And that error should have the \"category\" \"structure\"\n And that error should have the \"row\" \"1\"\n\n Scenario: Specify schema\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\n{\n\t\"fields\": [\n { \"name\": \"Name\", \"constraints\": { \"required\": true } },\n { \"name\": \"Id\", \"constraints\": { \"required\": true, \"minLength\": 1 } },\n { \"name\": \"Email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n And the schema is stored at the url \"http://example.com/schema.json\"\n When I run `csvlint http://example.com/example1.csv --schema http://example.com/schema.json`\n Then the output should contain \"http://example.com/example1.csv is VALID\"\n\n Scenario: Schema errors\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\n{\n \"fields\": [\n { \"name\": \"Name\", \"constraints\": { \"required\": true } },\n { \"name\": \"Id\", \"constraints\": { \"required\": true, \"minLength\": 3 } },\n { \"name\": \"Email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n And the schema is stored at the url \"http://example.com/schema.json\"\n When I run `csvlint http://example.com/example1.csv --schema http://example.com/schema.json`\n Then the output should contain \"http://example.com/example1.csv is INVALID\"\n And the output should contain \"1. Id: min_length. Row: 2,2. 5\"\n And the output should contain \"1. malformed_header. Row: 1. Bob,1234,bob@example.org\"\n\n Scenario: Schema errors with JSON\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\n{\n \"fields\": [\n { \"name\": \"Name\", \"constraints\": { \"required\": true } },\n { \"name\": \"Id\", \"constraints\": { \"required\": true, \"minLength\": 3 } },\n { \"name\": \"Email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n And the schema is stored at the url \"http://example.com/schema.json\"\n When I run `csvlint http://example.com/example1.csv --schema http://example.com/schema.json --json`\n Then the output should contain JSON\n And the JSON should have a state of \"invalid\"\n And the JSON should have 1 error\n And error 1 should have the \"type\" \"min_length\"\n And error 1 should have the \"header\" \"Id\"\n And error 1 should have the constraint \"min_length\" \"3\"\n\n Scenario: Invalid schema\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\nNO JSON HERE SON\n \"\"\"\n And the schema is stored at the url \"http://example.com/schema.json\"\n Then nothing should be outputted to STDERR\n When I run `csvlint http://example.com/example1.csv --schema http://example.com/schema.json`\n And the output should contain \"invalid metadata: malformed JSON\"\n\n Scenario: Schema that 404s\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And there is no file at the url \"http://example.com/schema404.json\"\n When I run `csvlint http://example.com/example1.csv --schema http://example.com/schema404.json`\n Then the output should contain \"http://example.com/schema404.json not found\"\n\n Scenario: Schema that doesn't exist\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I run `csvlint http://example.com/example1.csv --schema /fake/file/path.json`\n Then the output should contain \"/fake/file/path.json not found\"\n\n Scenario: Valid CSVw schema\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have metadata with the following content:\n \"\"\"\n{\n \"@context\": \"http://www.w3.org/ns/csvw\",\n \"url\": \"http://example.com/example1.csv\",\n \"dialect\": { \"header\": false },\n \"tableSchema\": {\n \"columns\": [\n { \"name\": \"Name\", \"required\": true },\n { \"name\": \"Id\", \"required\": true, \"datatype\": { \"base\": \"string\", \"minLength\": 1 } },\n { \"name\": \"Email\", \"required\": true }\n ]\n }\n}\n \"\"\"\n And the schema is stored at the url \"http://example.com/schema.json\"\n When I run `csvlint http://example.com/example1.csv --schema http://example.com/schema.json`\n Then the output should contain \"http://example.com/example1.csv is VALID\"\n\n Scenario: CSVw schema with invalid CSV\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have metadata with the following content:\n \"\"\"\n{\n \"@context\": \"http://www.w3.org/ns/csvw\",\n \"url\": \"http://example.com/example1.csv\",\n \"dialect\": { \"header\": false },\n \"tableSchema\": {\n \"columns\": [\n { \"name\": \"Name\", \"required\": true },\n { \"name\": \"Id\", \"required\": true, \"datatype\": { \"base\": \"string\", \"minLength\": 3 } },\n { \"name\": \"Email\", \"required\": true }\n ]\n }\n}\n \"\"\"\n And the schema is stored at the url \"http://example.com/schema.json\"\n When I run `csvlint http://example.com/example1.csv --schema http://example.com/schema.json`\n Then the output should contain \"http://example.com/example1.csv is INVALID\"\n And the output should contain \"1. min_length. Row: 2,2. 5\"\n\n Scenario: CSVw table Schema\n Given I have stubbed stdin to contain nothing\n And I have a metadata file called \"csvw/countries.json\"\n And the metadata is stored at the url \"http://w3c.github.io/csvw/tests/countries.json\"\n And I have a file called \"csvw/countries.csv\" at the url \"http://w3c.github.io/csvw/tests/countries.csv\"\n And I have a file called \"csvw/country_slice.csv\" at the url \"http://w3c.github.io/csvw/tests/country_slice.csv\"\n When I run `csvlint --schema http://w3c.github.io/csvw/tests/countries.json`\n Then the output should contain \"http://w3c.github.io/csvw/tests/countries.csv is VALID\"\n And the output should contain \"http://w3c.github.io/csvw/tests/country_slice.csv is VALID\"\n" }, { "path": "features/csv_options.feature", "content": "Feature: CSV options\n\n Scenario: Sucessfully parse a valid CSV\n Given I have a CSV with the following content:\n \"\"\"\n'Foo';'Bar';'Baz'\n'1';'2';'3'\n'3';'2';'1'\n \"\"\"\n And I set the delimiter to \";\"\n And I set quotechar to \"'\" \n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of true\n\n Scenario: Warn if options seem to return invalid data\n Given I have a CSV with the following content:\n \"\"\"\n'Foo';'Bar';'Baz'\n'1';'2';'3'\n'3';'2';'1'\n \"\"\"\n And I set the delimiter to \",\"\n And I set quotechar to \"\"\" \n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"check_options\"\n\n Scenario: Use esoteric line endings\n Given I have a CSV file called \"windows-line-endings.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of true\n " }, { "path": "features/csvupload.feature", "content": "Feature: Collect all the tests that should trigger dialect check related errors\n\n Scenario: Title rows, I wish to trigger a :title_row type message\n Given I have a CSV file called \"title-row.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"title_row\"\n\n# :nonrfc_line_breaks\n\n Scenario: LF line endings in file give an info message of type :nonrfc_line_breaks\n Given I have a CSV file called \"lf-line-endings.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I set header to \"true\"\n And I ask if there are info messages\n Then there should be 1 info message\n And one of the messages should have the type \"nonrfc_line_breaks\"\n\n Scenario: CRLF line endings in file produces no info messages of type :nonrfc_line_breaks\n Given I have a CSV file called \"crlf-line-endings.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I set header to \"true\"\n And I ask if there are info messages\n Then there should be 0 info messages\n\n# :line_breaks\n\n Scenario: Incorrect line endings specified in settings\n Given I have a CSV file called \"lf-line-endings.csv\"\n And I set the line endings to carriage return\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"line_breaks\"\n\n Scenario: inconsistent line endings in file cause an error\n Given I have a CSV file called \"inconsistent-line-endings.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"line_breaks\"\n\n\n Scenario: inconsistent line endings with unquoted fields in file cause an error\n Given I have a CSV file called \"inconsistent-line-endings-unquoted.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"line_breaks\"\n\n#:unclosed_quote\n\n Scenario: CSV with incorrect quoting\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\",\"Baz\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"unclosed_quote\"\n And that error should have the row \"2\"\n And that error should have the content \"\"Foo\",\"Bar\",\"Baz\"\n\n# :invalid_encoding\n\n Scenario: Report invalid Encoding\n Given I have a CSV file called \"invalid-byte-sequence.csv\"\n And I set an encoding header of \"UTF-8\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"invalid_encoding\"\n\n Scenario: Report invalid file\n#should this throw an excel error?\n Given I have a CSV file called \"spreadsheet.xls\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"invalid_encoding\"\n\n# :blank_rows\n\n Scenario: Successfully report a CSV with blank rows\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\",\"Baz\"\n\"\",\"\",\n\"Baz\",\"Bar\",\"Foo\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"blank_rows\"\n And that error should have the row \"3\"\n And that error should have the content \"\"\",\"\",\"\n\n Scenario: Successfully report a CSV with multiple trailing empty rows\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\",\"Baz\"\n\"Foo\",\"Bar\",\"Baz\"\n\n\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"blank_rows\"\n And that error should have the row \"4\"\n\n Scenario: Successfully report a CSV with an empty row\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\",\"Baz\"\n\n\"Foo\",\"Bar\",\"Baz\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"blank_rows\"\n And that error should have the row \"3\"\n\n#:check_options\n\n Scenario: Warn if options seem to return invalid data\n Given I have a CSV with the following content:\n \"\"\"\n'Foo';'Bar';'Baz'\n'1';'2';'3'\n'3';'2';'1'\n \"\"\"\n And I set the delimiter to \",\"\n And I set quotechar to \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"check_options\"\n" }, { "path": "features/csvw_schema_validation.feature", "content": "Feature: CSVW Schema Validation\n\n Scenario: Valid CSV\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have metadata with the following content:\n \"\"\"\n{\n \"@context\": \"http://www.w3.org/ns/csvw\",\n \"url\": \"http://example.com/example1.csv\",\n \"dialect\": { \"header\": false },\n \"tableSchema\": {\n \"columns\": [\n { \"name\": \"Name\", \"required\": true },\n { \"name\": \"Id\", \"required\": true, \"datatype\": { \"base\": \"string\", \"minLength\": 1 } },\n { \"name\": \"Email\", \"required\": true }\n ]\n }\n}\n \"\"\"\n When I ask if there are errors\n Then there should be 0 error\n\n Scenario: Schema invalid CSV\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have metadata with the following content:\n \"\"\"\n{\n \"@context\": \"http://www.w3.org/ns/csvw\",\n \"url\": \"http://example.com/example1.csv\",\n \"dialect\": { \"header\": false },\n \"tableSchema\": {\n \"columns\": [\n { \"name\": \"Name\", \"required\": true },\n { \"name\": \"Id\", \"required\": true, \"datatype\": { \"base\": \"string\", \"minLength\": 3 } },\n { \"name\": \"Email\", \"required\": true }\n ]\n }\n}\n \"\"\"\n When I ask if there are errors\n Then there should be 1 error\n\n Scenario: CSV with incorrect header\n Given I have a CSV with the following content:\n \"\"\"\n\"name\",\"id\",\"contact\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have metadata with the following content:\n \"\"\"\n{\n \"@context\": \"http://www.w3.org/ns/csvw\",\n \"url\": \"http://example.com/example1.csv\",\n \"tableSchema\": {\n \"columns\": [\n { \"titles\": \"name\", \"required\": true },\n { \"titles\": \"id\", \"required\": true, \"datatype\": { \"base\": \"string\", \"minLength\": 1 } },\n { \"titles\": \"email\", \"required\": true }\n ]\n }\n}\n \"\"\"\n When I ask if there are errors\n Then there should be 1 error\n\n Scenario: Schema with valid regex\n Given I have a CSV with the following content:\n \"\"\"\n \"firstname\",\"id\",\"email\"\n \"Bob\",\"1234\",\"bob@example.org\"\n \"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have metadata with the following content:\n \"\"\"\n{\n \"@context\": \"http://www.w3.org/ns/csvw\",\n \"url\": \"http://example.com/example1.csv\",\n \"tableSchema\": {\n \"columns\": [\n { \"titles\": \"firstname\", \"required\": true, \"datatype\": { \"base\": \"string\", \"format\": \"^[A-Za-z0-9_]*$\" } },\n { \"titles\": \"id\", \"required\": true, \"datatype\": { \"base\": \"string\", \"minLength\": 1 } },\n { \"titles\": \"email\", \"required\": true }\n ]\n }\n}\n \"\"\"\n When I ask if there are warnings\n Then there should be 0 warnings\n\n Scenario: Schema with invalid regex\n Given I have a CSV with the following content:\n \"\"\"\n \"firstname\",\"id\",\"email\"\n \"Bob\",\"1234\",\"bob@example.org\"\n \"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have metadata with the following content:\n \"\"\"\n{\n \"@context\": \"http://www.w3.org/ns/csvw\",\n \"url\": \"http://example.com/example1.csv\",\n \"tableSchema\": {\n \"columns\": [\n { \"titles\": \"firstname\", \"required\": true, \"datatype\": { \"base\": \"string\", \"format\": \"((\" } },\n { \"titles\": \"id\", \"required\": true, \"datatype\": { \"base\": \"string\", \"minLength\": 1 } },\n { \"titles\": \"email\", \"required\": true }\n ]\n }\n}\n \"\"\"\n When I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"invalid_regex\"\n" }, { "path": "features/fixtures/cr-line-endings.csv", "content": "\"Foo\",\"Bar\",\"Baz\"\r\"Biff\",\"Baff\",\"Boff\"\r\"Qux\",\"Teaspoon\",\"Doge\"" }, { "path": "features/fixtures/crlf-line-endings.csv", "content": "\"Foo\",\"Bsr\",\"Baz\"\r\n\"Biff\",\"Baff\",\"Boff\"\r\n\"Qux\",\"Teaspoon\",\"Doge\"" }, { "path": "features/fixtures/inconsistent-line-endings-unquoted.csv", "content": "Foo,Bsr,Baz\r\nBiff,Baff,Boff\nQux,Teaspoon,Doge" }, { "path": "features/fixtures/inconsistent-line-endings.csv", "content": "\"Foo\",\"Bsr\",\"Baz\"\r\n\"Biff\",\"Baff\",\"Boff\"\n\"Qux\",\"Teaspoon\",\"Doge\"" }, { "path": "features/fixtures/invalid-byte-sequence.csv", "content": "\"Data\",\"Dependencia Origem\",\"Histrico\",\"Data do Balancete\",\"Nmero do documento\",\"Valor\",\r\n\"10/31/2012\",\"\",\"Saldo Anterior\",\"\",\"0\",\"100.00\",\r\n\"11/01/2012\",\"0000-9\",\"Transferncia on line - 01/11 4885 256620-6 XXXXXXXXXXXXX\",\"\",\"224885000256620\",\"100.00\",\r\n\"11/01/2012\",\"\",\"Depsito COMPE - 033 0502 27588602104 XXXXXXXXXXXXXX\",\"\",\"101150\",\"100.00\",\r\n\"11/01/2012\",\"\",\"Proventos\",\"\",\"496774\",\"1000.00\",\r\n\"11/01/2012\",\"\",\"Benefcio\",\"\",\"496775\",\"100.00\",\r\n\"11/01/2012\",\"0000-0\",\"Compra com Carto - 01/11 09:45 XXXXXXXXXXX\",\"\",\"135102\",\"-1.00\",\r\n\"11/01/2012\",\"0000-0\",\"Compra com Carto - 01/11 09:48 XXXXXXXXXXX\",\"\",\"235338\",\"-10.00\",\r\n\"11/01/2012\",\"0000-0\",\"Compra com Carto - 01/11 12:35 XXXXXXXX\",\"\",\"345329\",\"-10.00\",\r\n\"11/01/2012\",\"0000-0\",\"Compra com Carto - 01/11 23:57 XXXXXXXXXXXXXXXX\",\"\",\"686249\",\"-10.00\",\r\n\"11/01/2012\",\"0000-0\",\"Saque com carto - 01/11 13:17 XXXXXXXXXXXXXXXX\",\"\",\"11317296267021\",\"-10.00\",\r\n\"11/01/2012\",\"\",\"Pagto conta telefone - VIVO DF\",\"\",\"110101\",\"-100.00\",\r\n\"11/01/2012\",\"\",\"Cobrana de I.O.F.\",\"\",\"391100701\",\"-1.00\",\r\n\"11/05/2012\",\"0000-0\",\"Compra com Carto - 02/11 16:57 XXXXXXXXXXXX\",\"\",\"161057\",\"-10.00\",\r\n\"11/05/2012\",\"0000-0\",\"Compra com Carto - 03/11 18:57 XXXXXXXXXXXXXXX\",\"\",\"168279\",\"-10.00\",\r\n\"11/05/2012\",\"0000-0\",\"Compra com Carto - 05/11 12:32 XXXXXXXXXXXXXXXXX\",\"\",\"245166\",\"-10.00\",\r\n\"11/05/2012\",\"0000-0\",\"Compra com Carto - 02/11 17:18 XXXXXXXXXXXXX\",\"\",\"262318\",\"-1.00\",\r\n\"11/05/2012\",\"0000-0\",\"Compra com Carto - 02/11 22:46 XXXXXXXXXXX\",\"\",\"382002\",\"-100.00\",\r\n\"11/05/2012\",\"0000-0\",\"Compra com Carto - 02/11 23:19 XXXXXXXXXXX\",\"\",\"683985\",\"-1.00\",\r\n\"11/05/2012\",\"0000-0\",\"Compra com Carto - 03/11 01:19 XXXXXXXXXXXXXXXX\",\"\",\"704772\",\"-10.00\",\r\n\"11/05/2012\",\"0000-0\",\"Compra com Carto - 03/11 11:08 XXXXXXXX\",\"\",\"840112\",\"-1.00\",\r\n\"11/05/2012\",\"0000-0\",\"Saque com carto - 05/11 19:24 XXXXXXXXXXXXXXXXX\",\"\",\"51924256267021\",\"-10.00\",\r\n\"11/05/2012\",\"0000-0\",\"Transferncia on line - 05/11 4885 256620-6 XXXXXXXXXXXXX\",\"\",\"224885000256620\",\"-100.00\",\r\n\"11/05/2012\",\"\",\"Pagamento de Ttulo - XXXXXXXXXXXXXXXXXXX\",\"\",\"110501\",\"-100.00\",\r\n" }, { "path": "features/fixtures/invalid_many_rows.csv", "content": "\"Foo\",\"Bar\",\"Baz\"\r\n\"1\",\"2\",\"3\"\r\n\"3\",\"2\",\"1\"\r\n\"1\",\"2\",\"3\" \"\r\n\"3\",\"two\",\"1\"\r\n\"1\",\"2\",\"3\"\r\n\"3\",\"2\",\"1\"\r\n\r\n\"3\",\"2\",\"1\"\r\n\"3\",\"2\",\"1\"\r\n\"\",\"\",\"\"\r\n\"3\",\"2\",\"1\"" }, { "path": "features/fixtures/lf-line-endings.csv", "content": "\"Foo\",\"Bsr\",\"Baz\"\n\"Biff\",\"Baff\",\"Boff\"\n\"Qux\",\"Teaspoon\",\"Doge\"" }, { "path": "features/fixtures/title-row.csv", "content": "\"This is a title row\",,\n\"Foo\",\"Bsr\",\"Baz\"\n\"Biff\",\"Baff\",\"Boff\"\n\"Qux\",\"Teaspoon\",\"Doge\"" }, { "path": "features/fixtures/valid.csv", "content": "\"Foo\",\"Bar\",\"Baz\"\r\n\"1\",\"2\",\"3\"\r\n\"3\",\"2\",\"1\"\r\n" }, { "path": "features/fixtures/valid_many_rows.csv", "content": "\"Foo\",\"Bar\",\"Baz\"\r\n\"1\",\"2\",\"3\"\r\n\"3\",\"2\",\"1\"\r\n\"1\",\"2\",\"3\"\r\n\"3\",\"2\",\"1\"\r\n\"1\",\"2\",\"3\"\r\n\"3\",\"2\",\"1\"\r\n" }, { "path": "features/fixtures/w3.org/.well-known/csvm", "content": "{+url}-metadata.json\ncsv-metadata.json\n{+url}.json\ncsvm.json" }, { "path": "features/fixtures/white space in filename.csv", "content": "\"Foo\",\"Bar\",\"Baz\"\r\n\"1\",\"2\",\"3\"\r\n\"3\",\"2\",\"1\"\r\n" }, { "path": "features/fixtures/windows-line-endings.csv", "content": "a,b,c\r\nd,e,f\r\n" }, { "path": "features/information.feature", "content": "Feature: Return information\n\n Background:\n Given I have a CSV with the following content:\n \"\"\"\n\"abc\",\"2\",\"3\"\n \"\"\"\n And it is encoded as \"utf-8\"\n And the content type is \"text/csv\"\n And it is stored at the url \"http://example.com/example1.csv?query=true\"\n\n Scenario: Return encoding\n Then the \"encoding\" should be \"UTF-8\"\n\n Scenario: Return content type\n Then the \"content_type\" should be \"text/csv; charset=utf-8\"\n\n Scenario: Return extension\n Then the \"extension\" should be \".csv\"\n\n Scenario: Return meta\n Then the metadata content type should be \"text/csv; charset=utf-8\"\n" }, { "path": "features/parse_csv.feature", "content": "Feature: Parse CSV\n\n Scenario: Successfully parse a valid CSV\n Given I have a CSV with the following content:\n \"\"\"\n\"Foo\",\"Bar\",\"Baz\"\n\"1\",\"2\",\"3\"\n\"3\",\"2\",\"1\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of true\n\n Scenario: Successfully parse a CSV with newlines in quoted fields\n Given I have a CSV with the following content:\n \"\"\"\n\"a\",\"b\",\"c\"\n\"d\",\"e\",\"this is\nvalid\"\n\"a\",\"b\",\"c\"\n\"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of true\n\n Scenario: Successfully parse a CSV with multiple newlines in quoted fields\n Given I have a CSV with the following content:\n \"\"\"\n\"a\",\"b\",\"c\"\n\"d\",\"this is\nvalid\",\"as is this\ntoo\"\n\"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of true\n\n Scenario: Successfully report an invalid CSV\n Given I have a CSV with the following content:\n \"\"\"\n\t\"Foo\",\t\"Bar\"\t,\t\"Baz\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of false\n\n Scenario: Successfully report a CSV with incorrect quoting\n Given I have a CSV with the following content:\n \"\"\"\n\"Foo\",\"Bar\",\"Baz\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of false\n\n Scenario: Successfully report a CSV with incorrect whitespace\n Given I have a CSV with the following content:\n \"\"\"\n\"Foo\",\"Bar\", \"Baz\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of false\n\n Scenario: Successfully report a CSV with ragged rows\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col2\"\n\"1\",\"2\",\"3\"\n\"4\",\"5\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if the CSV is valid\n Then I should get the value of false\n\n Scenario: Don't class blank values as inconsistencies\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"1\",\"2\",\"3\"\n\"4\",\"5\",\"6\"\n\"\",\"7\",\"8\"\n\"9\",\"10\",\"11\"\n\"\",\"12\",\"13\"\n\"\",\"14\",\"15\"\n\"16\",\"17\",\"18\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are warnings\n Then there should be 0 warnings\n" }, { "path": "features/schema_validation.feature", "content": "Feature: Schema Validation\n\n Scenario: Valid CSV\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\n{\n\t\"fields\": [\n { \"name\": \"Name\", \"constraints\": { \"required\": true } },\n { \"name\": \"Id\", \"constraints\": { \"required\": true, \"minLength\": 1 } },\n { \"name\": \"Email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n When I ask if there are errors\n Then there should be 0 error\n\n Scenario: Schema invalid CSV\n Given I have a CSV with the following content:\n \"\"\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\n{\n\t\"fields\": [\n { \"name\": \"Name\", \"constraints\": { \"required\": true } },\n { \"name\": \"Id\", \"constraints\": { \"required\": true, \"minLength\": 3 } },\n { \"name\": \"Email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n When I ask if there are errors\n Then there should be 1 error\n\n Scenario: CSV with incorrect header\n Given I have a CSV with the following content:\n \"\"\"\n\"name\",\"id\",\"contact\"\n\"Bob\",\"1234\",\"bob@example.org\"\n\"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\n{\n\t\"fields\": [\n { \"name\": \"name\", \"constraints\": { \"required\": true } },\n { \"name\": \"id\", \"constraints\": { \"required\": true, \"minLength\": 3 } },\n { \"name\": \"email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n When I ask if there are warnings\n Then there should be 1 warnings\n\n Scenario: Schema with valid regex\n Given I have a CSV with the following content:\n \"\"\"\n \"firstname\",\"id\",\"email\"\n \"Bob\",\"1234\",\"bob@example.org\"\n \"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\n{\n \"fields\": [\n { \"name\": \"Name\", \"constraints\": { \"required\": true, \"pattern\": \"^[A-Za-z0-9_]*$\" } },\n { \"name\": \"Id\", \"constraints\": { \"required\": true, \"minLength\": 1 } },\n { \"name\": \"Email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n When I ask if there are errors\n Then there should be 0 error\n\n Scenario: Schema with invalid regex\n Given I have a CSV with the following content:\n \"\"\"\n \"firstname\",\"id\",\"email\"\n \"Bob\",\"1234\",\"bob@example.org\"\n \"Alice\",\"5\",\"alice@example.com\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I have a schema with the following content:\n \"\"\"\n{\n \"fields\": [\n { \"name\": \"Name\", \"constraints\": { \"required\": true, \"pattern\": \"((\" } },\n { \"name\": \"Id\", \"constraints\": { \"required\": true, \"minLength\": 1 } },\n { \"name\": \"Email\", \"constraints\": { \"required\": true } }\n ]\n}\n \"\"\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"invalid_regex\"\n" }, { "path": "features/sources.feature", "content": "Feature: Parse CSV from Different Sources\n\n Scenario: Successfully parse a valid CSV from a StringIO\n Given I have a CSV with the following content:\n \"\"\"\n\"Foo\",\"Bar\",\"Baz\"\n\"1\",\"2\",\"3\"\n\"3\",\"2\",\"1\"\n \"\"\"\n And it is parsed as a StringIO\n When I ask if the CSV is valid\n Then I should get the value of true\n\n Scenario: Successfully parse a valid CSV from a File\n Given I parse a file called \"valid.csv\"\n When I ask if the CSV is valid\n Then I should get the value of true\n" }, { "path": "features/step_definitions/cli_steps.rb", "content": "Given(/^I have stubbed $stdin to contain \"(.*?)\"$/) do |file|\n expect($stdin).to receive(:read).and_return(File.read(file))\nend\n\nGiven(/^I have stubbed $stdin to contain nothing$/) do\n expect($stdin).to receive(:read).and_return(nil)\nend\n\nThen(/^nothing should be outputted to STDERR$/) do\n expect($stderr).to_not receive(:puts)\nend\n\nThen(/^the output should contain JSON$/) do\n @json = JSON.parse(all_stdout)\n expect(@json[\"validation\"]).to be_present\nend\n\nThen(/^the JSON should have a state of \"(.*?)\"$/) do |state|\n expect(@json[\"validation\"][\"state\"]).to eq(state)\nend\n\nThen(/^the JSON should have (\\d+) errors?$/) do |count|\n @index = count.to_i - 1\n expect(@json[\"validation\"][\"errors\"].count).to eq(count.to_i)\nend\n\nThen(/^that error should have the \"(.*?)\" \"(.*?)\"$/) do |k, v|\n expect(@json[\"validation\"][\"errors\"][@index][k].to_s).to eq(v)\nend\n\nThen(/^error (\\d+) should have the \"(.*?)\" \"(.*?)\"$/) do |index, k, v|\n expect(@json[\"validation\"][\"errors\"][index.to_i - 1][k].to_s).to eq(v)\nend\n\nThen(/^error (\\d+) should have the constraint \"(.*?)\" \"(.*?)\"$/) do |index, k, v|\n expect(@json[\"validation\"][\"errors\"][index.to_i - 1][\"constraints\"][k].to_s).to eq(v)\nend\n" }, { "path": "features/step_definitions/csv_options_steps.rb", "content": "Given(/^I set the delimiter to \"(.*?)\"$/) do |delimiter|\n @csv_options ||= default_csv_options\n @csv_options[\"delimiter\"] = delimiter\nend\n\nGiven(/^I set quotechar to \"(.*?)\"$/) do |doublequote|\n @csv_options ||= default_csv_options\n @csv_options[\"quoteChar\"] = doublequote\nend\n\nGiven(/^I set the line endings to linefeed$/) do\n @csv_options ||= default_csv_options\n @csv_options[\"lineTerminator\"] = \"\\n\"\nend\n\nGiven(/^I set the line endings to carriage return$/) do\n @csv_options ||= default_csv_options\n @csv_options[\"lineTerminator\"] = \"\\r\"\nend\n\nGiven(/^I set header to \"(.*?)\"$/) do |boolean|\n @csv_options ||= default_csv_options\n @csv_options[\"header\"] = boolean == \"true\"\nend\n" }, { "path": "features/step_definitions/information_steps.rb", "content": "Given(/^the content type is \"(.*?)\"$/) do |arg1|\n @content_type = \"text/csv\"\nend\n\nThen(/^the \"(.*?)\" should be \"(.*?)\"$/) do |type, encoding|\n validator = Csvlint::Validator.new(@url, default_csv_options)\n expect(validator.send(type.to_sym)).to eq(encoding)\nend\n\nThen(/^the metadata content type should be \"(.*?)\"$/) do |content_type|\n validator = Csvlint::Validator.new(@url, default_csv_options)\n expect(validator.headers[\"content-type\"]).to eq(content_type)\nend\n" }, { "path": "features/step_definitions/parse_csv_steps.rb", "content": "Given(/^I have a CSV with the following content:$/) do |string|\n @csv = string.to_s\nend\n\nGiven(/^it has a Link header holding \"(.*?)\"$/) do |link|\n @link = \"#{link}; type=\\\"application/csvm+json\\\"\"\nend\n\nGiven(/^it is stored at the url \"(.*?)\"$/) do |url|\n @url = url\n content_type = @content_type || \"text/csv\"\n charset = @encoding || \"UTF-8\"\n headers = {\"Content-Type\" => \"#{content_type}; charset=#{charset}\"}\n headers[\"Link\"] = @link if @link\n stub_request(:get, url).to_return(status: 200, body: @csv, headers: headers)\n stub_request(:get, URI.join(url, \"/.well-known/csvm\")).to_return(status: 404)\n stub_request(:get, url + \"-metadata.json\").to_return(status: 404)\n stub_request(:get, URI.join(url, \"csv-metadata.json\")).to_return(status: 404)\nend\n\nGiven(/^it is stored at the url \"(.*?)\" with no character set$/) do |url|\n @url = url\n content_type = @content_type || \"text/csv\"\n stub_request(:get, url).to_return(status: 200, body: @csv, headers: {\"Content-Type\" => content_type.to_s})\n stub_request(:get, URI.join(url, \"/.well-known/csvm\")).to_return(status: 404)\n stub_request(:get, url + \"-metadata.json\").to_return(status: 404)\n stub_request(:get, URI.join(url, \"csv-metadata.json\")).to_return(status: 404)\nend\n\nWhen(/^I ask if the CSV is valid$/) do\n @csv_options ||= default_csv_options\n @validator = Csvlint::Validator.new(@url, @csv_options)\n @valid = @validator.valid?\nend\n\nThen(/^I should get the value of true$/) do\n expect(@valid).to be(true)\nend\n\nThen(/^I should get the value of false$/) do\n expect(@valid).to be(false)\nend\n" }, { "path": "features/step_definitions/schema_validation_steps.rb", "content": "Given(/^I have a schema with the following content:$/) do |json|\n @schema_type = :json_table\n @schema_json = json\nend\n\nGiven(/^I have metadata with the following content:$/) do |json|\n @schema_type = :csvw_metadata\n @schema_json = json\nend\n\nGiven(/^I have a metadata file called \"([^\"]*)\"$/) do |filename|\n @schema_type = :csvw_metadata\n @schema_json = File.read(File.join(File.dirname(__FILE__), \"..\", \"fixtures\", filename))\nend\n\nGiven(/^the (schema|metadata) is stored at the url \"(.*?)\"$/) do |schema_type, schema_url|\n @schema_url = schema_url\n stub_request(:get, @schema_url).to_return(status: 200, body: @schema_json.to_str)\nend\n\nGiven(/^there is a file at \"(.*?)\" with the content:$/) do |url, content|\n stub_request(:get, url).to_return(status: 200, body: content.to_str)\nend\n\nGiven(/^I have a file called \"(.*?)\" at the url \"(.*?)\"$/) do |filename, url|\n content = File.read(File.join(File.dirname(__FILE__), \"..\", \"fixtures\", filename))\n content_type = /.csv$/.match?(filename) ? \"text/csv\" : \"application/csvm+json\"\n stub_request(:get, url).to_return(status: 200, body: content, headers: {\"Content-Type\" => \"#{content_type}; charset=UTF-8\"})\nend\n\nGiven(/^there is no file at the url \"(.*?)\"$/) do |url|\n stub_request(:get, url).to_return(status: 404)\nend\n" }, { "path": "features/step_definitions/sources_steps.rb", "content": "Given(/^it is parsed as a StringIO$/) do\n @url = StringIO.new(@csv)\nend\n\nGiven(/^I parse a file called \"(.*?)\"$/) do |filename|\n @url = File.new(File.join(File.dirname(__FILE__), \"..\", \"fixtures\", filename))\nend\n" }, { "path": "features/step_definitions/validation_errors_steps.rb", "content": "When(/^I ask if there are errors$/) do\n @csv_options ||= default_csv_options\n\n if @schema_json\n @schema = if @schema_type == :json_table\n Csvlint::Schema.from_json_table(@schema_url || \"http://example.org \", JSON.parse(@schema_json))\n else\n Csvlint::Schema.from_csvw_metadata(@schema_url || \"http://example.org \", JSON.parse(@schema_json))\n end\n end\n\n @validator = Csvlint::Validator.new(@url, @csv_options, @schema)\n @errors = @validator.errors\nend\n\nWhen(/^I carry out CSVW validation$/) do\n @csv_options ||= default_csv_options\n\n begin\n if @schema_json\n json = JSON.parse(@schema_json)\n @schema = if @schema_type == :json_table\n Csvlint::Schema.from_json_table(@schema_url || \"http://example.org \", json)\n else\n Csvlint::Schema.from_csvw_metadata(@schema_url || \"http://example.org \", json)\n end\n end\n\n if @url.nil?\n @errors = []\n @warnings = []\n @schema.tables.keys.each do |table_url|\n validator = Csvlint::Validator.new(table_url, @csv_options, @schema)\n @errors += validator.errors\n @warnings += validator.warnings\n end\n else\n validator = Csvlint::Validator.new(@url, @csv_options, @schema)\n @errors = validator.errors\n @warnings = validator.warnings\n end\n rescue JSON::ParserError => e\n @errors = [e]\n rescue Csvlint::Csvw::MetadataError => e\n @errors = [e]\n end\nend\n\nThen(/^there should be errors$/) do\n # this test is only used for CSVW testing; :invalid_encoding & :line_breaks mask lack of real errors\n @errors.delete_if { |e| e.instance_of?(Csvlint::ErrorMessage) && [:invalid_encoding, :line_breaks].include?(e.type) }\n expect(@errors.count).to be > 0\nend\n\nThen(/^there should not be errors$/) do\n expect(@errors.count).to eq(0)\nend\n\nThen(/^there should be (\\d+) error$/) do |count|\n expect(@errors.count).to eq(count.to_i)\nend\n\nThen(/^that error should have the type \"(.*?)\"$/) do |type|\n expect(@errors.first.type).to eq(type.to_sym)\nend\n\nThen(/^that error should have the row \"(.*?)\"$/) do |row|\n expect(@errors.first.row).to eq(row.to_i)\nend\n\nThen(/^that error should have the column \"(.*?)\"$/) do |column|\n expect(@errors.first.column).to eq(column.to_i)\nend\n\nThen(/^that error should have the content \"(.*)\"$/) do |content|\n expect(@errors.first.content.chomp).to eq(content.chomp)\nend\n\nThen(/^that error should have no content$/) do\n expect(@errors.first.content).to eq(nil)\nend\n\nGiven(/^I have a CSV that doesn't exist$/) do\n @url = \"http//www.example.com/fake-csv.csv\"\n stub_request(:get, @url).to_return(status: 404)\nend\n\nThen(/^there should be no \"(.*?)\" errors$/) do |type|\n @errors.each { |error| error.type.should_not == type.to_sym }\nend\n" }, { "path": "features/step_definitions/validation_info_steps.rb", "content": "Given(/^I ask if there are info messages$/) do\n @csv_options ||= default_csv_options\n\n if @schema_json\n @schema = if @schema_type == :json_table\n Csvlint::Schema.from_json_table(@schema_url || \"http://example.org \", JSON.parse(@schema_json))\n else\n Csvlint::Schema.from_csvw_metadata(@schema_url || \"http://example.org \", JSON.parse(@schema_json))\n end\n end\n\n @validator = Csvlint::Validator.new(@url, @csv_options, @schema)\n @info_messages = @validator.info_messages\nend\n\nThen(/^there should be (\\d+) info messages?$/) do |num|\n expect(@info_messages.count).to eq(num.to_i)\nend\n\nThen(/^one of the messages should have the type \"(.*?)\"$/) do |msg_type|\n expect(@info_messages.find { |x| x.type == msg_type.to_sym }).to be_present\nend\n" }, { "path": "features/step_definitions/validation_warnings_steps.rb", "content": "Given(/^it is encoded as \"(.*?)\"$/) do |encoding|\n @csv = @csv.encode(encoding)\n @encoding = encoding\nend\n\nGiven(/^I set an encoding header of \"(.*?)\"$/) do |encoding|\n @encoding = encoding\nend\n\nGiven(/^I do not set an encoding header$/) do\n @encoding = nil\nend\n\nGiven(/^I have a CSV file called \"(.*?)\"$/) do |filename|\n @csv = File.read(File.join(File.dirname(__FILE__), \"..\", \"fixtures\", filename))\nend\n\nWhen(/^I ask if there are warnings$/) do\n @csv_options ||= default_csv_options\n if @schema_json\n @schema = if @schema_type == :json_table\n Csvlint::Schema.from_json_table(@schema_url || \"http://example.org \", JSON.parse(@schema_json))\n else\n Csvlint::Schema.from_csvw_metadata(@schema_url || \"http://example.org \", JSON.parse(@schema_json))\n end\n end\n\n @validator = Csvlint::Validator.new(@url, @csv_options, @schema)\n @warnings = @validator.warnings\nend\n\nThen(/^there should be warnings$/) do\n expect(@warnings.count).to be > 0\nend\n\nThen(/^there should not be warnings$/) do\n # this test is only used for CSVW testing, and :inconsistent_values warnings don't count in CSVW\n @warnings.delete_if { |w| [:inconsistent_values, :check_options].include?(w.type) }\n expect(@warnings.count).to eq(0)\nend\n\nThen(/^there should be (\\d+) warnings$/) do |count|\n expect(@warnings.count).to eq(count.to_i)\nend\n\nGiven(/^the content type is set to \"(.*?)\"$/) do |type|\n @content_type = type\nend\n\nThen(/^that warning should have the row \"(.*?)\"$/) do |row|\n expect(@warnings.first.row).to eq(row.to_i)\nend\n\nThen(/^that warning should have the column \"(.*?)\"$/) do |column|\n expect(@warnings.first.column).to eq(column.to_i)\nend\n\nThen(/^that warning should have the type \"(.*?)\"$/) do |type|\n expect(@warnings.first.type).to eq(type.to_sym)\nend\n" }, { "path": "features/support/aruba.rb", "content": "require \"aruba\"\nrequire \"aruba/cucumber\"\n\nrequire \"csvlint/cli\"\n\nmodule Csvlint\n class CliRunner\n # Allow everything fun to be injected from the outside while defaulting to normal implementations.\n def initialize(argv, stdin = $stdin, stdout = $stdout, stderr = $stderr, kernel = Kernel)\n @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel\n end\n\n def execute!\n exit_code = begin\n # Thor accesses these streams directly rather than letting them be injected, so we replace them...\n $stderr = @stderr\n $stdin = @stdin\n $stdout = @stdout\n\n # Run our normal Thor app the way we know and love.\n Csvlint::Cli.start(@argv.dup.unshift(\"validate\"))\n\n # Thor::Base#start does not have a return value, assume success if no exception is raised.\n 0\n rescue => e\n # The ruby interpreter would pipe this to STDERR and exit 1 in the case of an unhandled exception\n b = e.backtrace\n @stderr.puts(\"#{b.shift}: #{e.message} (#{e.class})\")\n @stderr.puts(b.map { |s| \"\\tfrom #{s}\" }.join(\"\\n\"))\n 1\n rescue SystemExit => e\n e.status\n ensure\n # TODO: reset your app here, free up resources, etc.\n # Examples:\n # MyApp.logger.flush\n # MyApp.logger.close\n # MyApp.logger = nil\n #\n # MyApp.reset_singleton_instance_variables\n\n # ...then we put the streams back.\n $stderr = STDERR\n $stdin = STDIN\n $stdout = STDOUT\n end\n\n # Proxy our exit code back to the injected kernel.\n @kernel.exit(exit_code)\n end\n end\nend\n\nAruba.configure do |config|\n config.command_launcher = :in_process\n config.main_class = Csvlint::CliRunner\nend\n" }, { "path": "features/support/earl_formatter.rb", "content": "require \"rdf\"\nrequire \"rdf/turtle\"\n\nclass EarlFormatter\n def initialize(step_mother, io, options)\n output = RDF::Resource.new(\"\")\n @graph = RDF::Graph.new\n @graph << [CSVLINT, RDF.type, RDF::DOAP.Project]\n @graph << [CSVLINT, RDF.type, EARL.TestSubject]\n @graph << [CSVLINT, RDF.type, EARL.Software]\n @graph << [CSVLINT, RDF::DOAP.name, \"csvlint\"]\n @graph << [CSVLINT, RDF::DC.title, \"csvlint\"]\n @graph << [CSVLINT, RDF::DOAP.description, \"CSV validator\"]\n @graph << [CSVLINT, RDF::DOAP.homepage, RDF::Resource.new(\"https://github.com/theodi/csvlint.rb\")]\n @graph << [CSVLINT, RDF::DOAP.license, RDF::Resource.new(\"https://raw.githubusercontent.com/theodi/csvlint.rb/master/LICENSE.md\")]\n @graph << [CSVLINT, RDF::DOAP[\"programming-language\"], \"Ruby\"]\n @graph << [CSVLINT, RDF::DOAP.implements, RDF::Resource.new(\"http://www.w3.org/TR/tabular-data-model/\")]\n @graph << [CSVLINT, RDF::DOAP.implements, RDF::Resource.new(\"http://www.w3.org/TR/tabular-metadata/\")]\n @graph << [CSVLINT, RDF::DOAP.developer, ODI]\n @graph << [CSVLINT, RDF::DOAP.maintainer, ODI]\n @graph << [CSVLINT, RDF::DOAP.documenter, ODI]\n @graph << [CSVLINT, RDF::FOAF.maker, ODI]\n @graph << [CSVLINT, RDF::DC.creator, ODI]\n @graph << [output, RDF::FOAF[\"primaryTopic\"], CSVLINT]\n @graph << [output, RDF::DC.issued, DateTime.now]\n @graph << [output, RDF::FOAF.maker, ODI]\n @graph << [ODI, RDF.type, RDF::FOAF.Organization]\n @graph << [ODI, RDF.type, EARL.Assertor]\n @graph << [ODI, RDF::FOAF.name, \"Open Data Institute\"]\n @graph << [ODI, RDF::FOAF.homepage, \"https://theodi.org/\"]\n end\n\n def scenario_name(keyword, name, file_colon_line, source_indent)\n @test = RDF::Resource.new(\"http://www.w3.org/2013/csvw/tests/#{name.split(\" \")[0]}\")\n end\n\n def after_steps(steps)\n passed = true\n steps.each do |s|\n passed = false unless s.status == :passed\n end\n a = RDF::Node.new\n @graph << [a, RDF.type, EARL.Assertion]\n @graph << [a, EARL.assertedBy, ODI]\n @graph << [a, EARL.subject, CSVLINT]\n @graph << [a, EARL.test, @test]\n @graph << [a, EARL.mode, EARL.automatic]\n r = RDF::Node.new\n @graph << [a, EARL.result, r]\n @graph << [r, RDF.type, EARL.TestResult]\n @graph << [r, EARL.outcome, passed ? EARL.passed : EARL.failed]\n @graph << [r, RDF::DC.date, DateTime.now]\n end\n\n def after_features(features)\n RDF::Writer.for(:ttl).open(\"csvlint-earl.ttl\", {prefixes: {\"earl\" => EARL}, standard_prefixes: true, canonicalize: true, literal_shorthand: true}) do |writer|\n writer << @graph\n end\n end\n\n private\n\n EARL = RDF::Vocabulary.new(\"http://www.w3.org/ns/earl#\")\n ODI = RDF::Resource.new(\"https://theodi.org/\")\n CSVLINT = RDF::Resource.new(\"https://github.com/theodi/csvlint.rb\")\nend\n" }, { "path": "features/support/env.rb", "content": "require \"coveralls\"\nCoveralls.wear_merged!(\"test_frameworks\")\n\n$:.unshift File.join(File.dirname(__FILE__), \"..\", \"..\", \"lib\")\n\nrequire \"rspec/expectations\"\nrequire \"cucumber/rspec/doubles\"\nrequire \"csvlint\"\nrequire \"byebug\"\n\nrequire \"spork\"\n\nSpork.each_run do\n require \"csvlint\"\nend\n\nclass CustomWorld\n def default_csv_options\n {}\n end\nend\n\nWorld do\n CustomWorld.new\nend\n" }, { "path": "features/support/load_tests.rb", "content": "require \"json\"\nrequire \"open-uri\"\nrequire \"uri\"\n\nBASE_URI = \"https://w3c.github.io/csvw/tests/\"\nBASE_PATH = File.join(File.dirname(__FILE__), \"..\", \"fixtures\", \"csvw\")\nFEATURE_BASE_PATH = File.join(File.dirname(__FILE__), \"..\")\nVALIDATION_FEATURE_FILE_PATH = File.join(FEATURE_BASE_PATH, \"csvw_validation_tests.feature\")\nSCRIPT_FILE_PATH = File.join(File.dirname(__FILE__), \"..\", \"..\", \"bin\", \"run-csvw-tests\")\n\nDir.mkdir(BASE_PATH) unless Dir.exist?(BASE_PATH)\n\ndef cache_file(filename)\n file = File.join(BASE_PATH, filename)\n uri = URI.join(BASE_URI, filename)\n unless File.exist?(file)\n if filename.include? \"/\"\n levels = filename.split(\"/\")[0..-2]\n (0..levels.length).each do |i|\n dir = File.join(BASE_PATH, levels[0..i].join(\"/\"))\n Dir.mkdir(dir) unless Dir.exist?(dir)\n end\n end\n warn(\"storing #{file} locally\")\n File.open(file, \"wb\") do |f|\n f.puts URI.open(uri, \"rb\").read\n end\n end\n uri\nend\n\nunless File.exist? SCRIPT_FILE_PATH\n File.open(SCRIPT_FILE_PATH, \"w\") do |file|\n File.chmod(0o755, SCRIPT_FILE_PATH)\n manifest = JSON.parse(URI.open(\"#{BASE_URI}manifest-validation.jsonld\").read)\n manifest[\"entries\"].each do |entry|\n type = \"valid\"\n case entry[\"type\"]\n when \"csvt:WarningValidationTest\"\n type = \"warnings\"\n when \"csvt:NegativeValidationTest\"\n type = \"errors\"\n end\n file.puts \"echo \\\"#{entry[\"id\"].split(\"#\")[-1]}: #{entry[\"name\"].tr(\"`\", \"'\")}\\\"\"\n file.puts \"echo \\\"#{type}: #{entry[\"comment\"].gsub(\"\\\"\", \"\\\\\\\"\").tr(\"`\", \"'\")}\\\"\"\n if entry[\"action\"].end_with?(\".json\")\n file.puts \"csvlint --schema=features/fixtures/csvw/#{entry[\"action\"]}\"\n elsif entry[\"option\"] && entry[\"option\"][\"metadata\"]\n file.puts \"csvlint features/fixtures/csvw/#{entry[\"action\"]} --schema=features/fixtures/csvw/#{entry[\"option\"][\"metadata\"]}\"\n else\n file.puts \"csvlint features/fixtures/csvw/#{entry[\"action\"]}\"\n end\n file.puts \"echo\"\n end\n end\nend\n\nunless File.exist? VALIDATION_FEATURE_FILE_PATH\n File.open(VALIDATION_FEATURE_FILE_PATH, \"w\") do |file|\n file.puts \"# Auto-generated file based on standard validation CSVW tests from #{BASE_URI}manifest-validation.jsonld\"\n file.puts \"\"\n\n manifest = JSON.parse(URI.open(\"#{BASE_URI}manifest-validation.jsonld\").read)\n\n file.puts \"Feature: #{manifest[\"label\"]}\"\n file.puts \"\"\n\n manifest[\"entries\"].each do |entry|\n action_uri = cache_file(entry[\"action\"])\n metadata = nil\n provided_files = []\n missing_files = []\n file.puts \"\\t# #{entry[\"id\"]}\"\n file.puts \"\\t# #{entry[\"comment\"]}\"\n file.puts \"\\tScenario: #{entry[\"id\"]} #{entry[\"name\"].gsub(\"<\", \"less than\")}\"\n if entry[\"action\"].end_with?(\".json\")\n file.puts \"\\t\\tGiven I have a metadata file called \\\"csvw/#{entry[\"action\"]}\\\"\"\n file.puts \"\\t\\tAnd the metadata is stored at the url \\\"#{action_uri}\\\"\"\n else\n file.puts \"\\t\\tGiven I have a CSV file called \\\"csvw/#{entry[\"action\"]}\\\"\"\n file.puts \"\\t\\tAnd it has a Link header holding \\\"#{entry[\"httpLink\"]}\\\"\" if entry[\"httpLink\"]\n file.puts \"\\t\\tAnd it is stored at the url \\\"#{action_uri}\\\"\"\n if entry[\"option\"] && entry[\"option\"][\"metadata\"]\n # no need to store the file here, as it will be listed in the 'implicit' list, which all get stored\n metadata = URI.join(BASE_URI, entry[\"option\"][\"metadata\"])\n file.puts \"\\t\\tAnd I have a metadata file called \\\"csvw/#{entry[\"option\"][\"metadata\"]}\\\"\"\n file.puts \"\\t\\tAnd the metadata is stored at the url \\\"#{metadata}\\\"\"\n end\n provided_files << action_uri.to_s\n if entry[\"name\"].include?(\"/.well-known/csvm\")\n file.puts \"\\t\\tAnd I have a file called \\\"w3.org/.well-known/csvm\\\" at the url \\\"https://www.w3.org/.well-known/csvm\\\"\"\n missing_files << \"#{action_uri}.json\"\n missing_files << URI.join(action_uri, \"csvm.json\").to_s\n else\n missing_files << URI.join(action_uri, \"/.well-known/csvm\").to_s\n end\n missing_files << \"#{action_uri}-metadata.json\"\n missing_files << URI.join(action_uri, \"csv-metadata.json\").to_s\n end\n entry[\"implicit\"]&.each do |implicit|\n implicit_uri = cache_file(implicit)\n provided_files << implicit_uri.to_s\n unless implicit_uri == metadata\n file.puts \"\\t\\tAnd I have a file called \\\"csvw/#{implicit}\\\" at the url \\\"#{implicit_uri}\\\"\"\n end\n end\n missing_files.each do |uri|\n file.puts \"\\t\\tAnd there is no file at the url \\\"#{uri}\\\"\" unless provided_files.include? uri\n end\n file.puts \"\\t\\tWhen I carry out CSVW validation\"\n if entry[\"type\"] == \"csvt:WarningValidationTest\"\n file.puts \"\\t\\tThen there should not be errors\"\n file.puts \"\\t\\tAnd there should be warnings\"\n elsif entry[\"type\"] == \"csvt:NegativeValidationTest\"\n file.puts \"\\t\\tThen there should be errors\"\n else\n file.puts \"\\t\\tThen there should not be errors\"\n file.puts \"\\t\\tAnd there should not be warnings\"\n end\n file.puts \"\\t\"\n end\n end\nend\n" }, { "path": "features/support/webmock.rb", "content": "require \"webmock/cucumber\"\n\nWebMock.disable_net_connect!(allow: %r{csvw/tests})\n" }, { "path": "features/validation_errors.feature", "content": "Feature: Get validation errors\n\n Scenario: CSV with ragged rows\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"1\",\"2\",\"3\"\n\"4\",\"5\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"ragged_rows\"\n And that error should have the row \"3\"\n And that error should have the content \"\"4\",\"5\"\"\n\n Scenario: CSV with incorrect quoting\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\",\"Baz\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"unclosed_quote\"\n And that error should have the row \"2\"\n And that error should have the content \"\"Foo\",\"Bar\",\"Baz\"\n\n Scenario: Successfully report a CSV with incorrect whitespace\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\", \"Baz\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"whitespace\"\n And that error should have the row \"2\"\n And that error should have the content \"\"Foo\",\"Bar\", \"Baz\"\"\n\n Scenario: Successfully report a CSV with blank rows\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\",\"Baz\"\n\"\",\"\",\n\"Baz\",\"Bar\",\"Foo\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"blank_rows\"\n And that error should have the row \"3\"\n And that error should have the content \"\"\",\"\",\"\n\n Scenario: Successfully report a CSV with multiple trailing empty rows\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\",\"Baz\"\n\"Foo\",\"Bar\",\"Baz\"\n\n\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"blank_rows\"\n And that error should have the row \"4\"\n\n Scenario: Successfully report a CSV with an empty row\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"Foo\",\"Bar\",\"Baz\"\n\n\"Foo\",\"Bar\",\"Baz\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"blank_rows\"\n And that error should have the row \"3\"\n\n Scenario: Report invalid Encoding\n Given I have a CSV file called \"invalid-byte-sequence.csv\"\n And I set an encoding header of \"UTF-8\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"invalid_encoding\"\n\n Scenario: Correctly handle different encodings\n Given I have a CSV file called \"invalid-byte-sequence.csv\"\n And I set an encoding header of \"ISO-8859-1\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be no \"content_encoding\" errors\n\n Scenario: Report invalid file\n Given I have a CSV file called \"spreadsheet.xls\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"invalid_encoding\"\n\n Scenario: Incorrect extension\n Given I have a CSV with the following content:\n \"\"\"\n\"abc\",\"2\",\"3\"\n \"\"\"\n And the content type is set to \"application/excel\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"wrong_content_type\"\n\n Scenario: Handles urls that 404\n Given I have a CSV that doesn't exist\n When I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"not_found\"\n\n Scenario: Incorrect line endings specified in settings\n Given I have a CSV file called \"cr-line-endings.csv\"\n And I set the line endings to linefeed\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"line_breaks\"\n\n Scenario: inconsistent line endings in file cause an error\n Given I have a CSV file called \"inconsistent-line-endings.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"line_breaks\"\n\n\n Scenario: inconsistent line endings with unquoted fields in file cause an error\n Given I have a CSV file called \"inconsistent-line-endings-unquoted.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are errors\n Then there should be 1 error\n And that error should have the type \"line_breaks\"\n" }, { "path": "features/validation_info.feature", "content": "Feature: Get validation information messages\n\n Scenario: LF line endings in file give an info message\n Given I have a CSV file called \"lf-line-endings.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I set header to \"true\"\n And I ask if there are info messages\n Then there should be 1 info messages\n And one of the messages should have the type \"nonrfc_line_breaks\"\n\n Scenario: CRLF line endings in file produces no info messages\n Given I have a CSV file called \"crlf-line-endings.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I set header to \"true\"\n And I ask if there are info messages\n Then there should be 0 info messages\n" }, { "path": "features/validation_warnings.feature", "content": "Feature: Validation warnings\n\n Scenario: UTF-8 Encoding\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"abc\",\"2\",\"3\"\n \"\"\"\n And it is encoded as \"utf-8\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are warnings\n Then there should be 0 warnings\n\n Scenario: ISO-8859-1 Encoding\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"1\",\"2\",\"3\"\n \"\"\"\n And it is encoded as \"iso-8859-1\"\n And it is stored at the url \"http://example.com/example1.csv\"\n When I ask if there are warnings\n Then there should be 1 warnings\n\n Scenario: Correct content type\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"abc\",\"2\",\"3\"\n \"\"\"\n And the content type is set to \"text/csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are warnings\n Then there should be 0 warnings\n\n Scenario: No extension\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"abc\",\"2\",\"3\"\n \"\"\"\n And the content type is set to \"text/csv\"\n And it is stored at the url \"http://example.com/example1\"\n And I ask if there are warnings\n Then there should be 0 warnings\n\n Scenario: Allow query params after extension\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"abc\",\"2\",\"3\"\n \"\"\"\n And the content type is set to \"text/csv\"\n And it is stored at the url \"http://example.com/example1.csv?query=param\"\n And I ask if there are warnings\n Then there should be 0 warnings\n\n Scenario: User doesn't supply encoding\n Given I have a CSV with the following content:\n \"\"\"\n\"col1\",\"col2\",\"col3\"\n\"abc\",\"2\",\"3\"\n \"\"\"\n And it is stored at the url \"http://example.com/example1.csv\" with no character set\n When I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"no_encoding\"\n\n Scenario: Title rows\n Given I have a CSV file called \"title-row.csv\"\n And it is stored at the url \"http://example.com/example1.csv\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"title_row\"\n\n Scenario: catch excel warnings\n Given I parse a file called \"spreadsheet.xls\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"excel\"\n\n Scenario: catch excel warnings\n Given I parse a file called \"spreadsheet.xlsx\"\n And I ask if there are warnings\n Then there should be 1 warnings\n And that warning should have the type \"excel\"\n" }, { "path": "gemfiles/activesupport_5.2.gemfile", "content": "# This file was generated by Appraisal\n\nsource \"https://rubygems.org\"\n\ngem \"activesupport\", \"~> 5.2.0\"\n\ngemspec path: \"../\"\n" }, { "path": "gemfiles/activesupport_6.0.gemfile", "content": "# This file was generated by Appraisal\n\nsource \"https://rubygems.org\"\n\ngem \"activesupport\", \"~> 6.0.0\"\n\ngemspec path: \"../\"\n" }, { "path": "gemfiles/activesupport_6.1.gemfile", "content": "# This file was generated by Appraisal\n\nsource \"https://rubygems.org\"\n\ngem \"activesupport\", \"~> 6.1.0\"\n\ngemspec path: \"../\"\n" }, { "path": "gemfiles/activesupport_7.0.gemfile", "content": "# This file was generated by Appraisal\n\nsource \"https://rubygems.org\"\n\ngem \"activesupport\", \"~> 7.0.0\"\n\ngemspec path: \"../\"\n" }, { "path": "gemfiles/activesupport_7.1.gemfile", "content": "# This file was generated by Appraisal\n\nsource \"https://rubygems.org\"\n\ngem \"activesupport\", \"~> 7.1.0\"\n\ngemspec path: \"../\"\n" }, { "path": "gemfiles/activesupport_7.2.gemfile", "content": "# This file was generated by Appraisal\n\nsource \"https://rubygems.org\"\n\ngem \"activesupport\", \"~> 7.2.0\"\n\ngemspec path: \"../\"\n" }, { "path": "lib/csvlint/cli.rb", "content": "require \"csvlint\"\nrequire \"rainbow\"\nrequire \"active_support/json\"\nrequire \"json\"\nrequire \"thor\"\n\nrequire \"active_support/inflector\"\n\nmodule Csvlint\n class Cli < Thor\n desc \"myfile.csv OR csvlint http://example.com/myfile.csv\", \"Supports validating CSV files to check their syntax and contents\"\n\n option :dump_errors, desc: \"Pretty print error and warning objects.\", type: :boolean, aliases: :d\n option :schema, banner: \"FILENAME OR URL\", desc: \"Schema file\", aliases: :s\n option :json, desc: \"Output errors as JSON\", type: :boolean, aliases: :j\n option :werror, desc: \"Make all warnings into errors\", type: :boolean, aliases: :w\n\n def validate(source = nil)\n source = read_source(source)\n @schema = get_schema(options[:schema]) if options[:schema]\n fetch_schema_tables(@schema, options) if source.nil?\n\n Rainbow.enabled = $stdout.tty?\n\n valid = validate_csv(source, @schema, options[:dump_errors], options[:json], options[:werror])\n exit 1 unless valid\n end\n\n def help\n self.class.command_help(shell, :validate)\n end\n\n default_task :validate\n\n private\n\n def read_source(source)\n if source.nil?\n # If no source is present, try reading from stdin\n if !$stdin.tty?\n source = begin\n StringIO.new($stdin.read)\n rescue\n nil\n end\n return_error \"No CSV data to validate\" if !options[:schema] && source.nil?\n end\n else\n # If the source isn't a URL, it's a file\n unless /^http(s)?/.match?(source)\n begin\n source = File.new(source)\n rescue Errno::ENOENT\n return_error \"#{source} not found\"\n end\n end\n end\n\n source\n end\n\n def get_schema(schema)\n begin\n schema = Csvlint::Schema.load_from_uri(schema, false)\n rescue Csvlint::Csvw::MetadataError => e\n return_error \"invalid metadata: #{e.message}#{\" at \" + e.path if e.path}\"\n rescue OpenURI::HTTPError, Errno::ENOENT\n return_error \"#{options[:schema]} not found\"\n end\n\n if schema.instance_of?(Csvlint::Schema) && schema.description == \"malformed\"\n return_error \"invalid metadata: malformed JSON\"\n end\n\n schema\n end\n\n def fetch_schema_tables(schema, options)\n valid = true\n\n unless schema.instance_of? Csvlint::Csvw::TableGroup\n return_error \"No CSV data to validate.\"\n end\n schema.tables.keys.each do |source|\n unless /^http(s)?/.match?(source)\n begin\n source = source.sub(\"file:\", \"\")\n source = File.new(source)\n rescue Errno::ENOENT\n return_error \"#{source} not found\"\n end\n end\n valid &= validate_csv(source, schema, options[:dump_errors], nil, options[:werror])\n end\n\n exit 1 unless valid\n end\n\n def print_error(index, error, dump, color)\n location = \"\"\n location += error.row.to_s if error.row\n location += \"#{error.row ? \",\" : \"\"}#{error.column}\" if error.column\n if error.row || error.column\n location = \"#{error.row ? \"Row\" : \"Column\"}: #{location}\"\n end\n output_string = \"#{index + 1}. \"\n if error.column && @schema&.instance_of?(Csvlint::Schema)\n unless @schema.fields[error.column - 1].nil?\n output_string += \"#{@schema.fields[error.column - 1].name}: \"\n end\n end\n output_string += error.type.to_s\n output_string += \". #{location}\" unless location.empty?\n output_string += \". #{error.content}\" if error.content\n\n puts Rainbow(output_string).color(color)\n\n if dump\n pp error\n end\n end\n\n def print_errors(errors, dump)\n if errors.size > 0\n errors.each_with_index { |error, i| print_error(i, error, dump, :red) }\n end\n end\n\n def return_error(message)\n puts Rainbow(message).red\n exit 1\n end\n\n def validate_csv(source, schema, dump, json, werror)\n @error_count = 0\n\n validator = if json === true\n Csvlint::Validator.new(source, {}, schema)\n else\n Csvlint::Validator.new(source, {}, schema, {lambda: report_lines})\n end\n\n csv = if source.instance_of?(String)\n source\n elsif source.instance_of?(File)\n source.path\n else\n \"CSV\"\n end\n\n if json === true\n json = {\n validation: {\n state: validator.valid? ? \"valid\" : \"invalid\",\n errors: validator.errors.map { |v| hashify(v) },\n warnings: validator.warnings.map { |v| hashify(v) },\n info: validator.info_messages.map { |v| hashify(v) }\n }\n }.to_json\n print json\n else\n puts \"\\r\\n#{csv} is #{validator.valid? ? Rainbow(\"VALID\").green : Rainbow(\"INVALID\").red}\"\n print_errors(validator.errors, dump)\n print_errors(validator.warnings, dump)\n end\n\n return false if werror && validator.warnings.size > 0\n validator.valid?\n end\n\n def hashify(error)\n h = {\n type: error.type,\n category: error.category,\n row: error.row,\n col: error.column\n }\n\n if error.column && @schema&.instance_of?(Csvlint::Schema) && !@schema.fields[error.column - 1].nil?\n field = @schema.fields[error.column - 1]\n h[:header] = field.name\n h[:constraints] = field.constraints.map { |k, v| [k.underscore, v] }.to_h\n end\n\n h\n end\n\n def report_lines\n lambda do |row|\n new_errors = row.errors.count\n if new_errors > @error_count\n print Rainbow(\"!\").red\n else\n print Rainbow(\".\").green\n end\n @error_count = new_errors\n end\n end\n end\nend\n" }, { "path": "lib/csvlint/csvw/column.rb", "content": "module Csvlint\n module Csvw\n class Column\n include Csvlint::ErrorCollector\n\n attr_reader :id, :about_url, :datatype, :default, :lang, :name, :null, :number, :ordered, :property_url, :required, :separator, :source_number, :suppress_output, :text_direction, :default_name, :titles, :value_url, :virtual, :annotations\n\n def initialize(number, name, id: nil, about_url: nil, datatype: {\"@id\" => \"http://www.w3.org/2001/XMLSchema#string\"}, default: \"\", lang: \"und\", null: [\"\"], ordered: false, property_url: nil, required: false, separator: nil, source_number: nil, suppress_output: false, text_direction: :inherit, default_name: nil, titles: {}, value_url: nil, virtual: false, annotations: [], warnings: [])\n @number = number\n @name = name\n @id = id\n @about_url = about_url\n @datatype = datatype\n @default = default\n @lang = lang\n @null = null\n @ordered = ordered\n @property_url = property_url\n @required = required\n @separator = separator\n @source_number = source_number || number\n @suppress_output = suppress_output\n @text_direction = text_direction\n @default_name = default_name\n @titles = titles\n @value_url = value_url\n @virtual = virtual\n @annotations = annotations\n reset\n @warnings += warnings\n end\n\n def self.from_json(number, column_desc, base_url = nil, lang = \"und\", inherited_properties = {})\n annotations = {}\n warnings = []\n column_properties = {}\n inherited_properties = inherited_properties.clone\n\n column_desc.each do |property, value|\n if property == \"@type\"\n raise Csvlint::Csvw::MetadataError.new(\"columns[#{number}].@type\"), \"@type of column is not 'Column'\" if value != \"Column\"\n else\n v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)\n warnings += Array(warning).map { |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, \"#{property}: #{value}\", nil) } unless warning.nil? || warning.empty?\n if type == :annotation\n annotations[property] = v\n elsif type == :common || type == :column\n column_properties[property] = v\n elsif type == :inherited\n inherited_properties[property] = v\n else\n warnings << Csvlint::ErrorMessage.new(:invalid_property, :metadata, nil, nil, \"column: #{property}\", nil)\n end\n end\n end\n\n new(number, column_properties[\"name\"],\n id: column_properties[\"@id\"],\n datatype: inherited_properties[\"datatype\"] || {\"@id\" => \"http://www.w3.org/2001/XMLSchema#string\"},\n lang: inherited_properties[\"lang\"] || \"und\",\n null: inherited_properties[\"null\"] || [\"\"],\n default: inherited_properties[\"default\"] || \"\",\n about_url: inherited_properties[\"aboutUrl\"],\n property_url: inherited_properties[\"propertyUrl\"],\n value_url: inherited_properties[\"valueUrl\"],\n required: inherited_properties[\"required\"] || false,\n separator: inherited_properties[\"separator\"],\n ordered: inherited_properties[\"ordered\"] || false,\n default_name: column_properties[\"titles\"] && column_properties[\"titles\"][lang] ? column_properties[\"titles\"][lang][0] : nil,\n titles: column_properties[\"titles\"],\n suppress_output: column_properties[\"suppressOutput\"] || false,\n virtual: column_properties[\"virtual\"] || false,\n annotations: annotations,\n warnings: warnings)\n end\n\n def validate_header(header, strict)\n reset\n if strict || @titles\n valid_headers = @titles ? @titles.map { |l, v| v if Column.languages_match(l, lang) }.flatten : []\n unless valid_headers.include? header\n if strict\n build_errors(:invalid_header, :schema, 1, @number, header, @titles)\n else\n build_warnings(:invalid_header, :schema, 1, @number, header, @titles)\n end\n end\n end\n valid?\n end\n\n def validate(string_value, row = nil)\n reset\n string_value ||= @default\n if null.include? string_value\n validate_required(nil, row)\n nil\n\n else\n string_values = @separator.nil? ? [string_value] : string_value.split(@separator)\n values = []\n string_values.each do |s|\n invalid = false\n value, warning = DATATYPE_PARSER[@datatype[\"base\"] || @datatype[\"@id\"]].call(s, @datatype[\"format\"])\n if warning.nil?\n validate_required(value, row)\n invalid = !validate_format(value, row) || invalid\n invalid = !validate_length(value, row) || invalid\n invalid = !validate_value(value, row) || invalid\n values << (invalid ? {invalid: s} : value)\n else\n build_errors(warning, :schema, row, @number, s, @datatype)\n values << {invalid: s}\n end\n end\n values && @separator.nil? ? values[0] : values\n\n end\n end\n\n private\n\n class << self\n def create_date_parser(type, warning)\n lambda { |value, format|\n format = Csvlint::Csvw::DateFormat.new(nil, type) if format.nil?\n v = format.parse(value)\n return nil, warning if v.nil?\n [v, nil]\n }\n end\n\n def create_regexp_based_parser(regexp, warning)\n lambda { |value, format|\n return nil, warning unless value&.match?(regexp)\n [value, nil]\n }\n end\n\n def languages_match(l1, l2)\n return true if l1 == l2 || l1 == \"und\" || l2 == \"und\"\n return true if l1 =~ Regexp.new(\"^#{l2}-\") || l2 =~ Regexp.new(\"^#{l1}-\")\n false\n end\n end\n\n def validate_required(value, row)\n if @required && value.nil?\n build_errors(:required, :schema, row, number, value, {\"required\" => @required})\n return false\n end\n true\n end\n\n def validate_length(value, row)\n valid = true\n if datatype[\"length\"] || datatype[\"minLength\"] || datatype[\"maxLength\"]\n length = value.length\n length = value.gsub(/==?$/, \"\").length * 3 / 4 if datatype[\"@id\"] == \"http://www.w3.org/2001/XMLSchema#base64Binary\" || datatype[\"base\"] == \"http://www.w3.org/2001/XMLSchema#base64Binary\"\n length = value.length / 2 if datatype[\"@id\"] == \"http://www.w3.org/2001/XMLSchema#hexBinary\" || datatype[\"base\"] == \"http://www.w3.org/2001/XMLSchema#hexBinary\"\n\n if datatype[\"minLength\"] && length < datatype[\"minLength\"]\n build_errors(:min_length, :schema, row, number, value, {\"minLength\" => datatype[\"minLength\"]})\n valid = false\n end\n if datatype[\"maxLength\"] && length > datatype[\"maxLength\"]\n build_errors(:max_length, :schema, row, number, value, {\"maxLength\" => datatype[\"maxLength\"]})\n valid = false\n end\n if datatype[\"length\"] && length != datatype[\"length\"]\n build_errors(:length, :schema, row, number, value, {\"length\" => datatype[\"length\"]})\n valid = false\n end\n end\n valid\n end\n\n def validate_format(value, row)\n if datatype[\"format\"]\n unless DATATYPE_FORMAT_VALIDATION[datatype[\"base\"]].call(value, datatype[\"format\"])\n build_errors(:format, :schema, row, number, value, {\"format\" => datatype[\"format\"]})\n return false\n end\n end\n true\n end\n\n def validate_value(value, row)\n valid = true\n if datatype[\"minInclusive\"] && ((value.is_a? Hash) ? (value[:dateTime] < datatype[\"minInclusive\"][:dateTime]) : (value < datatype[\"minInclusive\"]))\n build_errors(:min_inclusive, :schema, row, number, value, {\"minInclusive\" => datatype[\"minInclusive\"]})\n valid = false\n end\n if datatype[\"maxInclusive\"] && ((value.is_a? Hash) ? (value[:dateTime] > datatype[\"maxInclusive\"][:dateTime]) : (value > datatype[\"maxInclusive\"]))\n build_errors(:max_inclusive, :schema, row, number, value, {\"maxInclusive\" => datatype[\"maxInclusive\"]})\n valid = false\n end\n if datatype[\"minExclusive\"] && ((value.is_a? Hash) ? (value[:dateTime] <= datatype[\"minExclusive\"][:dateTime]) : (value <= datatype[\"minExclusive\"]))\n build_errors(:min_exclusive, :schema, row, number, value, {\"minExclusive\" => datatype[\"minExclusive\"]})\n valid = false\n end\n if datatype[\"maxExclusive\"] && ((value.is_a? Hash) ? (value[:dateTime] >= datatype[\"maxExclusive\"][:dateTime]) : (value >= datatype[\"maxExclusive\"]))\n build_errors(:max_exclusive, :schema, row, number, value, {\"maxExclusive\" => datatype[\"maxExclusive\"]})\n valid = false\n end\n valid\n end\n\n REGEXP_VALIDATION = lambda { |value, format| value =~ format }\n\n NO_ADDITIONAL_VALIDATION = lambda { |value, format| true }\n\n DATATYPE_FORMAT_VALIDATION = {\n \"http://www.w3.org/1999/02/22-rdf-syntax-ns#XMLLiteral\" => REGEXP_VALIDATION,\n \"http://www.w3.org/1999/02/22-rdf-syntax-ns#HTML\" => REGEXP_VALIDATION,\n \"http://www.w3.org/ns/csvw#JSON\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#anyAtomicType\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#anyURI\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#base64Binary\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#boolean\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#date\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#dateTime\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#dateTimeStamp\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#decimal\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#integer\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#long\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#int\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#short\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#byte\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#nonNegativeInteger\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#positiveInteger\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#unsignedLong\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#unsignedInt\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#unsignedShort\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#unsignedByte\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#nonPositiveInteger\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#negativeInteger\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#double\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#duration\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#dayTimeDuration\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#yearMonthDuration\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#float\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#gDay\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#gMonth\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#gMonthDay\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#gYear\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#gYearMonth\" => NO_ADDITIONAL_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#hexBinary\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#QName\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#string\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#normalizedString\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#token\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#language\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#Name\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#NMTOKEN\" => REGEXP_VALIDATION,\n \"http://www.w3.org/2001/XMLSchema#time\" => NO_ADDITIONAL_VALIDATION\n }\n\n TRIM_VALUE = lambda { |value, format| [value.strip, nil] }\n ALL_VALUES_VALID = lambda { |value, format| [value, nil] }\n\n NUMERIC_PARSER = lambda { |value, format, integer = false|\n format = Csvlint::Csvw::NumberFormat.new(nil, nil, \".\", integer) if format.nil?\n v = format.parse(value)\n return nil, :invalid_number if v.nil?\n [v, nil]\n }\n\n DATATYPE_PARSER = {\n \"http://www.w3.org/1999/02/22-rdf-syntax-ns#XMLLiteral\" => TRIM_VALUE,\n \"http://www.w3.org/1999/02/22-rdf-syntax-ns#HTML\" => TRIM_VALUE,\n \"http://www.w3.org/ns/csvw#JSON\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#anyAtomicType\" => ALL_VALUES_VALID,\n \"http://www.w3.org/2001/XMLSchema#anyURI\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#base64Binary\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#boolean\" => lambda { |value, format|\n if format.nil?\n return true, nil if [\"true\", \"1\"].include? value\n return false, nil if [\"false\", \"0\"].include? value\n else\n return true, nil if value == format[0]\n return false, nil if value == format[1]\n end\n [value, :invalid_boolean]\n },\n \"http://www.w3.org/2001/XMLSchema#date\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#date\", :invalid_date),\n \"http://www.w3.org/2001/XMLSchema#dateTime\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#dateTime\", :invalid_date_time),\n \"http://www.w3.org/2001/XMLSchema#dateTimeStamp\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#dateTimeStamp\", :invalid_date_time_stamp),\n \"http://www.w3.org/2001/XMLSchema#decimal\" => lambda { |value, format|\n return nil, :invalid_decimal if /(E|e|^(NaN|INF|-INF)$)/.match?(value)\n NUMERIC_PARSER.call(value, format)\n },\n \"http://www.w3.org/2001/XMLSchema#integer\" => lambda { |value, format|\n v, w = NUMERIC_PARSER.call(value, format, true)\n return v, :invalid_integer unless w.nil?\n return nil, :invalid_integer unless v.is_a? Integer\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#long\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#integer\"].call(value, format)\n return v, :invalid_long unless w.nil?\n return nil, :invalid_long unless v <= 9223372036854775807 && v >= -9223372036854775808\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#int\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#integer\"].call(value, format)\n return v, :invalid_int unless w.nil?\n return nil, :invalid_int unless v <= 2147483647 && v >= -2147483648\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#short\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#integer\"].call(value, format)\n return v, :invalid_short unless w.nil?\n return nil, :invalid_short unless v <= 32767 && v >= -32768\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#byte\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#integer\"].call(value, format)\n return v, :invalid_byte unless w.nil?\n return nil, :invalid_byte unless v <= 127 && v >= -128\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#nonNegativeInteger\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#integer\"].call(value, format)\n return v, :invalid_nonNegativeInteger unless w.nil?\n return nil, :invalid_nonNegativeInteger unless v >= 0\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#positiveInteger\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#integer\"].call(value, format)\n return v, :invalid_positiveInteger unless w.nil?\n return nil, :invalid_positiveInteger unless v > 0\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#unsignedLong\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#nonNegativeInteger\"].call(value, format)\n return v, :invalid_unsignedLong unless w.nil?\n return nil, :invalid_unsignedLong unless v <= 18446744073709551615\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#unsignedInt\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#nonNegativeInteger\"].call(value, format)\n return v, :invalid_unsignedInt unless w.nil?\n return nil, :invalid_unsignedInt unless v <= 4294967295\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#unsignedShort\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#nonNegativeInteger\"].call(value, format)\n return v, :invalid_unsignedShort unless w.nil?\n return nil, :invalid_unsignedShort unless v <= 65535\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#unsignedByte\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#nonNegativeInteger\"].call(value, format)\n return v, :invalid_unsignedByte unless w.nil?\n return nil, :invalid_unsignedByte unless v <= 255\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#nonPositiveInteger\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#integer\"].call(value, format)\n return v, :invalid_nonPositiveInteger unless w.nil?\n return nil, :invalid_nonPositiveInteger unless v <= 0\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#negativeInteger\" => lambda { |value, format|\n v, w = DATATYPE_PARSER[\"http://www.w3.org/2001/XMLSchema#integer\"].call(value, format)\n return v, :invalid_negativeInteger unless w.nil?\n return nil, :invalid_negativeInteger unless v < 0\n [v, w]\n },\n \"http://www.w3.org/2001/XMLSchema#double\" => NUMERIC_PARSER,\n # regular expressions here taken from XML Schema datatypes spec\n \"http://www.w3.org/2001/XMLSchema#duration\" =>\n create_regexp_based_parser(/-?P((([0-9]+Y([0-9]+M)?([0-9]+D)?|([0-9]+M)([0-9]+D)?|([0-9]+D))(T(([0-9]+H)([0-9]+M)?([0-9]+(\\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\\.[0-9]+)?S)?|([0-9]+(\\.[0-9]+)?S)))?)|(T(([0-9]+H)([0-9]+M)?([0-9]+(\\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\\.[0-9]+)?S)?|([0-9]+(\\.[0-9]+)?S))))/, :invalid_duration),\n \"http://www.w3.org/2001/XMLSchema#dayTimeDuration\" =>\n create_regexp_based_parser(/-?P(([0-9]+D(T(([0-9]+H)([0-9]+M)?([0-9]+(\\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\\.[0-9]+)?S)?|([0-9]+(\\.[0-9]+)?S)))?)|(T(([0-9]+H)([0-9]+M)?([0-9]+(\\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\\.[0-9]+)?S)?|([0-9]+(\\.[0-9]+)?S))))/, :invalid_dayTimeDuration),\n \"http://www.w3.org/2001/XMLSchema#yearMonthDuration\" =>\n create_regexp_based_parser(/-?P([0-9]+Y([0-9]+M)?|([0-9]+M))/, :invalid_duration),\n \"http://www.w3.org/2001/XMLSchema#float\" => NUMERIC_PARSER,\n \"http://www.w3.org/2001/XMLSchema#gDay\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#gDay\", :invalid_gDay),\n \"http://www.w3.org/2001/XMLSchema#gMonth\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#gMonth\", :invalid_gMonth),\n \"http://www.w3.org/2001/XMLSchema#gMonthDay\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#gMonthDay\", :invalid_gMonthDay),\n \"http://www.w3.org/2001/XMLSchema#gYear\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#gYear\", :invalid_gYear),\n \"http://www.w3.org/2001/XMLSchema#gYearMonth\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#gYearMonth\", :invalid_gYearMonth),\n \"http://www.w3.org/2001/XMLSchema#hexBinary\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#QName\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#string\" => ALL_VALUES_VALID,\n \"http://www.w3.org/2001/XMLSchema#normalizedString\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#token\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#language\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#Name\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#NMTOKEN\" => TRIM_VALUE,\n \"http://www.w3.org/2001/XMLSchema#time\" =>\n create_date_parser(\"http://www.w3.org/2001/XMLSchema#time\", :invalid_time)\n }\n end\n end\nend\n" }, { "path": "lib/csvlint/csvw/date_format.rb", "content": "module Csvlint\n module Csvw\n class DateFormat\n attr_reader :pattern\n\n def initialize(pattern, datatype = nil)\n @pattern = pattern\n\n if @pattern.nil?\n @regexp = DEFAULT_REGEXP[datatype]\n @type = datatype\n else\n test_pattern = pattern.clone\n test_pattern.gsub!(/S+/, \"\")\n FIELDS.keys.sort_by { |f| -f.length }.each do |field|\n test_pattern.gsub!(field, \"\")\n end\n raise Csvw::DateFormatError, \"unrecognised date field symbols in date format\" if /[GyYuUrQqMLlwWdDFgEecahHKkjJmsSAzZOvVXx]/.match?(test_pattern)\n\n @regexp = DATE_PATTERN_REGEXP[@pattern]\n @type = @regexp.nil? ? \"http://www.w3.org/2001/XMLSchema#time\" : \"http://www.w3.org/2001/XMLSchema#date\"\n @regexp ||= TIME_PATTERN_REGEXP[@pattern]\n @type = @regexp.nil? ? \"http://www.w3.org/2001/XMLSchema#dateTime\" : @type\n @regexp ||= DATE_TIME_PATTERN_REGEXP[@pattern]\n\n if @regexp.nil?\n regexp = @pattern\n\n @type = \"http://www.w3.org/2001/XMLSchema#date\" if !(regexp =~ /HH/) && regexp =~ /yyyy/\n @type = \"http://www.w3.org/2001/XMLSchema#time\" if regexp =~ /HH/ && !(regexp =~ /yyyy/)\n @type = \"http://www.w3.org/2001/XMLSchema#dateTime\" if regexp =~ /HH/ && regexp =~ /yyyy/\n\n regexp = regexp.sub(\"HH\", FIELDS[\"HH\"].to_s)\n regexp = regexp.sub(\"mm\", FIELDS[\"mm\"].to_s)\n if /ss\\.S+/.match?(@pattern)\n max_fractional_seconds = @pattern.split(\".\")[-1].length\n regexp = regexp.sub(/ss\\.S+$/, \"(?#{FIELDS[\"ss\"]}(.[0-9]{1,#{max_fractional_seconds}})?)\")\n else\n regexp = regexp.sub(\"ss\", \"(?#{FIELDS[\"ss\"]})\")\n end\n\n if /yyyy/.match?(regexp)\n regexp = regexp.sub(\"yyyy\", FIELDS[\"yyyy\"].to_s)\n regexp = regexp.sub(\"MM\", FIELDS[\"MM\"].to_s)\n regexp = regexp.sub(\"M\", FIELDS[\"M\"].to_s)\n regexp = regexp.sub(\"dd\", FIELDS[\"dd\"].to_s)\n regexp = regexp.sub(/d(?=[-T \\/.])/, FIELDS[\"d\"].to_s)\n end\n\n regexp = regexp.sub(\"XXX\", FIELDS[\"XXX\"].to_s)\n regexp = regexp.sub(\"XX\", FIELDS[\"XX\"].to_s)\n regexp = regexp.sub(\"X\", FIELDS[\"X\"].to_s)\n regexp = regexp.sub(\"xxx\", FIELDS[\"xxx\"].to_s)\n regexp = regexp.sub(\"xx\", FIELDS[\"xx\"].to_s)\n regexp = regexp.sub(/x(?!:)/, FIELDS[\"x\"].to_s)\n\n @regexp = Regexp.new(\"^#{regexp}$\")\n end\n end\n end\n\n def match(value)\n value&.match?(@regexp) ? true : false\n end\n\n def parse(value)\n match = @regexp.match(value)\n return nil if match.nil?\n # STDERR.puts(@regexp)\n # STDERR.puts(value)\n # STDERR.puts(match.inspect)\n value = {}\n match.names.each do |field|\n unless match[field].nil?\n case field\n when \"timezone\"\n tz = match[\"timezone\"]\n tz = \"+00:00\" if tz == \"Z\"\n tz += \":00\" if tz.length == 3\n tz = \"#{tz[0..2]}:#{tz[3..4]}\" unless /:/.match?(tz)\n value[:timezone] = tz\n when \"second\"\n value[:second] = match[\"second\"].to_f\n else\n value[field.to_sym] = match[field].to_i\n end\n end\n end\n case @type\n when \"http://www.w3.org/2001/XMLSchema#date\"\n begin\n value[:dateTime] = Date.new(match[\"year\"].to_i, match[\"month\"].to_i, match[\"day\"].to_i)\n rescue ArgumentError\n return nil\n end\n when \"http://www.w3.org/2001/XMLSchema#dateTime\"\n begin\n value[:dateTime] = DateTime.new(match[\"year\"].to_i, match[\"month\"].to_i, match[\"day\"].to_i, match[\"hour\"].to_i, match[\"minute\"].to_i, (match.names.include?(\"second\") ? match[\"second\"].to_f : 0), (match.names.include?(\"timezone\") && match[\"timezone\"]) ? match[\"timezone\"] : \"\")\n rescue ArgumentError\n return nil\n end\n else\n value[:dateTime] = DateTime.new(value[:year] || 0, value[:month] || 1, value[:day] || 1, value[:hour] || 0, value[:minute] || 0, value[:second] || 0, value[:timezone] || \"+00:00\")\n end\n value[:string] = if value[:year]\n if value[:month]\n if value[:day]\n if value[:hour]\n # dateTime\n \"#{format(\"%04d\", value[:year])}-#{format(\"%02d\", value[:month])}-#{format(\"%02d\", value[:day])}T#{format(\"%02d\", value[:hour])}:#{format(\"%02d\", value[:minute] || 0)}:#{format(\"%02g\", value[:second] || 0)}#{value[:timezone] ? value[:timezone].sub(\"+00:00\", \"Z\") : \"\"}\"\n else\n # date\n \"#{format(\"%04d\", value[:year])}-#{format(\"%02d\", value[:month])}-#{format(\"%02d\", value[:day])}#{value[:timezone] ? value[:timezone].sub(\"+00:00\", \"Z\") : \"\"}\"\n end\n else\n # gYearMonth\n \"#{format(\"%04d\", value[:year])}-#{format(\"%02d\", value[:month])}#{value[:timezone] ? value[:timezone].sub(\"+00:00\", \"Z\") : \"\"}\"\n end\n else\n # gYear\n \"#{format(\"%04d\", value[:year])}#{value[:timezone] ? value[:timezone].sub(\"+00:00\", \"Z\") : \"\"}\"\n end\n elsif value[:month]\n if value[:day]\n # gMonthDay\n \"--#{format(\"%02d\", value[:month])}-#{format(\"%02d\", value[:day])}#{value[:timezone] ? value[:timezone].sub(\"+00:00\", \"Z\") : \"\"}\"\n else\n # gMonth\n \"--#{format(\"%02d\", value[:month])}#{value[:timezone] ? value[:timezone].sub(\"+00:00\", \"Z\") : \"\"}\"\n end\n elsif value[:day]\n # gDay\n \"---#{format(\"%02d\", value[:day])}#{value[:timezone] ? value[:timezone].sub(\"+00:00\", \"Z\") : \"\"}\"\n else\n \"#{format(\"%02d\", value[:hour])}:#{format(\"%02d\", value[:minute])}:#{format(\"%02g\", value[:second] || 0)}#{value[:timezone] ? value[:timezone].sub(\"+00:00\", \"Z\") : \"\"}\"\n end\n value\n end\n\n private\n\n FIELDS = {\n \"yyyy\" => /(?-?([1-9][0-9]{3,}|0[0-9]{3}))/,\n \"MM\" => /(?0[1-9]|1[0-2])/,\n \"M\" => /(?[1-9]|1[0-2])/,\n \"dd\" => /(?0[1-9]|[12][0-9]|3[01])/,\n \"d\" => /(?[1-9]|[12][0-9]|3[01])/,\n \"HH\" => /(?[01][0-9]|2[0-3])/,\n \"mm\" => /(?[0-5][0-9])/,\n \"ss\" => /([0-6][0-9])/,\n \"X\" => /(?Z|[-+]((0[0-9]|1[0-3])([0-5][0-9])?|14(00)?))/,\n \"XX\" => /(?Z|[-+]((0[0-9]|1[0-3])[0-5][0-9]|1400))/,\n \"XXX\" => /(?Z|[-+]((0[0-9]|1[0-3]):[0-5][0-9]|14:00))/,\n \"x\" => /(?[-+]((0[0-9]|1[0-3])([0-5][0-9])?|14(00)?))/,\n \"xx\" => /(?[-+]((0[0-9]|1[0-3])[0-5][0-9]|1400))/,\n \"xxx\" => /(?[-+]((0[0-9]|1[0-3]):[0-5][0-9]|14:00))/\n }\n\n DATE_PATTERN_REGEXP = {\n \"yyyy-MM-dd\" => Regexp.new(\"^#{FIELDS[\"yyyy\"]}-#{FIELDS[\"MM\"]}-#{FIELDS[\"dd\"]}$\"),\n \"yyyyMMdd\" => Regexp.new(\"^#{FIELDS[\"yyyy\"]}#{FIELDS[\"MM\"]}#{FIELDS[\"dd\"]}$\"),\n \"dd-MM-yyyy\" => Regexp.new(\"^#{FIELDS[\"dd\"]}-#{FIELDS[\"MM\"]}-#{FIELDS[\"yyyy\"]}$\"),\n \"d-M-yyyy\" => Regexp.new(\"^#{FIELDS[\"d\"]}-#{FIELDS[\"M\"]}-#{FIELDS[\"yyyy\"]}$\"),\n \"MM-dd-yyyy\" => Regexp.new(\"^#{FIELDS[\"MM\"]}-#{FIELDS[\"dd\"]}-#{FIELDS[\"yyyy\"]}$\"),\n \"M-d-yyyy\" => Regexp.new(\"^#{FIELDS[\"M\"]}-#{FIELDS[\"d\"]}-#{FIELDS[\"yyyy\"]}$\"),\n \"dd/MM/yyyy\" => Regexp.new(\"^#{FIELDS[\"dd\"]}/#{FIELDS[\"MM\"]}/#{FIELDS[\"yyyy\"]}$\"),\n \"d/M/yyyy\" => Regexp.new(\"^#{FIELDS[\"d\"]}/#{FIELDS[\"M\"]}/#{FIELDS[\"yyyy\"]}$\"),\n \"MM/dd/yyyy\" => Regexp.new(\"^#{FIELDS[\"MM\"]}/#{FIELDS[\"dd\"]}/#{FIELDS[\"yyyy\"]}$\"),\n \"M/d/yyyy\" => Regexp.new(\"^#{FIELDS[\"M\"]}/#{FIELDS[\"d\"]}/#{FIELDS[\"yyyy\"]}$\"),\n \"dd.MM.yyyy\" => Regexp.new(\"^#{FIELDS[\"dd\"]}.#{FIELDS[\"MM\"]}.#{FIELDS[\"yyyy\"]}$\"),\n \"d.M.yyyy\" => Regexp.new(\"^#{FIELDS[\"d\"]}.#{FIELDS[\"M\"]}.#{FIELDS[\"yyyy\"]}$\"),\n \"MM.dd.yyyy\" => Regexp.new(\"^#{FIELDS[\"MM\"]}.#{FIELDS[\"dd\"]}.#{FIELDS[\"yyyy\"]}$\"),\n \"M.d.yyyy\" => Regexp.new(\"^#{FIELDS[\"M\"]}.#{FIELDS[\"d\"]}.#{FIELDS[\"yyyy\"]}$\")\n }\n\n TIME_PATTERN_REGEXP = {\n \"HH:mm:ss\" => Regexp.new(\"^#{FIELDS[\"HH\"]}:#{FIELDS[\"mm\"]}:(?#{FIELDS[\"ss\"]})$\"),\n \"HHmmss\" => Regexp.new(\"^#{FIELDS[\"HH\"]}#{FIELDS[\"mm\"]}(?#{FIELDS[\"ss\"]})$\"),\n \"HH:mm\" => Regexp.new(\"^#{FIELDS[\"HH\"]}:#{FIELDS[\"mm\"]}$\"),\n \"HHmm\" => Regexp.new(\"^#{FIELDS[\"HH\"]}#{FIELDS[\"mm\"]}$\")\n }\n\n DATE_TIME_PATTERN_REGEXP = {\n \"yyyy-MM-ddTHH:mm:ss\" => Regexp.new(\"^#{FIELDS[\"yyyy\"]}-#{FIELDS[\"MM\"]}-#{FIELDS[\"dd\"]}T#{FIELDS[\"HH\"]}:#{FIELDS[\"mm\"]}:(?#{FIELDS[\"ss\"]})$\"),\n \"yyyy-MM-ddTHH:mm\" => Regexp.new(\"^#{FIELDS[\"yyyy\"]}-#{FIELDS[\"MM\"]}-#{FIELDS[\"dd\"]}T#{FIELDS[\"HH\"]}:#{FIELDS[\"mm\"]}$\")\n }\n\n DEFAULT_REGEXP = {\n \"http://www.w3.org/2001/XMLSchema#date\" =>\n Regexp.new(\"^#{FIELDS[\"yyyy\"]}-#{FIELDS[\"MM\"]}-#{FIELDS[\"dd\"]}#{FIELDS[\"XXX\"]}?$\"),\n \"http://www.w3.org/2001/XMLSchema#dateTime\" =>\n Regexp.new(\"^#{FIELDS[\"yyyy\"]}-#{FIELDS[\"MM\"]}-#{FIELDS[\"dd\"]}T#{FIELDS[\"HH\"]}:#{FIELDS[\"mm\"]}:(?#{FIELDS[\"ss\"]}(.[0-9]+)?)#{FIELDS[\"XXX\"]}?$\"),\n \"http://www.w3.org/2001/XMLSchema#dateTimeStamp\" =>\n Regexp.new(\"^#{FIELDS[\"yyyy\"]}-#{FIELDS[\"MM\"]}-#{FIELDS[\"dd\"]}T#{FIELDS[\"HH\"]}:#{FIELDS[\"mm\"]}:(?#{FIELDS[\"ss\"]}(.[0-9]+)?)#{FIELDS[\"XXX\"]}$\"),\n \"http://www.w3.org/2001/XMLSchema#gDay\" =>\n Regexp.new(\"^---#{FIELDS[\"dd\"]}#{FIELDS[\"XXX\"]}?$\"),\n \"http://www.w3.org/2001/XMLSchema#gMonth\" =>\n Regexp.new(\"^--#{FIELDS[\"MM\"]}#{FIELDS[\"XXX\"]}?$\"),\n \"http://www.w3.org/2001/XMLSchema#gMonthDay\" =>\n Regexp.new(\"^--#{FIELDS[\"MM\"]}-#{FIELDS[\"dd\"]}#{FIELDS[\"XXX\"]}?$\"),\n \"http://www.w3.org/2001/XMLSchema#gYear\" =>\n Regexp.new(\"^#{FIELDS[\"yyyy\"]}#{FIELDS[\"XXX\"]}?$\"),\n \"http://www.w3.org/2001/XMLSchema#gYearMonth\" =>\n Regexp.new(\"^#{FIELDS[\"yyyy\"]}-#{FIELDS[\"MM\"]}#{FIELDS[\"XXX\"]}?$\"),\n \"http://www.w3.org/2001/XMLSchema#time\" =>\n Regexp.new(\"^#{FIELDS[\"HH\"]}:#{FIELDS[\"mm\"]}:(?#{FIELDS[\"ss\"]}(.[0-9]+)?)#{FIELDS[\"XXX\"]}?$\")\n }\n end\n\n class DateFormatError < StandardError\n end\n end\nend\n" }, { "path": "lib/csvlint/csvw/metadata_error.rb", "content": "module Csvlint\n module Csvw\n class MetadataError < StandardError\n attr_reader :path\n\n def initialize(path = nil)\n @path = path\n end\n end\n end\nend\n" }, { "path": "lib/csvlint/csvw/number_format.rb", "content": "module Csvlint\n module Csvw\n class NumberFormat\n attr_reader :integer, :pattern, :prefix, :numeric_part, :suffix, :grouping_separator, :decimal_separator, :primary_grouping_size, :secondary_grouping_size, :fractional_grouping_size\n\n def initialize(pattern = nil, grouping_separator = nil, decimal_separator = \".\", integer = nil)\n @pattern = pattern\n @integer = integer\n if @integer.nil?\n @integer = if @pattern.nil?\n nil\n else\n !@pattern.include?(decimal_separator)\n end\n end\n @grouping_separator = grouping_separator || (@pattern.nil? ? nil : \",\")\n @decimal_separator = decimal_separator || \".\"\n if pattern.nil?\n @regexp = if integer\n INTEGER_REGEXP\n else\n Regexp.new(\"^(([-+]?[0-9]+(\\\\.[0-9]+)?([Ee][-+]?[0-9]+)?[%‰]?)|NaN|INF|-INF)$\")\n end\n else\n numeric_part_regexp = Regexp.new(\"(?[-+]?([0#Ee]|#{Regexp.escape(@grouping_separator)}|#{Regexp.escape(@decimal_separator)})+)\")\n number_format_regexp = Regexp.new(\"^(?.*?)#{numeric_part_regexp}(?.*?)$\")\n match = number_format_regexp.match(pattern)\n raise Csvw::NumberFormatError, \"invalid number format\" if match.nil?\n\n @prefix = match[\"prefix\"]\n @numeric_part = match[\"numeric_part\"]\n @suffix = match[\"suffix\"]\n\n parts = @numeric_part.split(\"E\")\n mantissa_part = parts[0]\n exponent_part = parts[1] || \"\"\n mantissa_parts = mantissa_part.split(@decimal_separator)\n # raise Csvw::NumberFormatError, \"more than two decimal separators in number format\" if parts.length > 2\n integer_part = mantissa_parts[0]\n fractional_part = mantissa_parts[1] || \"\"\n\n if [\"+\", \"-\"].include?(integer_part[0])\n numeric_part_regexp = \"\\\\#{integer_part[0]}\"\n integer_part = integer_part[1..-1]\n else\n numeric_part_regexp = \"[-+]?\"\n end\n\n min_integer_digits = integer_part.gsub(@grouping_separator, \"\").delete(\"#\").length\n min_fraction_digits = fractional_part.gsub(@grouping_separator, \"\").delete(\"#\").length\n max_fraction_digits = fractional_part.gsub(@grouping_separator, \"\").length\n min_exponent_digits = exponent_part.delete(\"#\").length\n max_exponent_digits = exponent_part.length\n\n integer_parts = integer_part.split(@grouping_separator)[1..-1]\n @primary_grouping_size = begin\n integer_parts[-1].length\n rescue\n 0\n end\n @secondary_grouping_size = begin\n integer_parts[-2].length\n rescue\n @primary_grouping_size\n end\n\n fractional_parts = fractional_part.split(@grouping_separator)[0..-2]\n @fractional_grouping_size = begin\n fractional_parts[0].length\n rescue\n 0\n end\n\n if @primary_grouping_size == 0\n integer_regexp = \"[0-9]*[0-9]{#{min_integer_digits}}\"\n else\n leading_regexp = \"([0-9]{0,#{@secondary_grouping_size - 1}}#{Regexp.escape(@grouping_separator)})?\"\n secondary_groups = \"([0-9]{#{@secondary_grouping_size}}#{Regexp.escape(@grouping_separator)})*\"\n if min_integer_digits > @primary_grouping_size\n remaining_req_digits = min_integer_digits - @primary_grouping_size\n req_secondary_groups = (remaining_req_digits / @secondary_grouping_size > 0) ? \"([0-9]{#{@secondary_grouping_size}}#{Regexp.escape(@grouping_separator)}){#{remaining_req_digits / @secondary_grouping_size}}\" : \"\"\n if remaining_req_digits % @secondary_grouping_size > 0\n final_req_digits = \"[0-9]{#{@secondary_grouping_size - (remaining_req_digits % @secondary_grouping_size)}}\"\n final_opt_digits = \"[0-9]{0,#{@secondary_grouping_size - (remaining_req_digits % @secondary_grouping_size)}}\"\n integer_regexp = \"((#{leading_regexp}#{secondary_groups}#{final_req_digits})|#{final_opt_digits})[0-9]{#{remaining_req_digits % @secondary_grouping_size}}#{Regexp.escape(@grouping_separator)}#{req_secondary_groups}[0-9]{#{@primary_grouping_size}}\"\n else\n integer_regexp = \"(#{leading_regexp}#{secondary_groups})?#{req_secondary_groups}[0-9]{#{@primary_grouping_size}}\"\n end\n else\n final_req_digits = (@primary_grouping_size > min_integer_digits) ? \"[0-9]{#{@primary_grouping_size - min_integer_digits}}\" : \"\"\n final_opt_digits = (@primary_grouping_size > min_integer_digits) ? \"[0-9]{0,#{@primary_grouping_size - min_integer_digits}}\" : \"\"\n integer_regexp = \"((#{leading_regexp}#{secondary_groups}#{final_req_digits})|#{final_opt_digits})[0-9]{#{min_integer_digits}}\"\n end\n end\n\n numeric_part_regexp += integer_regexp\n\n if max_fraction_digits > 0\n if @fractional_grouping_size == 0\n fractional_regexp = \"\"\n fractional_regexp += \"[0-9]{#{min_fraction_digits}}\" if min_fraction_digits > 0\n fractional_regexp += \"[0-9]{0,#{max_fraction_digits - min_fraction_digits}}\" unless min_fraction_digits == max_fraction_digits\n fractional_regexp = \"#{Regexp.escape(@decimal_separator)}#{fractional_regexp}\"\n fractional_regexp = \"(#{fractional_regexp})?\" if min_fraction_digits == 0\n numeric_part_regexp += fractional_regexp\n else\n fractional_regexp = \"\"\n\n if min_fraction_digits > 0\n if min_fraction_digits >= @fractional_grouping_size\n # first group of required digits - something like \"[0-9]{3}\"\n fractional_regexp += \"[0-9]{#{@fractional_grouping_size}}\"\n # additional groups of required digits - something like \"(,[0-9]{3}){1}\"\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{#{@fractional_grouping_size}}){#{min_fraction_digits / @fractional_grouping_size - 1}}\" if min_fraction_digits / @fractional_grouping_size > 1\n fractional_regexp += Regexp.escape(@grouping_separator).to_s if min_fraction_digits % @fractional_grouping_size > 0\n end\n # additional required digits - something like \",[0-9]{1}\"\n fractional_regexp += \"[0-9]{#{min_fraction_digits % @fractional_grouping_size}}\" if min_fraction_digits % @fractional_grouping_size > 0\n\n opt_fractional_digits = max_fraction_digits - min_fraction_digits\n if opt_fractional_digits > 0\n fractional_regexp += \"(\"\n\n if min_fraction_digits % @fractional_grouping_size > 0\n # optional fractional digits to complete the group\n fractional_regexp += \"[0-9]{0,#{[opt_fractional_digits, @fractional_grouping_size - (min_fraction_digits % @fractional_grouping_size)].min}}\"\n fractional_regexp += \"|\"\n fractional_regexp += \"[0-9]{#{[opt_fractional_digits, @fractional_grouping_size - (min_fraction_digits % @fractional_grouping_size)].min}}\"\n else\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{1,#{@fractional_grouping_size}})?\"\n fractional_regexp += \"|\"\n fractional_regexp += \"#{Regexp.escape(@grouping_separator)}[0-9]{#{@fractional_grouping_size}}\"\n end\n\n remaining_opt_fractional_digits = opt_fractional_digits - (@fractional_grouping_size - (min_fraction_digits % @fractional_grouping_size))\n if remaining_opt_fractional_digits > 0\n if remaining_opt_fractional_digits % @fractional_grouping_size > 0\n # optional fraction digits in groups\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{#{@fractional_grouping_size}}){0,#{remaining_opt_fractional_digits / @fractional_grouping_size}}\" if remaining_opt_fractional_digits > @fractional_grouping_size\n # remaining optional fraction digits\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{1,#{remaining_opt_fractional_digits % @fractional_grouping_size}})?\"\n else\n # optional fraction digits in groups\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{#{@fractional_grouping_size}}){0,#{(remaining_opt_fractional_digits / @fractional_grouping_size) - 1}}\" if remaining_opt_fractional_digits > @fractional_grouping_size\n # remaining optional fraction digits\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{1,#{@fractional_grouping_size}})?\"\n end\n\n # optional fraction digits in groups\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{#{@fractional_grouping_size}}){0,#{(remaining_opt_fractional_digits / @fractional_grouping_size) - 1}}\" if remaining_opt_fractional_digits > @fractional_grouping_size\n # remaining optional fraction digits\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{1,#{remaining_opt_fractional_digits % @fractional_grouping_size}})?\" if remaining_opt_fractional_digits % @fractional_grouping_size > 0\n end\n fractional_regexp += \")\"\n end\n elsif max_fraction_digits % @fractional_grouping_size > 0\n # optional fractional digits in groups\n fractional_regexp += \"([0-9]{#{@fractional_grouping_size}}#{Regexp.escape(@grouping_separator)}){0,#{max_fraction_digits / @fractional_grouping_size}}\"\n # remaining optional fraction digits\n fractional_regexp += \"(#{Regexp.escape(@grouping_separator)}[0-9]{1,#{max_fraction_digits % @fractional_grouping_size}})?\" if max_fraction_digits % @fractional_grouping_size > 0\n else\n fractional_regexp += \"([0-9]{#{@fractional_grouping_size}}#{Regexp.escape(@grouping_separator)}){0,#{(max_fraction_digits / @fractional_grouping_size) - 1}}\" if max_fraction_digits > @fractional_grouping_size\n fractional_regexp += \"[0-9]{1,#{@fractional_grouping_size}}\"\n end\n fractional_regexp = \"#{Regexp.escape(@decimal_separator)}#{fractional_regexp}\"\n fractional_regexp = \"(#{fractional_regexp})?\" if min_fraction_digits == 0\n numeric_part_regexp += fractional_regexp\n end\n end\n\n if max_exponent_digits > 0\n numeric_part_regexp += \"E\"\n numeric_part_regexp += \"[0-9]{0,#{max_exponent_digits - min_exponent_digits}}\" unless max_exponent_digits == min_exponent_digits\n numeric_part_regexp += \"[0-9]{#{min_exponent_digits}}\" unless min_exponent_digits == 0\n end\n\n @regexp = Regexp.new(\"^(?#{Regexp.escape(@prefix)})(?#{numeric_part_regexp})(?#{suffix})$\")\n end\n end\n\n def match(value)\n value&.match?(@regexp) ? true : false\n end\n\n def parse(value)\n if @pattern.nil?\n return nil if !@grouping_separator.nil? && value =~ Regexp.new(\"((^#{Regexp.escape(@grouping_separator)})|#{Regexp.escape(@grouping_separator)}{2})\")\n value.gsub!(@grouping_separator, \"\") unless @grouping_separator.nil?\n value.gsub!(@decimal_separator, \".\") unless @decimal_separator.nil?\n if value&.match?(@regexp)\n case value\n when \"NaN\"\n Float::NAN\n when \"INF\"\n Float::INFINITY\n when \"-INF\"\n -Float::INFINITY\n else\n case value[-1]\n when \"%\"\n value.to_f / 100\n when \"‰\"\n value.to_f / 1000\n else\n if @integer.nil?\n value.include?(\".\") ? value.to_f : value.to_i\n else\n @integer ? value.to_i : value.to_f\n end\n end\n end\n end\n else\n match = @regexp.match(value)\n return nil if match.nil?\n number = match[\"numeric_part\"]\n number.gsub!(@grouping_separator, \"\") unless @grouping_separator.nil?\n number.gsub!(@decimal_separator, \".\") unless @decimal_separator.nil?\n number = @integer ? number.to_i : number.to_f\n number = number.to_f / 100 if match[\"prefix\"].include?(\"%\") || match[\"suffix\"].include?(\"%\")\n number = number.to_f / 1000 if match[\"prefix\"].include?(\"‰\") || match[\"suffix\"].include?(\"‰\")\n number\n end\n end\n\n private\n\n INTEGER_REGEXP = /^[-+]?[0-9]+[%‰]?$/\n end\n\n class NumberFormatError < StandardError\n end\n end\nend\n" }, { "path": "lib/csvlint/csvw/property_checker.rb", "content": "module Csvlint\n module Csvw\n class PropertyChecker\n class << self\n def check_property(property, value, base_url, lang)\n if PROPERTIES.include? property\n PROPERTIES[property].call(value, base_url, lang)\n elsif property =~ /^([a-z]+):/ && NAMESPACES.include?(property.split(\":\")[0])\n value, warnings = check_common_property_value(value, base_url, lang)\n [value, warnings, :annotation]\n else\n # property name must be an absolute URI\n begin\n return value, :invalid_property, nil if URI(property).scheme.nil?\n value, warnings = check_common_property_value(value, base_url, lang)\n [value, warnings, :annotation]\n rescue\n [value, :invalid_property, nil]\n end\n end\n end\n\n private\n\n def check_common_property_value(value, base_url, lang)\n case value\n when Hash\n value = value.clone\n warnings = []\n value.each do |p, v|\n case p\n when \"@context\"\n raise Csvlint::Csvw::MetadataError.new(p), \"common property has @context property\"\n when \"@list\"\n raise Csvlint::Csvw::MetadataError.new(p), \"common property has @list property\"\n when \"@set\"\n raise Csvlint::Csvw::MetadataError.new(p), \"common property has @set property\"\n when \"@type\"\n if value[\"@value\"] && BUILT_IN_DATATYPES.include?(v)\n elsif !value[\"@value\"] && BUILT_IN_TYPES.include?(v)\n elsif ((v.is_a? String) && (v =~ /^([a-z]+):/)) && NAMESPACES.include?(v.split(\":\")[0])\n else\n # must be an absolute URI\n begin\n raise Csvlint::Csvw::MetadataError.new, \"common property has invalid @type (#{v})\" if URI(v).scheme.nil?\n rescue\n raise Csvlint::Csvw::MetadataError.new, \"common property has invalid @type (#{v})\"\n end\n end\n when \"@id\"\n unless base_url.nil?\n begin\n v = URI.join(base_url, v)\n rescue\n raise Csvlint::Csvw::MetadataError.new, \"common property has invalid @id (#{v})\"\n end\n end\n when \"@value\"\n raise Csvlint::Csvw::MetadataError.new, \"common property with @value has both @language and @type\" if value[\"@type\"] && value[\"@language\"]\n raise Csvlint::Csvw::MetadataError.new, \"common property with @value has properties other than @language or @type\" unless value.except(\"@type\").except(\"@language\").except(\"@value\").empty?\n when \"@language\"\n raise Csvlint::Csvw::MetadataError.new, \"common property with @language lacks a @value\" unless value[\"@value\"]\n raise Csvlint::Csvw::MetadataError.new, \"common property has invalid @language (#{v})\" if !((v.is_a? String) && (v =~ BCP47_LANGUAGE_REGEXP)) || !v.nil?\n else\n if p[0] == \"@\"\n raise Csvlint::Csvw::MetadataError.new, \"common property has property other than @id, @type, @value or @language beginning with @ (#{p})\"\n else\n v, w = check_common_property_value(v, base_url, lang)\n warnings += Array(w)\n end\n end\n value[p] = v\n end\n [value, warnings]\n when String\n if lang == \"und\"\n [value, nil]\n else\n [{\"@value\" => value, \"@language\" => lang}, nil]\n end\n when Array\n values = []\n warnings = []\n value.each do |v|\n v, w = check_common_property_value(v, base_url, lang)\n warnings += Array(w)\n values << v\n end\n [values, warnings]\n else\n [value, nil]\n end\n end\n\n def convert_value_facet(value, property, datatype)\n if value[property]\n if DATE_FORMAT_DATATYPES.include?(datatype)\n format = Csvlint::Csvw::DateFormat.new(nil, datatype)\n v = format.parse(value[property])\n if v.nil?\n value.delete(property)\n return [:\":invalid_#{property}\"]\n else\n value[property] = v\n return []\n end\n elsif NUMERIC_FORMAT_DATATYPES.include?(datatype)\n return []\n else\n raise Csvlint::Csvw::MetadataError.new(\"datatype.#{property}\"), \"#{property} is only allowed for numeric, date/time and duration types\"\n end\n end\n []\n end\n\n def array_property(type)\n lambda { |value, base_url, lang|\n return value, nil, type if value.instance_of? Array\n [false, :invalid_value, type]\n }\n end\n\n def boolean_property(type)\n lambda { |value, base_url, lang|\n return value, nil, type if value == true || value == false\n [false, :invalid_value, type]\n }\n end\n\n def string_property(type)\n lambda { |value, base_url, lang|\n return value, nil, type if value.instance_of? String\n [\"\", :invalid_value, type]\n }\n end\n\n def uri_template_property(type)\n lambda { |value, base_url, lang|\n return URITemplate.new(value), nil, type if value.instance_of? String\n [URITemplate.new(\"\"), :invalid_value, type]\n }\n end\n\n def numeric_property(type)\n lambda { |value, base_url, lang|\n return value, nil, type if value.is_a?(Integer) && value >= 0\n [nil, :invalid_value, type]\n }\n end\n\n def link_property(type)\n lambda { |value, base_url, lang|\n raise Csvlint::Csvw::MetadataError.new, \"URL #{value} starts with _:\" if /^_:/.match?(value.to_s)\n return (base_url.nil? ? URI(value) : URI.join(base_url, value)), nil, type if value.instance_of? String\n [base_url, :invalid_value, type]\n }\n end\n\n def language_property(type)\n lambda { |value, base_url, lang|\n return value, nil, type if BCP47_REGEXP.match?(value)\n [nil, :invalid_value, type]\n }\n end\n\n def natural_language_property(type)\n lambda { |value, base_url, lang|\n warnings = []\n if value.instance_of? String\n [{lang => [value]}, nil, type]\n elsif value.instance_of? Array\n valid_titles = []\n value.each do |title|\n if title.instance_of? String\n valid_titles << title\n else\n warnings << :invalid_value\n end\n end\n [{lang => valid_titles}, warnings, type]\n elsif value.instance_of? Hash\n value = value.clone\n value.each do |l, v|\n if BCP47_REGEXP.match?(l)\n valid_titles = []\n Array(v).each do |title|\n if title.instance_of? String\n valid_titles << title\n else\n warnings << :invalid_value\n end\n end\n value[l] = valid_titles\n else\n value.delete(l)\n warnings << :invalid_language\n end\n end\n warnings << :invalid_value if value.empty?\n [value, warnings, type]\n else\n [{}, :invalid_value, type]\n end\n }\n end\n\n def column_reference_property(type)\n lambda { |value, base_url, lang|\n [Array(value), nil, type]\n }\n end\n end\n\n PROPERTIES = {\n # context properties\n \"@language\" => language_property(:context),\n \"@base\" => link_property(:context),\n # common properties\n \"@id\" => link_property(:common),\n \"notes\" => lambda { |value, base_url, lang|\n return false, :invalid_value, :common unless value.instance_of? Array\n values = []\n warnings = []\n value.each do |v|\n v, w = check_common_property_value(v, base_url, lang)\n values << v\n warnings += w\n end\n [values, warnings, :common]\n },\n \"suppressOutput\" => boolean_property(:common),\n \"dialect\" => lambda { |value, base_url, lang|\n if value.instance_of? Hash\n value = value.clone\n warnings = []\n value.each do |p, v|\n if p == \"@id\"\n raise Csvlint::Csvw::MetadataError.new(\"dialect.@id\"), \"@id starts with _:\" if /^_:/.match?(v)\n elsif p == \"@type\"\n raise Csvlint::Csvw::MetadataError.new(\"dialect.@type\"), \"@type of dialect is not 'Dialect'\" if v != \"Dialect\"\n else\n v, warning, type = check_property(p, v, base_url, lang)\n if type == :dialect && (warning.nil? || warning.empty?)\n value[p] = v\n else\n value.delete(p)\n warnings << :invalid_property unless type == :dialect\n warnings += Array(warning)\n end\n end\n end\n [value, warnings, :common]\n else\n [{}, :invalid_value, :common]\n end\n },\n # inherited properties\n \"null\" => lambda { |value, base_url, lang|\n case value\n when String\n [[value], nil, :inherited]\n when Array\n values = []\n warnings = []\n value.each do |v|\n if v.instance_of? String\n values << v\n else\n warnings << :invalid_value\n end\n end\n [values, warnings, :inherited]\n else\n [[\"\"], :invalid_value, :inherited]\n end\n },\n \"default\" => string_property(:inherited),\n \"separator\" => lambda { |value, base_url, lang|\n return value, nil, :inherited if value.instance_of?(String) || value.nil?\n [nil, :invalid_value, :inherited]\n },\n \"lang\" => language_property(:inherited),\n \"datatype\" => lambda { |value, base_url, lang|\n value = value.clone\n warnings = []\n if value.instance_of? Hash\n if value[\"@id\"]\n raise Csvlint::Csvw::MetadataError.new(\"datatype.@id\"), \"datatype @id must not be the id of a built-in datatype (#{value[\"@id\"]})\" if BUILT_IN_DATATYPES.value?(value[\"@id\"])\n _, w, _ = PROPERTIES[\"@id\"].call(value[\"@id\"], base_url, lang)\n unless w.nil?\n warnings << w\n value.delete(\"@id\")\n end\n end\n\n if value[\"base\"]\n if BUILT_IN_DATATYPES.include? value[\"base\"]\n value[\"base\"] = BUILT_IN_DATATYPES[value[\"base\"]]\n else\n value[\"base\"] = BUILT_IN_DATATYPES[\"string\"]\n warnings << :invalid_datatype_base\n end\n else\n value[\"base\"] = BUILT_IN_DATATYPES[\"string\"]\n end\n elsif BUILT_IN_DATATYPES.include? value\n value = {\"@id\" => BUILT_IN_DATATYPES[value]}\n else\n value = {\"@id\" => BUILT_IN_DATATYPES[\"string\"]}\n warnings << :invalid_value\n end\n\n unless STRING_DATATYPES.include?(value[\"base\"]) || BINARY_DATATYPES.include?(value[\"base\"])\n raise Csvlint::Csvw::MetadataError.new(\"datatype.length\"), \"datatypes based on #{value[\"base\"]} cannot have a length facet\" if value[\"length\"]\n raise Csvlint::Csvw::MetadataError.new(\"datatype.minLength\"), \"datatypes based on #{value[\"base\"]} cannot have a minLength facet\" if value[\"minLength\"]\n raise Csvlint::Csvw::MetadataError.new(\"datatype.maxLength\"), \"datatypes based on #{value[\"base\"]} cannot have a maxLength facet\" if value[\"maxLength\"]\n end\n\n if value[\"minimum\"]\n value[\"minInclusive\"] = value[\"minimum\"]\n value.delete(\"minimum\")\n end\n if value[\"maximum\"]\n value[\"maxInclusive\"] = value[\"maximum\"]\n value.delete(\"maximum\")\n end\n\n warnings += convert_value_facet(value, \"minInclusive\", value[\"base\"])\n warnings += convert_value_facet(value, \"minExclusive\", value[\"base\"])\n warnings += convert_value_facet(value, \"maxInclusive\", value[\"base\"])\n warnings += convert_value_facet(value, \"maxExclusive\", value[\"base\"])\n\n minInclusive = value[\"minInclusive\"].is_a?(Hash) ? value[\"minInclusive\"][:dateTime] : value[\"minInclusive\"]\n maxInclusive = value[\"maxInclusive\"].is_a?(Hash) ? value[\"maxInclusive\"][:dateTime] : value[\"maxInclusive\"]\n minExclusive = value[\"minExclusive\"].is_a?(Hash) ? value[\"minExclusive\"][:dateTime] : value[\"minExclusive\"]\n maxExclusive = value[\"maxExclusive\"].is_a?(Hash) ? value[\"maxExclusive\"][:dateTime] : value[\"maxExclusive\"]\n\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype cannot specify both minimum/minInclusive (#{minInclusive}) and minExclusive (#{minExclusive}\" if minInclusive && minExclusive\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype cannot specify both maximum/maxInclusive (#{maxInclusive}) and maxExclusive (#{maxExclusive}\" if maxInclusive && maxExclusive\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype minInclusive (#{minInclusive}) cannot be more than maxInclusive (#{maxInclusive}\" if minInclusive && maxInclusive && minInclusive > maxInclusive\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype minInclusive (#{minInclusive}) cannot be more than or equal to maxExclusive (#{maxExclusive}\" if minInclusive && maxExclusive && minInclusive >= maxExclusive\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype minExclusive (#{minExclusive}) cannot be more than or equal to maxExclusive (#{maxExclusive}\" if minExclusive && maxExclusive && minExclusive > maxExclusive\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype minExclusive (#{minExclusive}) cannot be more than maxInclusive (#{maxInclusive}\" if minExclusive && maxInclusive && minExclusive >= maxInclusive\n\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype length (#{value[\"length\"]}) cannot be less than minLength (#{value[\"minLength\"]}\" if value[\"length\"] && value[\"minLength\"] && value[\"length\"] < value[\"minLength\"]\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype length (#{value[\"length\"]}) cannot be more than maxLength (#{value[\"maxLength\"]}\" if value[\"length\"] && value[\"maxLength\"] && value[\"length\"] > value[\"maxLength\"]\n raise Csvlint::Csvw::MetadataError.new(\"\"), \"datatype minLength (#{value[\"minLength\"]}) cannot be more than maxLength (#{value[\"maxLength\"]}\" if value[\"minLength\"] && value[\"maxLength\"] && value[\"minLength\"] > value[\"maxLength\"]\n\n if value[\"format\"]\n if REGEXP_FORMAT_DATATYPES.include?(value[\"base\"])\n begin\n value[\"format\"] = Regexp.new(value[\"format\"])\n rescue RegexpError\n value.delete(\"format\")\n warnings << :invalid_regex\n end\n elsif NUMERIC_FORMAT_DATATYPES.include?(value[\"base\"])\n value[\"format\"] = {\"pattern\" => value[\"format\"]} if value[\"format\"].instance_of? String\n begin\n value[\"format\"] = Csvlint::Csvw::NumberFormat.new(value[\"format\"][\"pattern\"], value[\"format\"][\"groupChar\"], value[\"format\"][\"decimalChar\"] || \".\", INTEGER_FORMAT_DATATYPES.include?(value[\"base\"]))\n rescue Csvlint::Csvw::NumberFormatError\n value[\"format\"] = Csvlint::Csvw::NumberFormat.new(nil, value[\"format\"][\"groupChar\"], value[\"format\"][\"decimalChar\"] || \".\", INTEGER_FORMAT_DATATYPES.include?(value[\"base\"]))\n warnings << :invalid_number_format\n end\n elsif value[\"base\"] == \"http://www.w3.org/2001/XMLSchema#boolean\"\n if value[\"format\"].instance_of? String\n value[\"format\"] = value[\"format\"].split(\"|\")\n unless value[\"format\"].length == 2\n value.delete(\"format\")\n warnings << :invalid_boolean_format\n end\n else\n value.delete(\"format\")\n warnings << :invalid_boolean_format\n end\n elsif DATE_FORMAT_DATATYPES.include?(value[\"base\"])\n if value[\"format\"].instance_of? String\n begin\n value[\"format\"] = Csvlint::Csvw::DateFormat.new(value[\"format\"])\n rescue Csvlint::CsvDateFormatError\n value.delete(\"format\")\n warnings << :invalid_date_format\n end\n else\n value.delete(\"format\")\n warnings << :invalid_date_format\n end\n end\n end\n [value, warnings, :inherited]\n },\n \"required\" => boolean_property(:inherited),\n \"ordered\" => boolean_property(:inherited),\n \"aboutUrl\" => uri_template_property(:inherited),\n \"propertyUrl\" => uri_template_property(:inherited),\n \"valueUrl\" => uri_template_property(:inherited),\n \"textDirection\" => lambda { |value, base_url, lang|\n value = value.to_sym\n return value, nil, :inherited if [:ltr, :rtl, :auto, :inherit].include? value\n [:inherit, :invalid_value, :inherited]\n },\n # column level properties\n \"virtual\" => boolean_property(:column),\n \"titles\" => natural_language_property(:column),\n \"name\" => lambda { |value, base_url, lang|\n return value, nil, :column if value.instance_of?(String) && value =~ NAME_REGEXP\n [nil, :invalid_value, :column]\n },\n # table level properties\n \"transformations\" => lambda { |value, base_url, lang|\n transformations = []\n warnings = []\n if value.instance_of? Array\n value.each_with_index do |transformation, i|\n if transformation.instance_of? Hash\n transformation = transformation.clone\n transformation.each do |p, v|\n if p == \"@id\"\n raise Csvlint::Csvw::MetadataError.new(\"transformations[#{i}].@id\"), \"@id starts with _:\" if /^_:/.match?(v)\n elsif p == \"@type\"\n raise Csvlint::Csvw::MetadataError.new(\"transformations[#{i}].@type\"), \"@type of transformation is not 'Template'\" if v != \"Template\"\n elsif p == \"url\"\n elsif p == \"titles\"\n else\n _, warning, type = check_property(p, v, base_url, lang)\n if type != :transformation && !(warning.nil? || warning.empty?)\n value.delete(p)\n warnings << :invalid_property unless type == :transformation\n warnings += Array(warning)\n end\n end\n end\n transformations << transformation\n else\n warnings << :invalid_transformation\n end\n end\n else\n warnings << :invalid_value\n end\n [transformations, warnings, :table]\n },\n \"tableDirection\" => lambda { |value, base_url, lang|\n value = value.to_sym\n return value, nil, :table if [:ltr, :rtl, :auto].include? value\n [:auto, :invalid_value, :table]\n },\n \"tableSchema\" => lambda { |value, base_url, lang|\n schema_base_url = base_url\n schema_lang = lang\n if value.instance_of? String\n schema_url = URI.join(base_url, value).to_s\n schema_base_url = schema_url\n schema_ref = schema_url.start_with?(\"file:\") ? File.new(schema_url[5..-1]) : schema_url\n schema = JSON.parse(URI.open(schema_ref).read)\n schema[\"@id\"] = schema[\"@id\"] ? URI.join(schema_url, schema[\"@id\"]).to_s : schema_url\n if schema[\"@context\"]\n if schema[\"@context\"].instance_of?(Array) && schema[\"@context\"].length > 1\n schema_base_url = schema[\"@context\"][1][\"@base\"] ? URI.join(schema_base_url, schema[\"@context\"][1][\"@base\"]).to_s : schema_base_url\n schema_lang = schema[\"@context\"][1][\"@language\"] || schema_lang\n end\n schema.delete(\"@context\")\n end\n elsif value.instance_of? Hash\n schema = value.clone\n else\n return {}, :invalid_value, :table\n end\n warnings = []\n schema.each do |p, v|\n if p == \"@id\"\n raise Csvlint::Csvw::MetadataError.new(\"tableSchema.@id\"), \"@id starts with _:\" if /^_:/.match?(v)\n elsif p == \"@type\"\n raise Csvlint::Csvw::MetadataError.new(\"tableSchema.@type\"), \"@type of schema is not 'Schema'\" if v != \"Schema\"\n else\n v, warning, type = check_property(p, v, schema_base_url, schema_lang)\n if (type == :schema || type == :inherited) && (warning.nil? || warning.empty?)\n schema[p] = v\n else\n schema.delete(p)\n warnings << :invalid_property unless type == :schema || type == :inherited\n warnings += Array(warning)\n end\n end\n end\n [schema, warnings, :table]\n },\n \"url\" => link_property(:table),\n # dialect properties\n \"commentPrefix\" => string_property(:dialect),\n \"delimiter\" => string_property(:dialect),\n \"doubleQuote\" => boolean_property(:dialect),\n \"encoding\" => lambda { |value, base_url, lang|\n return value, nil, :dialect if VALID_ENCODINGS.include? value\n [nil, :invalid_value, :dialect]\n },\n \"header\" => boolean_property(:dialect),\n \"headerRowCount\" => numeric_property(:dialect),\n \"lineTerminators\" => array_property(:dialect),\n \"quoteChar\" => string_property(:dialect),\n \"skipBlankRows\" => boolean_property(:dialect),\n \"skipColumns\" => numeric_property(:dialect),\n \"skipInitialSpace\" => boolean_property(:dialect),\n \"skipRows\" => numeric_property(:dialect),\n \"trim\" => lambda { |value, base_url, lang|\n value = :true if value == true || value == \"true\"\n value = :false if value == false || value == \"false\"\n value = :start if value == \"start\"\n value = :end if value == \"end\"\n return value, nil, :dialect if [:true, :false, :start, :end].include? value\n [true, :invalid_value, :dialect]\n },\n # schema properties\n \"columns\" => lambda { |value, base_url, lang| [value, nil, :schema] },\n \"primaryKey\" => column_reference_property(:schema),\n \"foreignKeys\" => lambda { |value, base_url, lang|\n foreign_keys = []\n warnings = []\n if value.instance_of? Array\n value.each_with_index do |foreign_key, i|\n if foreign_key.instance_of? Hash\n foreign_key = foreign_key.clone\n foreign_key.each do |p, v|\n v, warning, type = check_property(p, v, base_url, lang)\n if type == :foreign_key && (warning.nil? || warning.empty?)\n foreign_key[p] = v\n elsif /:/.match?(p)\n raise Csvlint::Csvw::MetadataError.new(\"foreignKey.#{p}\"), \"foreignKey includes a prefixed (common) property\"\n else\n foreign_key.delete(p)\n warnings << :invalid_property unless type == :foreign_key\n warnings += Array(warning)\n end\n end\n foreign_keys << foreign_key\n else\n warnings << :invalid_foreign_key\n end\n end\n else\n warnings << :invalid_value\n end\n [foreign_keys, warnings, :schema]\n },\n \"rowTitles\" => column_reference_property(:schema),\n # transformation properties\n \"targetFormat\" => lambda { |value, base_url, lang| [value, nil, :transformation] },\n \"scriptFormat\" => lambda { |value, base_url, lang| [value, nil, :transformation] },\n \"source\" => lambda { |value, base_url, lang| [value, nil, :transformation] },\n # foreignKey properties\n \"columnReference\" => column_reference_property(:foreign_key),\n \"reference\" => lambda { |value, base_url, lang|\n if value.instance_of? Hash\n value = value.clone\n warnings = []\n value.each do |p, v|\n if [\"resource\", \"schemaReference\", \"columnReference\"].include? p\n v, warning, _ = check_property(p, v, base_url, lang)\n if warning.nil? || warning.empty?\n value[p] = v\n else\n value.delete(p)\n warnings += Array(warning)\n end\n elsif /:/.match?(p)\n raise Csvlint::Csvw::MetadataError.new(\"foreignKey.reference.#{p}\"), \"foreignKey reference includes a prefixed (common) property\"\n else\n value.delete(p)\n warnings << :invalid_property\n end\n end\n raise Csvlint::Csvw::MetadataError.new(\"foreignKey.reference.columnReference\"), \"foreignKey reference columnReference is missing\" unless value[\"columnReference\"]\n raise Csvlint::Csvw::MetadataError.new(\"foreignKey.reference\"), \"foreignKey reference does not have either resource or schemaReference\" unless value[\"resource\"] || value[\"schemaReference\"]\n raise Csvlint::Csvw::MetadataError.new(\"foreignKey.reference\"), \"foreignKey reference has both resource and schemaReference\" if value[\"resource\"] && value[\"schemaReference\"]\n [value, warnings, :foreign_key]\n else\n raise Csvlint::Csvw::MetadataError.new(\"foreignKey.reference\"), \"foreignKey reference is not an object\"\n end\n },\n # foreignKey reference properties\n \"resource\" => lambda { |value, base_url, lang| [value, nil, :foreign_key_reference] },\n \"schemaReference\" => lambda { |value, base_url, lang|\n [URI.join(base_url, value).to_s, nil, :foreign_key_reference]\n }\n }\n\n NAMESPACES = {\n \"dcat\" => \"http://www.w3.org/ns/dcat#\",\n \"qb\" => \"http://purl.org/linked-data/cube#\",\n \"grddl\" => \"http://www.w3.org/2003/g/data-view#\",\n \"ma\" => \"http://www.w3.org/ns/ma-ont#\",\n \"org\" => \"http://www.w3.org/ns/org#\",\n \"owl\" => \"http://www.w3.org/2002/07/owl#\",\n \"prov\" => \"http://www.w3.org/ns/prov#\",\n \"rdf\" => \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n \"rdfa\" => \"http://www.w3.org/ns/rdfa#\",\n \"rdfs\" => \"http://www.w3.org/2000/01/rdf-schema#\",\n \"rif\" => \"http://www.w3.org/2007/rif#\",\n \"rr\" => \"http://www.w3.org/ns/r2rml#\",\n \"sd\" => \"http://www.w3.org/ns/sparql-service-description#\",\n \"skos\" => \"http://www.w3.org/2004/02/skos/core#\",\n \"skosxl\" => \"http://www.w3.org/2008/05/skos-xl#\",\n \"wdr\" => \"http://www.w3.org/2007/05/powder#\",\n \"void\" => \"http://rdfs.org/ns/void#\",\n \"wdrs\" => \"http://www.w3.org/2007/05/powder-s#\",\n \"xhv\" => \"http://www.w3.org/1999/xhtml/vocab#\",\n \"xml\" => \"http://www.w3.org/XML/1998/namespace\",\n \"xsd\" => \"http://www.w3.org/2001/XMLSchema#\",\n \"csvw\" => \"http://www.w3.org/ns/csvw#\",\n \"cnt\" => \"http://www.w3.org/2008/content\",\n \"earl\" => \"http://www.w3.org/ns/earl#\",\n \"ht\" => \"http://www.w3.org/2006/http#\",\n \"oa\" => \"http://www.w3.org/ns/oa#\",\n \"ptr\" => \"http://www.w3.org/2009/pointers#\",\n \"cc\" => \"http://creativecommons.org/ns#\",\n \"ctag\" => \"http://commontag.org/ns#\",\n \"dc\" => \"http://purl.org/dc/terms/\",\n \"dcterms\" => \"http://purl.org/dc/terms/\",\n \"dc11\" => \"http://purl.org/dc/elements/1.1/\",\n \"foaf\" => \"http://xmlns.com/foaf/0.1/\",\n \"gr\" => \"http://purl.org/goodrelations/v1#\",\n \"ical\" => \"http://www.w3.org/2002/12/cal/icaltzd#\",\n \"og\" => \"http://ogp.me/ns#\",\n \"rev\" => \"http://purl.org/stuff/rev#\",\n \"sioc\" => \"http://rdfs.org/sioc/ns#\",\n \"v\" => \"http://rdf.data-vocabulary.org/#\",\n \"vcard\" => \"http://www.w3.org/2006/vcard/ns#\",\n \"schema\" => \"http://schema.org/\"\n }\n\n BCP47_REGULAR_REGEXP = \"(art-lojban|cel-gaulish|no-bok|no-nyn|zh-guoyu|zh-hakka|zh-min|zh-min-nan|zh-xiang)\"\n BCP47_IRREGULAR_REGEXP = \"(en-GB-oed|i-ami|i-bnn|i-default|i-enochian|i-hak|i-klingon|i-lux|i-mingo|i-navajo|i-pwn|i-tao|i-tay|i-tsu|sgn-BE-FR|sgn-BE-NL|sgn-CH-DE)\"\n BCP47_GRANDFATHERED_REGEXP = \"(?\" + BCP47_IRREGULAR_REGEXP + \"|\" + BCP47_REGULAR_REGEXP + \")\"\n BCP47_PRIVATE_USE_REGEXP = \"(?x(-[A-Za-z0-9]{1,8})+)\"\n BCP47_SINGLETON_REGEXP = \"[0-9A-WY-Za-wy-z]\"\n BCP47_EXTENSION_REGEXP = \"(?\" + BCP47_SINGLETON_REGEXP + \"(-[A-Za-z0-9]{2,8})+)\"\n BCP47_VARIANT_REGEXP = \"(?[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3})\"\n BCP47_REGION_REGEXP = \"(?[A-Za-z]{2}|[0-9]{3})\"\n BCP47_SCRIPT_REGEXP = \"(?