Repository: XAMPPRocky/tokei Branch: master Commit: 505d6481c98c Files: 258 Total size: 525.5 KB Directory structure: gitextract_x6tby1jr/ ├── .git-blame-ignore-revs ├── .gitattributes ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ ├── mean_bean_ci.yml │ ├── mean_bean_deploy.yml │ ├── publish_image.yaml │ └── release-plz.yaml ├── .gitignore ├── .mailmap ├── .tokeignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.toml ├── Earthfile ├── LICENCE-APACHE ├── LICENCE-MIT ├── README.md ├── benchmark.sh ├── build.rs ├── ci/ │ ├── build.bash │ ├── common.bash │ ├── set_rust_version.bash │ └── test.bash ├── fuzz/ │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ └── fuzz_targets/ │ ├── parse_from_slice.rs │ ├── parse_from_slice_panic.rs │ └── parse_from_slice_total.rs ├── languages.json ├── src/ │ ├── cli.rs │ ├── cli_utils.rs │ ├── config.rs │ ├── consts.rs │ ├── input.rs │ ├── language/ │ │ ├── embedding.rs │ │ ├── language_type.rs │ │ ├── language_type.tera.rs │ │ ├── languages.rs │ │ ├── mod.rs │ │ └── syntax.rs │ ├── lib.rs │ ├── main.rs │ ├── sort.rs │ ├── stats.rs │ └── utils/ │ ├── ext.rs │ ├── fs.rs │ ├── macros.rs │ └── mod.rs ├── tests/ │ ├── accuracy.rs │ ├── data/ │ │ ├── Daml.daml │ │ ├── Dockerfile │ │ ├── MSBuild.csproj │ │ ├── Makefile │ │ ├── Modelica.mo │ │ ├── NuGet.Config │ │ ├── PKGBUILD │ │ ├── Rakefile │ │ ├── SConstruct │ │ ├── Snakefile │ │ ├── Tera.tera │ │ ├── abnf.abnf │ │ ├── alloy.als │ │ ├── apl.apl │ │ ├── arduino.ino │ │ ├── arturo.art │ │ ├── asciidoc.adoc │ │ ├── asn1.asn1 │ │ ├── ats.dats │ │ ├── awk.awk │ │ ├── ballerina.bal │ │ ├── bazel.bzl │ │ ├── bean.bean │ │ ├── bicep.bicep │ │ ├── bitbake.bb │ │ ├── bqn.bqn │ │ ├── brightscript.brs │ │ ├── c.c │ │ ├── c3.c3 │ │ ├── cairo.cairo │ │ ├── cangjie.cj │ │ ├── chapel.chpl │ │ ├── cil.cil │ │ ├── circom.circom │ │ ├── clojure.clj │ │ ├── clojurec.cljc │ │ ├── clojurescript.cljs │ │ ├── cmake.cmake │ │ ├── codeql.ql │ │ ├── cogent.cogent │ │ ├── cpp.cpp │ │ ├── cppm.cppm │ │ ├── crystal.cr │ │ ├── csharp.cs │ │ ├── cuda.cu │ │ ├── cue.cue │ │ ├── cython.pyx │ │ ├── d.d │ │ ├── d2.d2 │ │ ├── dhall.dhall │ │ ├── dreammaker.dm │ │ ├── dust.dust │ │ ├── ebuild.ebuild │ │ ├── edgeql.edgeql │ │ ├── edn.edn │ │ ├── eight.8th │ │ ├── elvish.elv │ │ ├── emacs_dev_env.ede │ │ ├── emacs_lisp.el │ │ ├── emojicode.🍇 │ │ ├── esdl.esdl │ │ ├── example.umpl │ │ ├── factor.factor │ │ ├── fennel.fnl │ │ ├── flatbuffers.fbs │ │ ├── forgecfg.cfg │ │ ├── fsharp.fs │ │ ├── fstar.fst │ │ ├── ftl.ftl │ │ ├── futhark.fut │ │ ├── gas.S │ │ ├── gdb.gdb │ │ ├── gdshader.gdshader │ │ ├── gherkin.feature │ │ ├── gleam.gleam │ │ ├── glimmer_js.gjs │ │ ├── glimmer_ts.gts │ │ ├── gml.gml │ │ ├── go.go │ │ ├── gohtml.gohtml │ │ ├── graphql.gql │ │ ├── gwion.gw │ │ ├── haml.haml │ │ ├── hcl.tf │ │ ├── headache.ha │ │ ├── hex0.hex0 │ │ ├── hex1.hex1 │ │ ├── hex2.hex2 │ │ ├── hicad.mac │ │ ├── hledger.hledger │ │ ├── hpp.hpp │ │ ├── html.html │ │ ├── janet.janet │ │ ├── java.java │ │ ├── javascript.js │ │ ├── jinja2.j2 │ │ ├── jq.jq │ │ ├── jslt.jslt │ │ ├── jsonnet.jsonnet │ │ ├── jupyter.ipynb │ │ ├── justfile │ │ ├── k.k │ │ ├── kaem.kaem │ │ ├── kakoune_script.kak │ │ ├── koka.kk │ │ ├── ksh.ksh │ │ ├── kvlanguage.kv │ │ ├── lalrpop.lalrpop │ │ ├── linguafranca.lf │ │ ├── liquid.liquid │ │ ├── livescript.ls │ │ ├── llvm.ll │ │ ├── logtalk.lgt │ │ ├── lolcode.lol │ │ ├── m1.m1 │ │ ├── m4.m4 │ │ ├── menhir.mly │ │ ├── meson.build │ │ ├── metal.metal │ │ ├── mlatu.mlt │ │ ├── moduledef.def │ │ ├── mojo.mojo │ │ ├── monkeyc.mc │ │ ├── nextflow.nf │ │ ├── nqp.nqp │ │ ├── odin.odin │ │ ├── open_policy_agent.rego │ │ ├── openscad.scad │ │ ├── opentype.fea │ │ ├── org_mode.org │ │ ├── pan.pan │ │ ├── pcss.pcss │ │ ├── pest.pest │ │ ├── phix.e │ │ ├── plantuml.puml │ │ ├── pofile.po │ │ ├── pofile_pot.pot │ │ ├── poke.pk │ │ ├── pony.pony │ │ ├── postcss.sss │ │ ├── powershell.ps1 │ │ ├── pug.pug │ │ ├── puppet.pp │ │ ├── pyret.arr │ │ ├── python.py │ │ ├── q.q │ │ ├── qml.qml │ │ ├── racket.rkt │ │ ├── raku.raku │ │ ├── razor.cshtml │ │ ├── razorcomponent.razor │ │ ├── redscript.reds │ │ ├── renpy.rpy │ │ ├── roc.roc │ │ ├── ron.ron │ │ ├── rpmspec.spec │ │ ├── ruby.rb │ │ ├── ruby_env │ │ ├── ruby_html.erb │ │ ├── rust.rs │ │ ├── scheme.scm │ │ ├── shaderlab.shader │ │ ├── slang.slang │ │ ├── slint.slint │ │ ├── solidity.sol │ │ ├── sql.sql │ │ ├── srecode.srt │ │ ├── stan.stan │ │ ├── stata.do │ │ ├── stratego.str │ │ ├── stylus.styl │ │ ├── svelte.svelte │ │ ├── swift.swift │ │ ├── swig.i │ │ ├── tact.tact │ │ ├── templ.templ │ │ ├── thrift.thrift │ │ ├── tsx.tsx │ │ ├── ttcn.ttcn3 │ │ ├── twig.twig │ │ ├── typescript.ts │ │ ├── typst.typ │ │ ├── uiua.ua │ │ ├── unison.u │ │ ├── urweb.ur │ │ ├── urweb_urp.urp │ │ ├── urweb_urs.urs │ │ ├── vb6_bas.bas │ │ ├── vb6_cls.cls │ │ ├── vb6_frm.frm │ │ ├── vbscript.vbs │ │ ├── velocity.vm │ │ ├── vhdl.vhd │ │ ├── visualbasic.vb │ │ ├── vqe.qasm │ │ ├── vue.vue │ │ ├── webassembly.wat │ │ ├── wenyan.wy │ │ ├── wgsl.wgsl │ │ ├── xsl.xsl │ │ ├── xtend.xtend │ │ ├── yaml.yaml │ │ ├── zencode.zs │ │ ├── zig.zig │ │ └── zokrates.zok │ └── embedding/ │ └── file_triggeringprincipal_frame_1.html └── tokei.example.toml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .git-blame-ignore-revs ================================================ # Format JSON d7c548537cd5828b2d58e09f3207ddacc517b227 f356d27ab21e0f93839da90393c0edf9225740c2 ================================================ FILE: .gitattributes ================================================ tests/data/* linguist-documentation ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: XAMPPRocky ================================================ FILE: .github/workflows/mean_bean_ci.yml ================================================ name: Mean Bean CI on: push: branches: - master pull_request: jobs: # This job downloads and stores `cross` as an artifact, so that it can be # redownloaded across all of the jobs. Currently this copied pasted between # `ci.yml` and `deploy.yml`. Make sure to update both places when making # changes. install-cross: runs-on: ubuntu-latest steps: - uses: XAMPPRocky/get-github-release@v1 id: cross with: owner: rust-embedded repo: cross matches: ${{ matrix.platform }} token: ${{ secrets.GITHUB_TOKEN }} - uses: actions/upload-artifact@v4 with: name: cross-${{ matrix.platform }} path: ${{ steps.cross.outputs.install_path }} strategy: matrix: platform: [linux-musl, apple-darwin] windows: runs-on: windows-latest # Windows technically doesn't need this, but if we don't block windows on it # some of the windows jobs could fill up the concurrent job queue before # one of the install-cross jobs has started, so this makes sure all # artifacts are downloaded first. needs: install-cross steps: - uses: actions/checkout@v3 - run: ci/set_rust_version.bash ${{ matrix.channel }} ${{ matrix.target }} shell: bash - run: ci/build.bash cargo ${{ matrix.target }} shell: bash - run: ci/test.bash cargo ${{ matrix.target }} shell: bash strategy: fail-fast: true matrix: channel: [stable, beta, nightly] target: # MSVC - i686-pc-windows-msvc - x86_64-pc-windows-msvc # GNU: You typically only need to test Windows GNU if you're # specifically targeting it, and it can cause issues with some # dependencies if you're not so it's disabled by self. # - i686-pc-windows-gnu # - x86_64-pc-windows-gnu macos: runs-on: macos-latest needs: install-cross steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v4 with: name: cross-apple-darwin path: /usr/local/bin/ - run: chmod +x /usr/local/bin/cross - run: ci/set_rust_version.bash ${{ matrix.channel }} ${{ matrix.target }} - run: ci/build.bash cross ${{ matrix.target }} # Only test on macOS platforms since we can't simulate iOS. - run: ci/test.bash cross ${{ matrix.target }} if: matrix.target == 'x86_64-apple-darwin' strategy: fail-fast: true matrix: channel: [stable, beta, nightly] target: # macOS - x86_64-apple-darwin # iOS - aarch64-apple-ios - x86_64-apple-ios linux: runs-on: ubuntu-latest needs: install-cross steps: - uses: actions/checkout@v3 - name: Download Cross uses: actions/download-artifact@v4 with: name: cross-linux-musl path: /tmp/ - run: chmod +x /tmp/cross - run: ci/set_rust_version.bash ${{ matrix.channel }} ${{ matrix.target }} - run: ci/build.bash /tmp/cross ${{ matrix.target }} # These targets have issues with being tested so they are disabled # by default. You can try disabling to see if they work for # your project. - run: ci/test.bash /tmp/cross ${{ matrix.target }} if: | !contains(matrix.target, 'android') && !contains(matrix.target, 'bsd') && !contains(matrix.target, 'solaris') && matrix.target != 'armv5te-unknown-linux-musleabi' && matrix.target != 'sparc64-unknown-linux-gnu' strategy: fail-fast: true matrix: channel: [stable, beta, nightly] target: # WASM, off by default as most rust projects aren't compatible yet. # - wasm32-unknown-emscripten # Linux - aarch64-unknown-linux-gnu - aarch64-unknown-linux-musl - arm-unknown-linux-gnueabi - arm-unknown-linux-gnueabihf - arm-unknown-linux-musleabi - arm-unknown-linux-musleabihf - armv5te-unknown-linux-musleabi - armv7-unknown-linux-gnueabihf - armv7-unknown-linux-musleabihf - i586-unknown-linux-gnu - i586-unknown-linux-musl - i686-unknown-linux-gnu - i686-unknown-linux-musl # - mips-unknown-linux-gnu # - mips-unknown-linux-musl # - mips64-unknown-linux-gnuabi64 # - mips64el-unknown-linux-gnuabi64 # - mipsel-unknown-linux-gnu # - mipsel-unknown-linux-musl - powerpc-unknown-linux-gnu - powerpc64le-unknown-linux-gnu - s390x-unknown-linux-gnu - x86_64-unknown-linux-gnu - x86_64-unknown-linux-musl # Android # - aarch64-linux-android # - arm-linux-androideabi # - armv7-linux-androideabi # - i686-linux-android # - x86_64-linux-android # *BSD # The FreeBSD targets can have issues linking so they are disabled # by default. # - i686-unknown-freebsd # - x86_64-unknown-freebsd # - x86_64-unknown-netbsd # Solaris # - sparcv9-sun-solaris # - x86_64-sun-solaris # Bare Metal # These are no-std embedded targets, so they will only build if your # crate is `no_std` compatible. # - thumbv6m-none-eabi # - thumbv7em-none-eabi # - thumbv7em-none-eabihf # - thumbv7m-none-eabi ================================================ FILE: .github/workflows/mean_bean_deploy.yml ================================================ on: workflow_run: workflows: ["Release-plz"] branches: [master] types: - completed name: Mean Bean Deploy env: BIN: tokei jobs: # This job downloads and stores `cross` as an artifact, so that it can be # redownloaded across all of the jobs. Currently this copied pasted between # `ci.yml` and `deploy.yml`. Make sure to update both places when making # changes. install-cross: runs-on: ubuntu-latest steps: - uses: XAMPPRocky/get-github-release@v1 id: cross with: owner: rust-embedded repo: cross matches: ${{ matrix.platform }} token: ${{ secrets.GITHUB_TOKEN }} - uses: actions/upload-artifact@v4 with: name: cross-${{ matrix.platform }} path: ${{ steps.cross.outputs.install_path }} strategy: matrix: platform: [linux-musl, apple-darwin] windows: runs-on: windows-latest needs: install-cross strategy: matrix: target: # MSVC - i686-pc-windows-msvc - x86_64-pc-windows-msvc # GNU # - i686-pc-windows-gnu # - x86_64-pc-windows-gnu steps: - uses: actions/checkout@v3 # FIXME: Hack around thinLTO being broken. - run: echo "RUSTFLAGS=-Clto=fat" >> $GITHUB_ENV - run: bash ci/set_rust_version.bash stable ${{ matrix.target }} - run: bash ci/build.bash cargo ${{ matrix.target }} RELEASE # We're using using a fork of `actions/create-release` that detects # whether a release is already available or not first. - uses: XAMPPRocky/create-release@v1.0.2 id: create_release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ github.ref }} release_name: ${{ github.ref }} # Draft should **always** be false. GitHub doesn't provide a way to # get draft releases from its API, so there's no point using it. draft: false prerelease: true - uses: actions/upload-release-asset@v1 id: upload-release-asset env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: target/${{ matrix.target }}/release/${{ env.BIN }}.exe asset_name: ${{ env.BIN }}-${{ matrix.target }}.exe asset_content_type: application/zip macos: runs-on: macos-latest needs: install-cross strategy: matrix: target: # macOS - x86_64-apple-darwin # iOS # - aarch64-apple-ios # - armv7-apple-ios # - armv7s-apple-ios # - i386-apple-ios # - x86_64-apple-ios steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v4 with: name: cross-apple-darwin path: /usr/local/bin/ - run: chmod +x /usr/local/bin/cross - run: ci/set_rust_version.bash stable ${{ matrix.target }} - run: ci/build.bash cross ${{ matrix.target }} RELEASE - run: tar -czvf ${{ env.BIN }}.tar.gz --directory=target/${{ matrix.target }}/release ${{ env.BIN }} - uses: XAMPPRocky/create-release@v1.0.2 id: create_release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ github.ref }} release_name: ${{ github.ref }} draft: false prerelease: true - uses: actions/upload-release-asset@v1 id: upload-release-asset env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ${{ env.BIN }}.tar.gz asset_name: ${{ env.BIN }}-${{ matrix.target }}.tar.gz asset_content_type: application/gzip linux: runs-on: ubuntu-latest needs: install-cross strategy: fail-fast: false matrix: target: # WASM, off by default as most rust projects aren't compatible yet. # - wasm32-unknown-emscripten # Linux - aarch64-unknown-linux-gnu - arm-unknown-linux-gnueabi - armv7-unknown-linux-gnueabihf - i686-unknown-linux-gnu - i686-unknown-linux-musl - mips-unknown-linux-gnu - mips64-unknown-linux-gnuabi64 - mips64el-unknown-linux-gnuabi64 - mipsel-unknown-linux-gnu - powerpc64-unknown-linux-gnu - powerpc64le-unknown-linux-gnu - s390x-unknown-linux-gnu - x86_64-unknown-linux-gnu - x86_64-unknown-linux-musl # Android - aarch64-linux-android - arm-linux-androideabi - armv7-linux-androideabi - i686-linux-android - x86_64-linux-android # *BSD # The FreeBSD targets can have issues linking so they are disabled # by default. # - i686-unknown-freebsd # - x86_64-unknown-freebsd - x86_64-unknown-netbsd # Solaris - sparcv9-sun-solaris # Bare Metal # These are no-std embedded targets, so they will only build if your # crate is `no_std` compatible. # - thumbv6m-none-eabi # - thumbv7em-none-eabi # - thumbv7em-none-eabihf # - thumbv7m-none-eabi steps: - uses: actions/checkout@v3 - uses: actions/download-artifact@v4 with: name: cross-linux-musl path: /tmp/ - run: chmod +x /tmp/cross - run: ci/set_rust_version.bash stable ${{ matrix.target }} - run: ci/build.bash /tmp/cross ${{ matrix.target }} RELEASE - run: tar -czvf ${{ env.BIN }}.tar.gz --directory=target/${{ matrix.target }}/release ${{ env.BIN }} - uses: XAMPPRocky/create-release@v1.0.2 id: create_release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ github.ref }} release_name: ${{ github.ref }} draft: false prerelease: false - name: Upload Release Asset id: upload-release-asset uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} asset_path: ${{ env.BIN }}.tar.gz asset_name: ${{ env.BIN }}-${{ matrix.target }}.tar.gz asset_content_type: application/gzip ================================================ FILE: .github/workflows/publish_image.yaml ================================================ name: Publish Docker Images on: push: branches: - master tags: - v* jobs: image: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true runs-on: ubuntu-latest permissions: packages: write contents: read attestations: write steps: - uses: earthly/actions-setup@v1 with: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Check out the repo uses: actions/checkout@v4 - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v5 with: images: ghcr.io/${{ github.repository }} tags: | type=semver,pattern={{raw}} type=raw,value=latest,enable={{is_default_branch}} - name: Log in to the Container registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Push Docker Image run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} earthly --ci --push +docker --image_name="{}" ================================================ FILE: .github/workflows/release-plz.yaml ================================================ name: Release-plz permissions: pull-requests: write contents: write on: push: branches: - master jobs: release-plz: name: Release-plz runs-on: ubuntu-22.04 steps: - name: Checkout repository uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable - name: Run release-plz uses: MarcoIeni/release-plz-action@v0.5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} ================================================ FILE: .gitignore ================================================ # Created by https://www.toptal.com/developers/gitignore/api/rust # Edit at https://www.toptal.com/developers/gitignore?templates=rust ### Rust ### # Generated by Cargo # will have compiled files and executables debug/ target/ # These are backup files generated by rustfmt **/*.rs.bk # MSVC Windows builds of rustc generate these, which store debugging information *.pdb # End of https://www.toptal.com/developers/gitignore/api/rust ### IDE ### .vscode .idea/ *.iml ### Other ### # macOS .DS_Store # settings .settings .tokeirc # benchmark results.csv node_modules *.code-workspace ================================================ FILE: .mailmap ================================================ Erin Power Erin Power Erin Power <4464295+XAMPPRocky@users.noreply.github.com> Erin Power ================================================ FILE: .tokeignore ================================================ tests/data resources ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ## [14.0.0](https://github.com/XAMPPRocky/tokei/compare/v13.0.0...v14.0.0) - 2025-12-26 ### Added - add support for C++20 modules ([#1278](https://github.com/XAMPPRocky/tokei/pull/1278)) - Add language support for Ark TypeScript ([#1300](https://github.com/XAMPPRocky/tokei/pull/1300)) ### Other - Fix downcast type mismatches in clap_builder ([#1310](https://github.com/XAMPPRocky/tokei/pull/1310)) - remove tokei.rs references - Add support for Koka ([#1306](https://github.com/XAMPPRocky/tokei/pull/1306)) - Stop recommending comma-separated CLI args ([#1305](https://github.com/XAMPPRocky/tokei/pull/1305)) - Update clap-cargo from 0.13 to 0.18 ([#1298](https://github.com/XAMPPRocky/tokei/pull/1298)) ## [13.0.0-alpha.9](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.8...v13.0.0-alpha.9) - 2025-07-21 ### Other - Update README.md - Fix CRLF or mixed CRLF/LF line terminations in Markdown files ([#1219](https://github.com/XAMPPRocky/tokei/pull/1219)) - Fix a minor typo in CLI help text ([#1217](https://github.com/XAMPPRocky/tokei/pull/1217)) - Fix a missing space in CLI help text ([#1218](https://github.com/XAMPPRocky/tokei/pull/1218)) - Relax lifetime constraints in language::embedding ([#1225](https://github.com/XAMPPRocky/tokei/pull/1225)) ## [13.0.0-alpha.8](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.7...v13.0.0-alpha.8) - 2025-01-14 ### Other - add Mojo support ([#1107](https://github.com/XAMPPRocky/tokei/pull/1107)) ([#1185](https://github.com/XAMPPRocky/tokei/pull/1185)) - Add support for 8th language ([#1192](https://github.com/XAMPPRocky/tokei/pull/1192)) - Add support for `Roc` language ([#1197](https://github.com/XAMPPRocky/tokei/pull/1197)) - Add support for Ballerina language ([#1196](https://github.com/XAMPPRocky/tokei/pull/1196)) - Remove 'conf' extension from the Bitbake config. ([#1001](https://github.com/XAMPPRocky/tokei/pull/1001)) - Add support for Cairo language ([#1193](https://github.com/XAMPPRocky/tokei/pull/1193)) - Add support for Uiua language ([#1191](https://github.com/XAMPPRocky/tokei/pull/1191)) ## [13.0.0-alpha.7](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.6...v13.0.0-alpha.7) - 2024-11-10 ### Other - Fix alternative output formats ([#1188](https://github.com/XAMPPRocky/tokei/pull/1188)) - Add missing extension `fsti` for F* ([#1184](https://github.com/XAMPPRocky/tokei/pull/1184)) ## [13.0.0-alpha.6](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.5...v13.0.0-alpha.6) - 2024-10-11 ### Added - add `mbti` extension for MoonBit ([#1168](https://github.com/XAMPPRocky/tokei/pull/1168)) ### Other - Add language definition for Lauterbach PRACTICE Script ([#1162](https://github.com/XAMPPRocky/tokei/pull/1162)) - Add support for justfiles ([#1175](https://github.com/XAMPPRocky/tokei/pull/1175)) - Add Virgil ([#1178](https://github.com/XAMPPRocky/tokei/pull/1178)) - Add templ support ([#1122](https://github.com/XAMPPRocky/tokei/pull/1122)) - Update README.md with HiCAD from d4a1814 ([#1143](https://github.com/XAMPPRocky/tokei/pull/1143)) - Add BQN support ([#1151](https://github.com/XAMPPRocky/tokei/pull/1151)) - add more extensions for `Hlsl` ([#1164](https://github.com/XAMPPRocky/tokei/pull/1164)) - Add Phix ([#1167](https://github.com/XAMPPRocky/tokei/pull/1167)) - Add APL support ([#1152](https://github.com/XAMPPRocky/tokei/pull/1152)) - Add support for SIL ([#1153](https://github.com/XAMPPRocky/tokei/pull/1153)) - Use `OR` operator in Cargo.toml `license` field ([#1165](https://github.com/XAMPPRocky/tokei/pull/1165)) - Disable legacy Cargo features ([#1158](https://github.com/XAMPPRocky/tokei/pull/1158)) - add slint language support ([#1054](https://github.com/XAMPPRocky/tokei/pull/1054)) - Add Pyret support ([#1032](https://github.com/XAMPPRocky/tokei/pull/1032)) - Recognize GNUmakefile ([#1021](https://github.com/XAMPPRocky/tokei/pull/1021)) ## [13.0.0-alpha.5](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.4...v13.0.0-alpha.5) - 2024-08-23 ### Fixed - fix issue https://github.com/XAMPPRocky/tokei/issues/1147 ([#1149](https://github.com/XAMPPRocky/tokei/pull/1149)) ### Other - Fix issue [#1145](https://github.com/XAMPPRocky/tokei/pull/1145) (part 2) ([#1148](https://github.com/XAMPPRocky/tokei/pull/1148)) ## [13.0.0-alpha.4](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.3...v13.0.0-alpha.4) - 2024-08-22 ### Fixed - fix issue https://github.com/XAMPPRocky/tokei/issues/1145 ([#1146](https://github.com/XAMPPRocky/tokei/pull/1146)) ### Other - Add support for Glimmer JS/TS ([#1052](https://github.com/XAMPPRocky/tokei/pull/1052)) - Fix issue [#1141](https://github.com/XAMPPRocky/tokei/pull/1141) ([#1142](https://github.com/XAMPPRocky/tokei/pull/1142)) ## [13.0.0-alpha.3](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.2...v13.0.0-alpha.3) - 2024-08-20 ### Fixed - fix issue https://github.com/XAMPPRocky/tokei/issues/1138 ([#1139](https://github.com/XAMPPRocky/tokei/pull/1139)) ## [13.0.0-alpha.2](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.1...v13.0.0-alpha.2) - 2024-08-19 ### Added - Add support for Monkey C ([#1081](https://github.com/XAMPPRocky/tokei/pull/1081)) - added plantuml support ([#1125](https://github.com/XAMPPRocky/tokei/pull/1125)) - add language Tact ([#1103](https://github.com/XAMPPRocky/tokei/pull/1103)) - add support for bicep ([#1100](https://github.com/XAMPPRocky/tokei/pull/1100)) - add hledger ([#1121](https://github.com/XAMPPRocky/tokei/pull/1121)) - add SELinux CIL policy source files ([#1124](https://github.com/XAMPPRocky/tokei/pull/1124)) - --files argument now sorts alphabetically ([#1059](https://github.com/XAMPPRocky/tokei/pull/1059)) - add support for LALRPOP ([#1077](https://github.com/XAMPPRocky/tokei/pull/1077)) ### Fixed - read hidden from config file ([#1093](https://github.com/XAMPPRocky/tokei/pull/1093)) ### Other - Fix cargo audit issues ([#1137](https://github.com/XAMPPRocky/tokei/pull/1137)) - Add support for MDX ([#1046](https://github.com/XAMPPRocky/tokei/pull/1046)) - Add PRQL to README.md ([#1088](https://github.com/XAMPPRocky/tokei/pull/1088)) - add fypp extension `.fpp` to `languages.json` for Modern Fortran ([#1060](https://github.com/XAMPPRocky/tokei/pull/1060)) - Add support for Lex ([#1087](https://github.com/XAMPPRocky/tokei/pull/1087)) - Add d2 ([#1091](https://github.com/XAMPPRocky/tokei/pull/1091)) - Add support for Stata ([#1112](https://github.com/XAMPPRocky/tokei/pull/1112)) - Add support for CUE ([#1049](https://github.com/XAMPPRocky/tokei/pull/1049)) - bump libc from 0.2.147 to 0.2.155 ([#1104](https://github.com/XAMPPRocky/tokei/pull/1104)) - add cangjie language support ([#1127](https://github.com/XAMPPRocky/tokei/pull/1127)) ([#1128](https://github.com/XAMPPRocky/tokei/pull/1128)) - Add support for JSLT ([#1129](https://github.com/XAMPPRocky/tokei/pull/1129)) - Add Arturo support ([#1108](https://github.com/XAMPPRocky/tokei/pull/1108)) - Support Bazel's MODULE files and *.bzlmod files ([#1130](https://github.com/XAMPPRocky/tokei/pull/1130)) - read only first 128B from the file when searching for shebang ([#1040](https://github.com/XAMPPRocky/tokei/pull/1040)) - add OpenCL as a languages.json entry ([#980](https://github.com/XAMPPRocky/tokei/pull/980)) - Add GetText Portable Object (PO) files ([#814](https://github.com/XAMPPRocky/tokei/pull/814)) - Support godot shader ([#1118](https://github.com/XAMPPRocky/tokei/pull/1118)) - Add Modelica language ([#1061](https://github.com/XAMPPRocky/tokei/pull/1061)) - Add menhir support ([#781](https://github.com/XAMPPRocky/tokei/pull/781)) - Update README.md - [issue_1114] remove Cargo.lock from .gitignore ([#1115](https://github.com/XAMPPRocky/tokei/pull/1115)) - [issue_891] give more space for Files column ([#933](https://github.com/XAMPPRocky/tokei/pull/933)) - GitHub Action to publish docker images ([#1096](https://github.com/XAMPPRocky/tokei/pull/1096)) - Support MoonBit language. ([#1095](https://github.com/XAMPPRocky/tokei/pull/1095)) - Add OpenSCAD ([#1097](https://github.com/XAMPPRocky/tokei/pull/1097)) - add jinja extension for Jinja2 ([#1083](https://github.com/XAMPPRocky/tokei/pull/1083)) - Fix slang ([#1089](https://github.com/XAMPPRocky/tokei/pull/1089)) - Temporarily remove Hare - Support .pyi python file ([#1075](https://github.com/XAMPPRocky/tokei/pull/1075)) - add luau extension to lua ([#1066](https://github.com/XAMPPRocky/tokei/pull/1066)) - Adding support for Snakemake ([#1045](https://github.com/XAMPPRocky/tokei/pull/1045)) - Add Janet to languages.json ([#1042](https://github.com/XAMPPRocky/tokei/pull/1042)) - Add OpenQASM support ([#1041](https://github.com/XAMPPRocky/tokei/pull/1041)) - typst ([#1037](https://github.com/XAMPPRocky/tokei/pull/1037)) - Add the ZoKrates language ([#1035](https://github.com/XAMPPRocky/tokei/pull/1035)) - Add PRQL ([#1030](https://github.com/XAMPPRocky/tokei/pull/1030)) - remove refs ([#1006](https://github.com/XAMPPRocky/tokei/pull/1006)) - Add lingua franca language ([#993](https://github.com/XAMPPRocky/tokei/pull/993)) - Add support for Razor Components ([#992](https://github.com/XAMPPRocky/tokei/pull/992)) - Add arch's PKGBUILD files ([#972](https://github.com/XAMPPRocky/tokei/pull/972)) - Add Hare support ([#971](https://github.com/XAMPPRocky/tokei/pull/971)) - Add Max support ([#963](https://github.com/XAMPPRocky/tokei/pull/963)) - Add support for Chapel ([#960](https://github.com/XAMPPRocky/tokei/pull/960)) - Add language support for Slang ([#956](https://github.com/XAMPPRocky/tokei/pull/956)) - Update TypeScript language ([#953](https://github.com/XAMPPRocky/tokei/pull/953)) - Added support for Circom ([#949](https://github.com/XAMPPRocky/tokei/pull/949)) - link to earthly project ([#1078](https://github.com/XAMPPRocky/tokei/pull/1078)) ## [13.0.0-alpha.1](https://github.com/XAMPPRocky/tokei/compare/v13.0.0-alpha.0...v13.0.0-alpha.1) - 2024-03-04 ### Fixed - fixed language names not showing when in Light mode (light background) ([#1048](https://github.com/XAMPPRocky/tokei/pull/1048)) ### Other - Create release-plz.yaml - Update mean_bean_ci.yml - Fix LD Script language data ([#1028](https://github.com/XAMPPRocky/tokei/pull/1028)) - Fix language data example in CONTRIBUTING.md ([#1029](https://github.com/XAMPPRocky/tokei/pull/1029)) - Update dependencies - Add widget install instructions - Update mean_bean_ci.yml - Dockerize tokei ([#930](https://github.com/XAMPPRocky/tokei/pull/930)) - Ignore format commits for `languages.json` ([#1013](https://github.com/XAMPPRocky/tokei/pull/1013)) - Upgrade GitHub Actions ([#955](https://github.com/XAMPPRocky/tokei/pull/955)) - add --languages ouput formatter ([#1007](https://github.com/XAMPPRocky/tokei/pull/1007)) - Add Nuget Config, Bazel and EdgeQL Support, Fix Output Formatter ([#999](https://github.com/XAMPPRocky/tokei/pull/999)) - show nushell in the readme ([#991](https://github.com/XAMPPRocky/tokei/pull/991)) - Add support for Redscript ([#994](https://github.com/XAMPPRocky/tokei/pull/994)) - Add support for jq ([#965](https://github.com/XAMPPRocky/tokei/pull/965)) - Add support for Astro ([#966](https://github.com/XAMPPRocky/tokei/pull/966)) - Use XDG conventions on macOS too ([#989](https://github.com/XAMPPRocky/tokei/pull/989)) - Add JSON5 support for languages.json ([#986](https://github.com/XAMPPRocky/tokei/pull/986)) - Delete Smalltalk.cs.st ([#990](https://github.com/XAMPPRocky/tokei/pull/990)) - Add support for smalltalk ([#839](https://github.com/XAMPPRocky/tokei/pull/839)) - Disable *-android - Add HiCAD to languages.json ([#985](https://github.com/XAMPPRocky/tokei/pull/985)) - Add Nushell to languages.json ([#982](https://github.com/XAMPPRocky/tokei/pull/982)) # 12.1.0 ## Introduction Tokei is a fast and accurate code analysis CLI tool and library, allowing you to easily and quickly see how many blank lines, comments, and lines of code are in your codebase. All releases and work on Tokei and tokei.rs ([the free companion badge service][rs-info]) are [funded by the community through GitHub Sponsors][sponsor]. You can always download the latest version of tokei through GitHub Releases or Cargo. Tokei is also available through other [package managers][pkg], though they may not always contain the latest release. ``` cargo install tokei ``` [pkg]: https://github.com/XAMPPRocky/tokei#package-managers [rs-info]: https://github.com/XAMPPRocky/tokei/blob/master/README.md#Badges [sponsor]: https://github.com/sponsors/XAMPPRocky ## What's New? - [Added `-n/--num-format=[commas, dots, plain, underscores]` for adding separator formatting for numbers.](https://github.com/XAMPPRocky/tokei/pull/591) - [The total is now included in output formats such as JSON.](https://github.com/XAMPPRocky/tokei/pull/580) - [`--no-ignore` now implies other ignore flags.](https://github.com/XAMPPRocky/tokei/pull/588) - [Added `--no-ignore-dot` flag to ignore files such as `.ignore`.](https://github.com/XAMPPRocky/tokei/pull/588) - [Added single line comments to F\*](https://github.com/XAMPPRocky/tokei/pull/670) - Updated various dependencies. ### Added Languages - [ABNF](https://github.com/XAMPPRocky/tokei/pull/577) - [CodeQL](https://github.com/XAMPPRocky/tokei/pull/604) - [LiveScript](https://github.com/XAMPPRocky/tokei/pull/607) - [Stylus](https://github.com/XAMPPRocky/tokei/pull/619) - [DAML](https://github.com/XAMPPRocky/tokei/pull/620) - [Tera](https://github.com/XAMPPRocky/tokei/pull/627) - [TTCN-3](https://github.com/XAMPPRocky/tokei/pull/621) - [Beancount](https://github.com/XAMPPRocky/tokei/pull/630) - [Gleam](https://github.com/XAMPPRocky/tokei/pull/646) - [JSONNet](https://github.com/XAMPPRocky/tokei/pull/634) - [Stan](https://github.com/XAMPPRocky/tokei/pull/633) - [Gwion](https://github.com/XAMPPRocky/tokei/pull/659) # 12.0.0 ## What's New? Tokei 12 comes with some of the biggest user facing changes since 1.0, now in the latest version tokei will now **analyse and count multiple languages embedded in your source code** as well as adding support for **Jupyter Notebooks**. Now for the first time is able to handle and display different languages contained in a single source file. This is currently available for a limited set of languages, with plans to add more support for more in the future. The currently supported languages are; ### HTML + Siblings (Vue, Svelte, Etc...) Tokei will now analyse and report the source code contained in `"#).unwrap()); pub static START_STYLE: Lazy = Lazy::new(|| Regex::new(r#""#).unwrap()); pub static END_STYLE: Lazy = Lazy::new(|| Regex::new(r#""#).unwrap()); pub static START_TEMPLATE: Lazy = Lazy::new(|| Regex::new(r#""#).unwrap()); pub static END_TEMPLATE: Lazy = Lazy::new(|| Regex::new(r#""#).unwrap()); pub static STARTING_MARKDOWN_REGEX: Lazy = Lazy::new(|| Regex::new(r#"```\S+\s"#).unwrap()); pub static ENDING_MARKDOWN_REGEX: Lazy = Lazy::new(|| Regex::new(r#"```\s?"#).unwrap()); pub static STARTING_LF_BLOCK_REGEX: Lazy = Lazy::new(|| Regex::new(r#"\{="#).unwrap()); pub static ENDING_LF_BLOCK_REGEX: Lazy = Lazy::new(|| Regex::new(r#"=}"#).unwrap()); /// A memory of a regex matched. /// The values provided by `Self::start` and `Self::end` are in the same space as the /// start value supplied to `RegexCache::build` pub struct Capture<'a> { start: usize, text: &'a [u8], } impl Capture<'_> { #[inline(always)] fn start(&self) -> usize { self.start } #[inline(always)] pub fn end(&self) -> usize { self.start + self.text.len() } #[inline(always)] pub fn as_bytes(&self) -> &[u8] { self.text } } impl<'a> std::fmt::Debug for Capture<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Capture") .field("start", &self.start) .field("end", &self.end()) .field("text", &String::from_utf8_lossy(self.text)) .finish() } } pub(crate) struct RegexCache<'a> { inner: Option>, } /// Embedding regexes are similar between different sets of languages. /// `RegexFamily` records both which family the language belongs to, /// as well as the actual matches pub(crate) enum RegexFamily<'a> { HtmlLike(HtmlLike<'a>), LinguaFranca(SimpleCapture<'a>), Markdown(SimpleCapture<'a>), Rust, } pub(crate) struct HtmlLike<'a> { start_script: Option]>>, start_style: Option]>>, start_template: Option]>>, } pub(crate) struct SimpleCapture<'a> { starts: Option]>>, } impl<'a> HtmlLike<'a> { pub fn start_script_in_range<'this>( &'this self, start: usize, end: usize, ) -> Option>> { filter_range(self.start_script.as_ref()?, start, end) } pub fn start_style_in_range<'this>( &'this self, start: usize, end: usize, ) -> Option>> { filter_range(self.start_style.as_ref()?, start, end) } pub fn start_template_in_range<'this>( &'this self, start: usize, end: usize, ) -> Option>> { filter_range(self.start_template.as_ref()?, start, end) } } impl<'a> SimpleCapture<'a> { pub fn starts_in_range<'this>( &'this self, start: usize, end: usize, ) -> Option<&'this Capture<'a>> { filter_range(self.starts.as_ref()?, start, end).and_then(|mut it| it.next()) } fn make_capture( regex: &Regex, lines: &'a [u8], start: usize, end: usize, ) -> Option> { let capture = SimpleCapture { starts: save_captures(regex, lines, start, end), }; if capture.starts.is_some() { Some(capture) } else { None } } } fn filter_range<'dataset, 'cap>( dataset: &'dataset [Capture<'cap>], start: usize, end: usize, ) -> Option>> { let pos = dataset .binary_search_by_key(&start, |cap| cap.start()) .ok()?; if pos >= dataset.len() || dataset[pos].end() > end { None } else { Some( dataset[pos..] .iter() .take_while(move |cap| cap.end() <= end), ) } } impl<'a> RegexCache<'a> { /// Returns the language family for which regexes were matched, if any pub(crate) fn family(&self) -> Option<&RegexFamily> { self.inner.as_ref() } /// Tries to memoize any matches of embedding regexes that occur within lines[start..end] /// for the given language. Any `Capture` values eventually recovered will use the same /// zero for their start as the given `start` argument. pub(crate) fn build(lang: LanguageType, lines: &'a [u8], start: usize, end: usize) -> Self { let inner = match lang { LanguageType::Markdown | LanguageType::UnrealDeveloperMarkdown => { SimpleCapture::make_capture(&STARTING_MARKDOWN_REGEX, lines, start, end) .map(RegexFamily::Markdown) } LanguageType::Rust => Some(RegexFamily::Rust), LanguageType::LinguaFranca => { SimpleCapture::make_capture(&STARTING_LF_BLOCK_REGEX, lines, start, end) .map(RegexFamily::LinguaFranca) } LanguageType::Html | LanguageType::RubyHtml | LanguageType::Svelte | LanguageType::Vue | LanguageType::GlimmerJs | LanguageType::GlimmerTs => { let html = HtmlLike { start_script: save_captures(&START_SCRIPT, lines, start, end), start_style: save_captures(&START_STYLE, lines, start, end), start_template: save_captures(&START_TEMPLATE, lines, start, end), }; if html.start_script.is_some() || html.start_style.is_some() || html.start_template.is_some() { Some(RegexFamily::HtmlLike(html)) } else { None } } _ => None, }; Self { inner } } } fn save_captures<'a>( regex: &Regex, lines: &'a [u8], start: usize, end: usize, ) -> Option]>> { let v: Vec<_> = regex .captures(&lines[start..end])? .iter() .flatten() .map(|cap| Capture { start: start + cap.start(), text: cap.as_bytes(), }) .collect(); if v.is_empty() { None } else { Some(v.into()) } } ================================================ FILE: src/language/language_type.rs ================================================ use std::{ borrow::Cow, fmt, fs::File, io::{self, Read}, path::{Path, PathBuf}, str::FromStr, }; use crate::{ config::Config, language::syntax::{FileContext, LanguageContext, SyntaxCounter}, stats::{CodeStats, Report}, utils::{ext::SliceExt, fs as fsutils}, }; use encoding_rs_io::DecodeReaderBytesBuilder; use grep_searcher::{LineIter, LineStep}; use once_cell::sync::Lazy; use rayon::prelude::*; use serde::Serialize; use self::LanguageType::*; include!(concat!(env!("OUT_DIR"), "/language_type.rs")); impl Serialize for LanguageType { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { serializer.serialize_str(self.name()) } } impl LanguageType { /// Parses a given [`Path`] using the [`LanguageType`]. Returning [`Report`] /// on success and giving back ownership of [`PathBuf`] on error. pub fn parse(self, path: PathBuf, config: &Config) -> Result { let text = { let f = match File::open(&path) { Ok(f) => f, Err(e) => return Err((e, path)), }; let mut s = Vec::new(); let mut reader = DecodeReaderBytesBuilder::new().build(f); if let Err(e) = reader.read_to_end(&mut s) { return Err((e, path)); } s }; let mut stats = Report::new(path); stats += self.parse_from_slice(text, config); Ok(stats) } /// Parses the text provided as the given [`LanguageType`]. pub fn parse_from_str>(self, text: A, config: &Config) -> CodeStats { self.parse_from_slice(text.as_ref().as_bytes(), config) } /// Parses the bytes provided as the given [`LanguageType`]. pub fn parse_from_slice>(self, text: A, config: &Config) -> CodeStats { let text = text.as_ref(); if self == Jupyter { return self .parse_jupyter(text.as_ref(), config) .unwrap_or_default(); } let syntax = { let mut syntax_mut = SyntaxCounter::new(self); if self == LinguaFranca { syntax_mut.lf_embedded_language = self.find_lf_target_language(text); } syntax_mut }; if let Some(end) = syntax.shared.important_syntax.find(text).and_then(|m| { // Get the position of the last line before the important // syntax. text[..=m.start()] .iter() .rev() .position(|&c| c == b'\n') .filter(|&p| p != 0) .map(|p| m.start() - p) }) { let (skippable_text, rest) = text.split_at(end + 1); let is_fortran = syntax.shared.is_fortran; let is_literate = syntax.shared.is_literate; let comments = syntax.shared.line_comments; trace!( "Using Simple Parse on {:?}", String::from_utf8_lossy(skippable_text) ); let parse_lines = move || self.parse_lines(config, rest, CodeStats::new(), syntax); let simple_parse = move || { LineIter::new(b'\n', skippable_text) .par_bridge() .map(|line| { // FORTRAN has a rule where it only counts as a comment if it's the // first character in the column, so removing starting whitespace // could cause a miscount. let line = if is_fortran { line } else { line.trim() }; if line.trim().is_empty() { (1, 0, 0) } else if is_literate || comments.iter().any(|c| line.starts_with(c.as_bytes())) { (0, 0, 1) } else { (0, 1, 0) } }) .reduce(|| (0, 0, 0), |a, b| (a.0 + b.0, a.1 + b.1, a.2 + b.2)) }; let (mut stats, (blanks, code, comments)) = rayon::join(parse_lines, simple_parse); stats.blanks += blanks; stats.code += code; stats.comments += comments; stats } else { self.parse_lines(config, text, CodeStats::new(), syntax) } } #[inline] fn parse_lines( self, config: &Config, lines: &[u8], mut stats: CodeStats, mut syntax: SyntaxCounter, ) -> CodeStats { let mut stepper = LineStep::new(b'\n', 0, lines.len()); while let Some((start, end)) = stepper.next(lines) { let line = &lines[start..end]; // FORTRAN has a rule where it only counts as a comment if it's the // first character in the column, so removing starting whitespace // could cause a miscount. let line = if syntax.shared.is_fortran { line } else { line.trim() }; trace!("{}", String::from_utf8_lossy(line)); if syntax.try_perform_single_line_analysis(line, &mut stats) { continue; } let started_in_comments = !syntax.stack.is_empty() || (config.treat_doc_strings_as_comments == Some(true) && syntax.quote.is_some() && syntax.quote_is_doc_quote); let ended_with_comments = match syntax.perform_multi_line_analysis(lines, start, end, config) { crate::language::syntax::AnalysisReport::Normal(end) => end, crate::language::syntax::AnalysisReport::ChildLanguage(FileContext { language, end, stats: blob, }) => { match language { LanguageContext::Markdown { balanced, language } => { // Add the lines for the code fences. stats.comments += if balanced { 2 } else { 1 }; // Add the code inside the fence to the stats. *stats.blobs.entry(language).or_default() += blob; } LanguageContext::Rust => { // Add all the markdown blobs. *stats.blobs.entry(LanguageType::Markdown).or_default() += blob; } LanguageContext::LinguaFranca => { let child_lang = syntax.get_lf_target_language(); *stats.blobs.entry(child_lang).or_default() += blob; } LanguageContext::Html { language } => { stats.code += 1; // Add all the markdown blobs. *stats.blobs.entry(language).or_default() += blob; } } // Advance to after the language code and the delimiter.. stepper = LineStep::new(b'\n', end, lines.len()); continue; } }; trace!("{}", String::from_utf8_lossy(line)); if syntax.shared.is_literate || syntax.line_is_comment(line, config, ended_with_comments, started_in_comments) { stats.comments += 1; trace!("Comment No.{}", stats.comments); trace!("Was the Comment stack empty?: {}", !started_in_comments); } else { stats.code += 1; trace!("Code No.{}", stats.code); } } stats } fn parse_jupyter(&self, json: &[u8], config: &Config) -> Option { #[derive(Deserialize)] struct Jupyter { cells: Vec, metadata: JupyterMetadata, } #[derive(Clone, Copy, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] enum CellType { Markdown, Code, } #[derive(Deserialize)] struct JupyterCell { cell_type: CellType, source: Vec, } #[derive(Deserialize)] struct JupyterMetadata { kernelspec: serde_json::Value, language_info: serde_json::Value, } let jupyter: Jupyter = serde_json::from_slice(json).ok()?; let mut jupyter_stats = CodeStats::new(); let language = jupyter .metadata .kernelspec .get("language") .and_then(serde_json::Value::as_str) .and_then(|v| LanguageType::from_str(v).ok()) .or_else(|| { jupyter .metadata .language_info .get("file_extension") .and_then(serde_json::Value::as_str) .and_then(LanguageType::from_file_extension) }) .unwrap_or(LanguageType::Python); let iter = jupyter .cells .par_iter() .map(|cell| match cell.cell_type { CellType::Markdown => ( LanguageType::Markdown, LanguageType::Markdown.parse_from_str(cell.source.join(""), config), ), CellType::Code => ( language, language.parse_from_str(cell.source.join(""), config), ), }) .collect::>(); for (language, stats) in iter { *jupyter_stats.blobs.entry(language).or_default() += &stats; jupyter_stats += &stats; } Some(jupyter_stats) } /// The embedded language in LF is declared in a construct that looks like this: `target C;`, `target Python`. /// This is the first thing in the file (although there may be comments before). fn find_lf_target_language(&self, bytes: &[u8]) -> Option { use regex::bytes::Regex; static LF_TARGET_REGEX: Lazy = Lazy::new(|| Regex::new(r#"(?m)\btarget\s+(\w+)\s*($|;|\{)"#).unwrap()); LF_TARGET_REGEX.captures(bytes).and_then(|captures| { let name = captures.get(1).unwrap().as_bytes(); if name == b"CCpp" { // this is a special alias for the C target in LF Some(C) } else { let name_str = &String::from_utf8_lossy(name); let by_name = LanguageType::from_name(name_str); if by_name.is_none() { trace!("LF target not recognized: {}", name_str); } by_name } }) } } #[cfg(test)] mod tests { use super::*; use std::{fs, path::Path}; #[test] fn rust_allows_nested() { assert!(LanguageType::Rust.allows_nested()); } fn assert_stats(stats: &CodeStats, blanks: usize, code: usize, comments: usize) { assert_eq!(stats.blanks, blanks, "expected {} blank lines", blanks); assert_eq!(stats.code, code, "expected {} code lines", code); assert_eq!( stats.comments, comments, "expected {} comment lines", comments ); } #[test] fn jupyter_notebook_has_correct_totals() { let sample_notebook = fs::read_to_string(Path::new("tests").join("data").join("jupyter.ipynb")).unwrap(); let stats = LanguageType::Jupyter .parse_jupyter(sample_notebook.as_bytes(), &Config::default()) .unwrap(); assert_stats(&stats, 115, 528, 333); } #[test] fn lf_embedded_language_is_counted() { let file_text = fs::read_to_string(Path::new("tests").join("data").join("linguafranca.lf")).unwrap(); let stats = LinguaFranca.parse_from_str(file_text, &Config::default()); assert_stats(&stats, 9, 11, 8); assert_eq!(stats.blobs.len(), 1, "num embedded languages"); let rust_stats = stats.blobs.get(&Rust).expect("should have a Rust entry"); assert_stats(rust_stats, 2, 5, 1); } } ================================================ FILE: src/language/language_type.tera.rs ================================================ use arbitrary::Arbitrary; /// Represents a individual programming language. Can be used to provide /// information about the language, such as multi line comments, single line /// comments, string literal syntax, whether a given language allows nesting /// comments. #[derive(Deserialize)] #[derive(Arbitrary, Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[non_exhaustive] #[allow(clippy::upper_case_acronyms)] pub enum LanguageType { {% for key, value in languages -%} #[allow(missing_docs)] {% if value.name is defined %} #[serde(alias = "{{value.name}}")] {% else %} #[serde(alias = "{{key}}")] {% endif %} {{key}}, {% endfor %} } impl LanguageType { /// Returns the display name of a language. /// /// ``` /// # use tokei::*; /// let bash = LanguageType::Bash; /// /// assert_eq!(bash.name(), "BASH"); /// ``` pub fn name(self) -> &'static str { match self { {% for key, value in languages -%} {{key}} => {% if value.name %}"{{value.name}}"{% else %}"{{key}}"{% endif %}, {% endfor %} } } pub(crate) fn _is_blank(self) -> bool { match self { {% for key, v in languages -%} {{key}} => {{ v.blank | default(value=false) }}, {% endfor %} } } pub(crate) fn is_fortran(self) -> bool { self == LanguageType::FortranModern || self == LanguageType::FortranLegacy } /// Returns whether the language is "literate", meaning that it considered /// to primarily be documentation and is counted primarily as comments /// rather than procedural code. pub fn is_literate(self) -> bool { match self { {% for key, v in languages -%} {{key}} => {{ v.literate | default(value=false) }}, {% endfor %} } } /// Provides every variant in a Vec pub fn list() -> &'static [(Self, &'static [&'static str])] { &[{% for key, val in languages -%} ({{key}}, {% if val.extensions %} &[{% for extension in val.extensions %}"{{extension}}", {% endfor %}], {% else %} &[], {% endif %}), {% endfor %}] } /// Returns the single line comments of a language. /// ``` /// use tokei::LanguageType; /// let lang = LanguageType::Rust; /// assert_eq!(lang.line_comments(), &["//"]); /// ``` pub fn line_comments(self) -> &'static [&'static str] { match self { {% for key, value in languages -%} {{key}} => &[{% for item in value.line_comment | default(value=[]) %}"{{item}}",{% endfor %}], {% endfor %} } } /// Returns the single line comments of a language. /// ``` /// use tokei::LanguageType; /// let lang = LanguageType::Rust; /// assert_eq!(lang.multi_line_comments(), &[("/*", "*/")]); /// ``` pub fn multi_line_comments(self) -> &'static [(&'static str, &'static str)] { match self { {% for key, value in languages -%} {{key}} => &[ {%- for items in value.multi_line_comments | default(value=[]) -%} ({% for item in items %}"{{item}}",{% endfor %}), {%- endfor -%} ], {% endfor %} } } /// Returns whether the language allows nested multi line comments. /// ``` /// use tokei::LanguageType; /// let lang = LanguageType::Rust; /// assert!(lang.allows_nested()); /// ``` pub fn allows_nested(self) -> bool { match self { {% for key, v in languages -%} {{key}} => {{ v.nested | default(value=false) }}, {% endfor %} } } /// Returns what nested comments the language has. (Currently only D has /// any of this type.) /// ``` /// use tokei::LanguageType; /// let lang = LanguageType::D; /// assert_eq!(lang.nested_comments(), &[("/+", "+/")]); /// ``` pub fn nested_comments(self) -> &'static [(&'static str, &'static str)] { match self { {% for key, value in languages -%} {{key}} => &[ {%- for items in value.nested_comments | default(value=[]) -%} ({% for item in items %}"{{item}}",{% endfor %}), {%- endfor -%} ], {% endfor %} } } /// Returns the quotes of a language. /// ``` /// use tokei::LanguageType; /// let lang = LanguageType::C; /// assert_eq!(lang.quotes(), &[("\"", "\"")]); /// ``` pub fn quotes(self) -> &'static [(&'static str, &'static str)] { match self { {% for key, value in languages -%} {{key}} => &[ {%- for items in value.quotes | default(value=[]) -%} ({% for item in items %}"{{item}}",{% endfor %}), {%- endfor -%} ], {% endfor %} } } /// Returns the verbatim quotes of a language. /// ``` /// use tokei::LanguageType; /// let lang = LanguageType::CSharp; /// assert_eq!(lang.verbatim_quotes(), &[("@\"", "\"")]); /// ``` pub fn verbatim_quotes(self) -> &'static [(&'static str, &'static str)] { match self { {% for key, value in languages -%} {{key}} => &[ {%- for items in value.verbatim_quotes | default(value=[]) -%} ({% for item in items %}"{{item}}",{% endfor %}), {%- endfor -%} ], {% endfor %} } } /// Returns the doc quotes of a language. /// ``` /// use tokei::LanguageType; /// let lang = LanguageType::Python; /// assert_eq!(lang.doc_quotes(), &[("\"\"\"", "\"\"\""), ("'''", "'''")]); /// ``` pub fn doc_quotes(self) -> &'static [(&'static str, &'static str)] { match self { {% for key, value in languages -%} {{key}} => &[ {% for items in value.doc_quotes | default(value=[])-%} ({% for item in items %}"{{item}}",{% endfor %}), {%- endfor %} ], {%- endfor %} } } /// Returns the shebang of a language. /// ``` /// use tokei::LanguageType; /// let lang = LanguageType::Bash; /// assert_eq!(lang.shebangs(), &["#!/bin/bash"]); /// ``` pub fn shebangs(self) -> &'static [&'static str] { match self { {% for key, lang in languages -%} {{key}} => &[{% for item in lang.shebangs | default(value=[]) %}"{{item}}",{% endfor %}], {% endfor %} } } pub(crate) fn any_multi_line_comments(self) -> &'static [(&'static str, &'static str)] { match self { {% for key, value in languages -%} {{key}} => &[ {%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) -%} {%- set starting_nested_comments = value.nested_comments | default(value=[]) -%} {%- for item in starting_multi_line_comments | concat(with=starting_nested_comments) -%} ("{{item.0}}", "{{item.1}}"), {%- endfor -%} ], {% endfor %} } } pub(crate) fn any_comments(self) -> &'static [&'static str] { match self { {% for key, value in languages -%} {{key}} => &[ {%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) -%} {%- set starting_nested_comments = value.nested_comments | default(value=[]) -%} {%- for item in starting_multi_line_comments | concat(with=starting_nested_comments) -%} "{{item.0}}", "{{item.1}}", {%- endfor -%} {%- for item in value.line_comment | default(value=[]) -%} "{{item}}", {%- endfor -%} ], {% endfor %} } } /// Returns the parts of syntax that determines whether tokei can skip large /// parts of analysis. pub fn important_syntax(self) -> &'static [&'static str] { match self { {% for key, value in languages -%} {%- set starting_quotes = value.quotes | default(value=[]) | map(attribute="0") -%} {%- set starting_doc_quotes = value.doc_quotes | default(value=[]) | map(attribute="0") -%} {%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) | map(attribute="0") -%} {%- set starting_nested_comments = value.nested_comments | default(value=[]) | map(attribute="0") -%} {%- set important_syntax = value.important_syntax | default(value=[]) -%} {{key}} => &[ {%- for item in starting_quotes | concat(with=starting_doc_quotes) | concat(with=starting_multi_line_comments) | concat(with=starting_nested_comments) | concat(with=important_syntax) -%} "{{item}}", {%- endfor -%} {%- for context in value.contexts | default(value=[]) -%} {% if value.kind == "html" %} "<{{context.tag}}", {% endif %} {%- endfor -%} ], {% endfor %} } } /// Get language from a file path. May open and read the file. /// /// ```no_run /// use tokei::{Config, LanguageType}; /// /// let rust = LanguageType::from_path("./main.rs", &Config::default()); /// /// assert_eq!(rust, Some(LanguageType::Rust)); /// ``` pub fn from_path>(entry: P, _config: &Config) -> Option { let entry = entry.as_ref(); if let Some(filename) = fsutils::get_filename(entry) { match &*filename { {% for key, value in languages -%} {%- if value.filenames -%} {%- for item in value.filenames -%} | "{{item}}" {%- endfor -%} => return Some({{key}}), {% endif -%} {%- endfor %} _ => () } } match fsutils::get_extension(entry) { Some(extension) => LanguageType::from_file_extension(extension.as_str()), None => LanguageType::from_shebang(entry), } } /// Get language from a file extension. /// /// ```no_run /// use tokei::LanguageType; /// /// let rust = LanguageType::from_file_extension("rs"); /// /// assert_eq!(rust, Some(LanguageType::Rust)); /// ``` #[must_use] pub fn from_file_extension(extension: &str) -> Option { match extension { {% for key, value in languages -%} {%- if value.extensions -%} {%- for item in value.extensions %}| "{{item}}" {% endfor %}=> Some({{key}}), {% endif -%} {%- endfor %} extension => { warn!("Unknown extension: {}", extension); None }, } } /// Get language from its name. /// /// ```no_run /// use tokei::LanguageType; /// /// let rust = LanguageType::from_name("Rust"); /// /// assert_eq!(rust, Some(LanguageType::Rust)); /// ``` #[must_use] pub fn from_name(name: &str) -> Option { match name { {% for key, value in languages -%} {% if value.name and value.name != key -%} | "{{value.name}}" {% endif -%} | "{{key}}" => Some({{key}}), {% endfor %} unknown => { warn!("Unknown language name: {}", unknown); None }, } } /// Get language from its MIME type if available. /// /// ```no_run /// use tokei::LanguageType; /// /// let lang = LanguageType::from_mime("application/javascript"); /// /// assert_eq!(lang, Some(LanguageType::JavaScript)); /// ``` #[must_use] pub fn from_mime(mime: &str) -> Option { match mime { {% for key, value in languages -%} {%- if value.mime -%} {%- for item in value.mime %}| "{{item}}" {% endfor %}=> Some({{key}}), {% endif -%} {%- endfor %} _ => { warn!("Unknown MIME: {}", mime); None }, } } /// Get language from a shebang. May open and read the file. /// /// ```no_run /// use tokei::LanguageType; /// /// let rust = LanguageType::from_shebang("./main.rs"); /// /// assert_eq!(rust, Some(LanguageType::Rust)); /// ``` pub fn from_shebang>(entry: P) -> Option { // Read at max `READ_LIMIT` bytes from the given file. // A typical shebang line has a length less than 32 characters; // e.g. '#!/bin/bash' - 11B / `#!/usr/bin/env python3` - 22B // It is *very* unlikely the file contains a valid shebang syntax // if we don't find a newline character after searching the first 128B. const READ_LIMIT: usize = 128; let mut file = File::open(entry).ok()?; let mut buf = [0; READ_LIMIT]; let len = file.read(&mut buf).ok()?; let buf = &buf[..len]; let first_line = buf.split(|b| *b == b'\n').next()?; let first_line = std::str::from_utf8(first_line).ok()?; let mut words = first_line.split_whitespace(); match words.next() { {# First match against any shebang paths, and then check if the language matches any found in the environment shebang path. #} {% for key, value in languages -%} {%- if value.shebangs %} {%- for item in value.shebangs %}| Some("{{item}}") {% endfor %}=> Some({{key}}), {% endif -%} {%- endfor %} Some("#!/usr/bin/env") => { if let Some(word) = words.next() { match word { {% for key, value in languages -%} {%- if value.env -%} {%- for item in value.env %} {% if loop.index == 1 %} _ if word.starts_with("{{item}}") {% else %} || word.starts_with("{{item}}") {% endif %} {% endfor %}=> Some({{key}}), {% endif -%} {%- endfor %} env => { warn!("Unknown environment: {:?}", env); None } } } else { None } } _ => None, } } } impl FromStr for LanguageType { type Err = &'static str; fn from_str(from: &str) -> Result { match &*from.to_lowercase() { {% for key, value in languages %} {% if value.name %}"{{value.name | lower}}"{% else %}"{{key | lower}}"{% endif %} => Ok({{key}}), {% endfor %} _ => Err("Language not found, please use `-l` to see all available\ languages."), } } } impl fmt::Display for LanguageType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.name()) } } impl<'a> From for Cow<'a, LanguageType> { fn from(from: LanguageType) -> Self { Cow::Owned(from) } } impl<'a> From<&'a LanguageType> for Cow<'a, LanguageType> { fn from(from: &'a LanguageType) -> Self { Cow::Borrowed(from) } } ================================================ FILE: src/language/languages.rs ================================================ use std::{ collections::{btree_map, BTreeMap}, iter::IntoIterator, ops::{AddAssign, Deref, DerefMut}, path::Path, }; use rayon::prelude::*; use crate::{ config::Config, language::{Language, LanguageType}, utils, }; /// A newtype representing a list of languages counted in the provided /// directory. /// ([_List of /// Languages_](https://github.com/XAMPPRocky/tokei#supported-languages)) #[derive(Debug, Default, PartialEq)] pub struct Languages { inner: BTreeMap, } impl serde::Serialize for Languages { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { self.inner.serialize(serializer) } } impl<'de> serde::Deserialize<'de> for Languages { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { let map = <_>::deserialize(deserializer)?; Ok(Self::from_previous(map)) } } impl Languages { fn from_previous(map: BTreeMap) -> Self { use std::collections::btree_map::Entry; let mut me = Self::new(); for (name, input_language) in map { match me.entry(name) { Entry::Occupied(mut entry) => { *entry.get_mut() += input_language; } Entry::Vacant(entry) => { entry.insert(input_language); } } } me } /// Populates the `Languages` struct with statistics about languages /// provided by [`Language`]. /// /// Takes a `&[&str]` of paths to recursively traverse, paths can be /// relative, absolute or glob paths. A second `&[&str]` of paths to ignore, /// these strings use the `.gitignore` syntax, such as `target` /// or `**/*.bk`. /// /// ```no_run /// use tokei::{Config, Languages}; /// /// let mut languages = Languages::new(); /// languages.get_statistics(&["."], &[".git", "target"], &Config::default()); /// ``` /// /// [`Language`]: struct.Language.html pub fn get_statistics>( &mut self, paths: &[A], ignored: &[&str], config: &Config, ) { utils::fs::get_all_files(paths, ignored, &mut self.inner, config); self.inner.par_iter_mut().for_each(|(_, l)| l.total()); } /// Constructs a new, Languages struct. Languages is always empty and does /// not allocate. /// /// ```rust /// # use tokei::*; /// let languages = Languages::new(); /// ``` #[must_use] pub fn new() -> Self { Languages::default() } /// Summary of the Languages struct. #[must_use] pub fn total(self: &Languages) -> Language { let mut total = Language::new(); for (ty, l) in self { let language = l.summarise(); total.comments += language.comments; total.blanks += language.blanks; total.code += language.code; total.inaccurate |= language.inaccurate; total.children.insert(*ty, language.reports.clone()); } total } } impl IntoIterator for Languages { type Item = as IntoIterator>::Item; type IntoIter = as IntoIterator>::IntoIter; fn into_iter(self) -> Self::IntoIter { self.inner.into_iter() } } impl<'a> IntoIterator for &'a Languages { type Item = (&'a LanguageType, &'a Language); type IntoIter = btree_map::Iter<'a, LanguageType, Language>; fn into_iter(self) -> Self::IntoIter { self.inner.iter() } } impl<'a> IntoIterator for &'a mut Languages { type Item = (&'a LanguageType, &'a mut Language); type IntoIter = btree_map::IterMut<'a, LanguageType, Language>; fn into_iter(self) -> Self::IntoIter { self.inner.iter_mut() } } impl AddAssign> for Languages { fn add_assign(&mut self, rhs: BTreeMap) { for (name, language) in rhs { if let Some(result) = self.inner.get_mut(&name) { *result += language; } } } } impl Deref for Languages { type Target = BTreeMap; fn deref(&self) -> &Self::Target { &self.inner } } impl DerefMut for Languages { fn deref_mut(&mut self) -> &mut BTreeMap { &mut self.inner } } ================================================ FILE: src/language/mod.rs ================================================ mod embedding; pub mod language_type; pub mod languages; mod syntax; use std::{collections::BTreeMap, mem, ops::AddAssign}; pub use self::{language_type::*, languages::Languages}; use crate::{sort::Sort, stats::Report}; /// A struct representing statistics about a single Language. #[derive(Clone, Debug, Deserialize, Default, PartialEq, Serialize)] pub struct Language { /// The total number of blank lines. pub blanks: usize, /// The total number of lines of code. pub code: usize, /// The total number of comments(both single, and multi-line) pub comments: usize, /// A collection of statistics of individual files. pub reports: Vec, /// A map of any languages found in the reports. pub children: BTreeMap>, /// Whether this language had problems with file parsing pub inaccurate: bool, } impl Language { /// Constructs a new empty Language with the comments provided. /// /// ``` /// # use tokei::*; /// let mut rust = Language::new(); /// ``` #[must_use] pub fn new() -> Self { Self::default() } /// Returns the total number of lines. #[inline] #[must_use] pub fn lines(&self) -> usize { self.blanks + self.code + self.comments } /// Add a `Report` to the Language. This will not update the totals in the /// Language struct. pub fn add_report(&mut self, report: Report) { for (lang, stats) in &report.stats.blobs { let mut new_report = Report::new(report.name.clone()); new_report.stats = stats.clone(); self.children.entry(*lang).or_default().push(new_report); } self.reports.push(report); } /// Marks this language as possibly not reflecting correct stats. #[inline] pub fn mark_inaccurate(&mut self) { self.inaccurate = true; } /// Creates a new `Language` from `self`, which is a summarised version /// of the language that doesn't contain any children. It will count /// non-blank lines in child languages as code unless the child language is /// considered "literate" then it will be counted as comments. #[must_use] pub fn summarise(&self) -> Language { let mut summary = self.clone(); for reports in self.children.values() { for stats in reports.iter().map(|r| r.stats.summarise()) { summary.comments += stats.comments; summary.code += stats.code; summary.blanks += stats.blanks; } } summary } /// Totals up the statistics of the `Stat` structs currently contained in /// the language. /// /// ```no_run /// use std::{collections::BTreeMap, path::PathBuf}; /// use tokei::Language; /// /// let mut language = Language::new(); /// /// // Add stats... /// /// assert_eq!(0, language.lines()); /// /// language.total(); /// /// assert_eq!(10, language.lines()); /// ``` pub fn total(&mut self) { let mut blanks = 0; let mut code = 0; let mut comments = 0; for report in &self.reports { blanks += report.stats.blanks; code += report.stats.code; comments += report.stats.comments; } self.blanks = blanks; self.code = code; self.comments = comments; } /// Checks if the language is empty. Empty meaning it doesn't have any /// statistics. /// /// ``` /// # use tokei::*; /// let rust = Language::new(); /// /// assert!(rust.is_empty()); /// ``` #[must_use] pub fn is_empty(&self) -> bool { self.code == 0 && self.comments == 0 && self.blanks == 0 && self.children.is_empty() } /// Sorts each of the `Report`s contained in the language based /// on what category is provided. /// /// ```no_run /// use std::{collections::BTreeMap, path::PathBuf}; /// use tokei::{Language, Sort}; /// /// let mut language = Language::new(); /// /// // Add stats... /// /// language.sort_by(Sort::Lines); /// assert_eq!(20, language.reports[0].stats.lines()); /// /// language.sort_by(Sort::Code); /// assert_eq!(8, language.reports[0].stats.code); /// ``` pub fn sort_by(&mut self, category: Sort) { match category { Sort::Blanks => self .reports .sort_by(|a, b| b.stats.blanks.cmp(&a.stats.blanks)), Sort::Comments => self .reports .sort_by(|a, b| b.stats.comments.cmp(&a.stats.comments)), Sort::Code => self.reports.sort_by(|a, b| b.stats.code.cmp(&a.stats.code)), Sort::Files => self.reports.sort_by(|a, b| a.name.cmp(&b.name)), Sort::Lines => self .reports .sort_by(|a, b| b.stats.lines().cmp(&a.stats.lines())), } } } impl AddAssign for Language { fn add_assign(&mut self, mut rhs: Self) { self.comments += rhs.comments; self.blanks += rhs.blanks; self.code += rhs.code; self.reports.extend(mem::take(&mut rhs.reports)); self.children.extend(mem::take(&mut rhs.children)); self.inaccurate |= rhs.inaccurate; } } ================================================ FILE: src/language/syntax.rs ================================================ use std::sync::Arc; use aho_corasick::AhoCorasick; use dashmap::DashMap; use grep_searcher::LineStep; use log::Level::Trace; use once_cell::sync::Lazy; use super::embedding::{ RegexCache, RegexFamily, ENDING_LF_BLOCK_REGEX, ENDING_MARKDOWN_REGEX, END_SCRIPT, END_STYLE, END_TEMPLATE, }; use crate::LanguageType::LinguaFranca; use crate::{stats::CodeStats, utils::ext::SliceExt, Config, LanguageType}; /// Tracks the syntax of the language as well as the current state in the file. /// Current has what could be consider three types of mode. /// - `plain` mode: This is the normal state, blanks are counted as blanks, /// string literals can trigger `string` mode, and comments can trigger /// `comment` mode. /// - `string` mode: This when the state machine is current inside a string /// literal for a given language, comments cannot trigger `comment` mode while /// in `string` mode. /// - `comment` mode: This when the state machine is current inside a comment /// for a given language, strings cannot trigger `string` mode while in /// `comment` mode. #[derive(Clone, Debug)] pub(crate) struct SyntaxCounter { pub(crate) shared: Arc, pub(crate) quote: Option<&'static str>, pub(crate) quote_is_doc_quote: bool, pub(crate) stack: Vec<&'static str>, pub(crate) quote_is_verbatim: bool, pub(crate) lf_embedded_language: Option, } #[derive(Clone, Debug)] pub(crate) struct FileContext { pub(crate) language: LanguageContext, pub(crate) stats: CodeStats, pub(crate) end: usize, } impl FileContext { pub fn new(language: LanguageContext, end: usize, stats: CodeStats) -> Self { Self { language, stats, end, } } } #[derive(Clone, Debug)] pub(crate) enum LanguageContext { Html { language: LanguageType, }, LinguaFranca, Markdown { balanced: bool, language: LanguageType, }, Rust, } #[derive(Clone, Debug)] pub(crate) struct SharedMatchers { pub language: LanguageType, pub allows_nested: bool, pub doc_quotes: &'static [(&'static str, &'static str)], pub important_syntax: AhoCorasick, #[allow(dead_code)] pub any_comments: &'static [&'static str], pub is_fortran: bool, pub is_literate: bool, pub line_comments: &'static [&'static str], pub any_multi_line_comments: &'static [(&'static str, &'static str)], pub multi_line_comments: &'static [(&'static str, &'static str)], pub nested_comments: &'static [(&'static str, &'static str)], pub string_literals: &'static [(&'static str, &'static str)], pub verbatim_string_literals: &'static [(&'static str, &'static str)], } impl SharedMatchers { pub fn new(language: LanguageType) -> Arc { static MATCHERS: Lazy>> = Lazy::new(DashMap::new); MATCHERS .entry(language) .or_insert_with(|| Arc::new(Self::init(language))) .value() .clone() } pub fn init(language: LanguageType) -> Self { fn init_corasick(pattern: &[&'static str]) -> AhoCorasick { AhoCorasick::builder() .match_kind(aho_corasick::MatchKind::LeftmostLongest) .start_kind(aho_corasick::StartKind::Unanchored) .prefilter(true) .kind(Some(aho_corasick::AhoCorasickKind::DFA)) .build(pattern) .unwrap() } Self { language, allows_nested: language.allows_nested(), doc_quotes: language.doc_quotes(), is_fortran: language.is_fortran(), is_literate: language.is_literate(), important_syntax: init_corasick(language.important_syntax()), any_comments: language.any_comments(), line_comments: language.line_comments(), multi_line_comments: language.multi_line_comments(), any_multi_line_comments: language.any_multi_line_comments(), nested_comments: language.nested_comments(), string_literals: language.quotes(), verbatim_string_literals: language.verbatim_quotes(), } } } #[derive(Debug)] pub(crate) enum AnalysisReport { /// No child languages were found, contains a boolean representing whether /// the line ended with comments or not. Normal(bool), ChildLanguage(FileContext), } impl SyntaxCounter { pub(crate) fn new(language: LanguageType) -> Self { Self { shared: SharedMatchers::new(language), quote_is_doc_quote: false, quote_is_verbatim: false, stack: Vec::with_capacity(1), lf_embedded_language: None, quote: None, } } /// Returns whether the syntax is currently in plain mode. pub(crate) fn is_plain_mode(&self) -> bool { self.quote.is_none() && self.stack.is_empty() } /// Returns whether the syntax is currently in string mode. pub(crate) fn _is_string_mode(&self) -> bool { self.quote.is_some() } /// Returns whether the syntax is currently in comment mode. pub(crate) fn _is_comment_mode(&self) -> bool { !self.stack.is_empty() } pub(crate) fn get_lf_target_language(&self) -> LanguageType { // in case the target declaration was not found, default it to that language const DEFAULT_LANG: LanguageType = LinguaFranca; self.lf_embedded_language.unwrap_or(DEFAULT_LANG) } #[inline] pub(crate) fn parse_line_comment(&self, window: &[u8]) -> bool { if self.quote.is_some() || !self.stack.is_empty() { false } else if let Some(comment) = self .shared .line_comments .iter() .find(|c| window.starts_with(c.as_bytes())) { trace!("Start {:?}", comment); true } else { false } } /// Try to see if we can determine what a line is from examining the whole /// line at once. Returns `true` if successful. pub(crate) fn try_perform_single_line_analysis( &self, line: &[u8], stats: &mut crate::stats::CodeStats, ) -> bool { if !self.is_plain_mode() { false } else if line.trim().is_empty() { stats.blanks += 1; trace!("Blank No.{}", stats.blanks); true } else if self.shared.important_syntax.is_match(line) { false } else { trace!("^ Skippable"); if self.shared.is_literate || self .shared .line_comments .iter() .any(|c| line.starts_with(c.as_bytes())) { stats.comments += 1; trace!("Comment No.{}", stats.comments); } else { stats.code += 1; trace!("Code No.{}", stats.code); } true } } pub(crate) fn perform_multi_line_analysis( &mut self, lines: &[u8], start: usize, end: usize, config: &Config, ) -> AnalysisReport { let mut ended_with_comments = false; let mut skip = 0; macro_rules! skip { ($skip:expr) => {{ skip = $skip - 1; }}; } let regex_cache = RegexCache::build(self.shared.language, lines, start, end); for i in start..end { if skip != 0 { skip -= 1; continue; } let window = &lines[i..]; if window.trim().is_empty() { break; } ended_with_comments = false; let is_end_of_quote_or_multi_line = self .parse_end_of_quote(window) .or_else(|| self.parse_end_of_multi_line(window)); if let Some(skip_amount) = is_end_of_quote_or_multi_line { ended_with_comments = true; skip!(skip_amount); continue; } else if self.quote.is_some() { continue; } if let Some(child) = self.parse_context(lines, i, end, config, ®ex_cache) { return AnalysisReport::ChildLanguage(child); } let is_quote_or_multi_line = self .parse_quote(window) .or_else(|| self.parse_multi_line_comment(window)); if let Some(skip_amount) = is_quote_or_multi_line { skip!(skip_amount); continue; } if self.parse_line_comment(window) { ended_with_comments = true; break; } } AnalysisReport::Normal(ended_with_comments) } /// Performs a set of heuristics to determine whether a line is a comment or /// not. The procedure is as follows. /// /// - Yes/No: Counted as Comment /// /// 1. Check if we're in string mode /// 1. Check if string literal is a doc string and whether tokei has /// been configured to treat them as comments. /// - Yes: When the line starts with the doc string or when we are /// continuing from a previous line. /// - No: The string is a normal string literal or tokei isn't /// configured to count them as comments. /// 2. If we're not in string mode, check if we left it this on this line. /// - Yes: When we found a doc quote and we started in comments. /// 3. Yes: When the whole line is a comment e.g. `/* hello */` /// 4. Yes: When the previous line started a multi-line comment. /// 5. Yes: When the line starts with a comment. /// 6. No: Any other input. pub(crate) fn line_is_comment( &self, line: &[u8], config: &crate::Config, _ended_with_comments: bool, started_in_comments: bool, ) -> bool { let trimmed = line.trim(); let whole_line_is_comment = || { self.shared .line_comments .iter() .any(|c| trimmed.starts_with(c.as_bytes())) || self .shared .any_multi_line_comments .iter() .any(|(start, end)| { trimmed.starts_with(start.as_bytes()) && trimmed.ends_with(end.as_bytes()) }) }; let starts_with_comment = || { let quote = match self.stack.last() { Some(q) => q, _ => return false, }; self.shared .any_multi_line_comments .iter() .any(|(start, end)| end == quote && trimmed.starts_with(start.as_bytes())) }; // `Some(true)` in order to respect the current configuration. #[allow(clippy::if_same_then_else)] if self.quote.is_some() { if self.quote_is_doc_quote && config.treat_doc_strings_as_comments == Some(true) { self.quote.map_or(false, |q| line.starts_with(q.as_bytes())) || (self.quote.is_some()) } else { false } } else if self .shared .doc_quotes .iter() .any(|(_, e)| line.contains_slice(e.as_bytes())) && started_in_comments { true } else if (whole_line_is_comment)() { true } else if started_in_comments { true } else { (starts_with_comment)() } } #[inline] pub(crate) fn parse_context( &mut self, lines: &[u8], start: usize, end: usize, config: &Config, regex_cache: &RegexCache, ) -> Option { use std::str::FromStr; // static TYPE_REGEX: Lazy = Lazy::new(|| Regex::new(r#"type="(.*)".*>"#).unwrap()); if self.quote.is_some() || !self.stack.is_empty() { return None; } match regex_cache.family()? { RegexFamily::Markdown(md) => { if !lines[start..end].contains_slice(b"```") { return None; } let opening_fence = md.starts_in_range(start, end)?; let start_of_code = opening_fence.end(); let closing_fence = ENDING_MARKDOWN_REGEX.find(&lines[start_of_code..]); if let Some(m) = &closing_fence { trace!("{:?}", String::from_utf8_lossy(m.as_bytes())); } let end_of_code = closing_fence .map_or_else(|| lines.len(), |fence| start_of_code + fence.start()); let end_of_code_block = closing_fence.map_or_else(|| lines.len(), |fence| start_of_code + fence.end()); let balanced = closing_fence.is_some(); let identifier = &opening_fence.as_bytes().trim()[3..]; let language = identifier .split(|&b| b == b',') .find_map(|s| LanguageType::from_str(&String::from_utf8_lossy(s)).ok())?; trace!( "{} BLOCK: {:?}", language, String::from_utf8_lossy(&lines[start_of_code..end_of_code]) ); let stats = language.parse_from_slice(lines[start_of_code..end_of_code].trim(), config); Some(FileContext::new( LanguageContext::Markdown { balanced, language }, end_of_code_block, stats, )) } RegexFamily::Rust => { let rest = &lines[start..]; let comment_syntax = if rest.trim_start().starts_with(b"///") { b"///" } else if rest.trim_start().starts_with(b"//!") { b"//!" } else { return None; }; let mut stepper = LineStep::new(b'\n', start, lines.len()); let mut markdown = Vec::new(); let mut end_of_block = lines.len(); while let Some((start, end)) = stepper.next(lines) { if lines[start..].trim().starts_with(comment_syntax) { trace!("{}", String::from_utf8_lossy(&lines[start..end])); let line = lines[start..end].trim_start(); let stripped_line = &line[3.min(line.len())..]; markdown.extend_from_slice(stripped_line); end_of_block = end; } else { end_of_block = start; break; } } trace!("Markdown found: {:?}", String::from_utf8_lossy(&markdown)); let doc_block = LanguageType::Markdown.parse_from_slice(markdown.trim(), config); Some(FileContext::new( LanguageContext::Rust, end_of_block, doc_block, )) } RegexFamily::LinguaFranca(lf) => { let opening_fence = lf.starts_in_range(start, end)?; let start_of_code = opening_fence.end(); let closing_fence = ENDING_LF_BLOCK_REGEX.find(&lines[start_of_code..]); let end_of_code = closing_fence .map_or_else(|| lines.len(), |fence| start_of_code + fence.start()); let block_contents = &lines[start_of_code..end_of_code]; trace!("LF block: {:?}", String::from_utf8_lossy(block_contents)); let stats = self.get_lf_target_language().parse_from_slice( block_contents.trim_first_and_last_line_of_whitespace(), config, ); trace!("-> stats: {:?}", stats); Some(FileContext::new( LanguageContext::LinguaFranca, end_of_code, stats, )) } RegexFamily::HtmlLike(html) => { if let Some(mut captures) = html.start_script_in_range(start, end) { let start_of_code = captures.next().unwrap().end(); let closing_tag = END_SCRIPT.find(&lines[start_of_code..])?; let end_of_code = start_of_code + closing_tag.start(); let language = captures .next() .and_then(|m| { LanguageType::from_mime(&String::from_utf8_lossy(m.as_bytes().trim())) }) .unwrap_or(LanguageType::JavaScript); let script_contents = &lines[start_of_code..end_of_code]; if script_contents.trim().is_empty() { return None; } let stats = language.parse_from_slice( script_contents.trim_first_and_last_line_of_whitespace(), config, ); Some(FileContext::new( LanguageContext::Html { language }, end_of_code, stats, )) } else if let Some(mut captures) = html.start_style_in_range(start, end) { let start_of_code = captures.next().unwrap().end(); let closing_tag = END_STYLE.find(&lines[start_of_code..])?; let end_of_code = start_of_code + closing_tag.start(); let language = captures .next() .and_then(|m| { LanguageType::from_str( &String::from_utf8_lossy(m.as_bytes().trim()).to_lowercase(), ) .ok() }) .unwrap_or(LanguageType::Css); let style_contents = &lines[start_of_code..end_of_code]; if style_contents.trim().is_empty() { return None; } let stats = language.parse_from_slice( style_contents.trim_first_and_last_line_of_whitespace(), config, ); Some(FileContext::new( LanguageContext::Html { language }, end_of_code, stats, )) } else if let Some(mut captures) = html.start_template_in_range(start, end) { let start_of_code = captures.next().unwrap().end(); let closing_tag = END_TEMPLATE.find(&lines[start_of_code..])?; let end_of_code = start_of_code + closing_tag.start(); let language = captures .next() .and_then(|m| { LanguageType::from_str( &String::from_utf8_lossy(m.as_bytes().trim()).to_lowercase(), ) .ok() }) .unwrap_or(LanguageType::Html); let template_contents = &lines[start_of_code..end_of_code]; if template_contents.trim().is_empty() { return None; } let stats = language.parse_from_slice( template_contents.trim_first_and_last_line_of_whitespace(), config, ); Some(FileContext::new( LanguageContext::Html { language }, end_of_code, stats, )) } else { None } } } } #[inline] pub(crate) fn parse_quote(&mut self, window: &[u8]) -> Option { if !self.stack.is_empty() { return None; } if let Some((start, end)) = self .shared .doc_quotes .iter() .find(|(s, _)| window.starts_with(s.as_bytes())) { trace!("Start Doc {:?}", start); self.quote = Some(end); self.quote_is_verbatim = false; self.quote_is_doc_quote = true; return Some(start.len()); } if let Some((start, end)) = self .shared .verbatim_string_literals .iter() .find(|(s, _)| window.starts_with(s.as_bytes())) { trace!("Start verbatim {:?}", start); self.quote = Some(end); self.quote_is_verbatim = true; self.quote_is_doc_quote = false; return Some(start.len()); } if let Some((start, end)) = self .shared .string_literals .iter() .find(|(s, _)| window.starts_with(s.as_bytes())) { trace!("Start {:?}", start); self.quote = Some(end); self.quote_is_verbatim = false; self.quote_is_doc_quote = false; return Some(start.len()); } None } #[inline] pub(crate) fn parse_end_of_quote(&mut self, window: &[u8]) -> Option { #[allow(clippy::if_same_then_else)] if self._is_string_mode() && window.starts_with(self.quote?.as_bytes()) { let quote = self.quote.take().unwrap(); trace!("End {:?}", quote); Some(quote.len()) } else if !self.quote_is_verbatim && window.starts_with(br"\\") { Some(2) } else if !self.quote_is_verbatim && window.starts_with(br"\") && self .shared .string_literals .iter() .any(|(start, _)| window[1..].starts_with(start.as_bytes())) { // Tell the state machine to skip the next character because it // has been escaped if the string isn't a verbatim string. Some(2) } else { None } } #[inline] pub(crate) fn parse_multi_line_comment(&mut self, window: &[u8]) -> Option { if self.quote.is_some() { return None; } let iter = self .shared .multi_line_comments .iter() .chain(self.shared.nested_comments); for &(start, end) in iter { if window.starts_with(start.as_bytes()) { if self.stack.is_empty() || self.shared.allows_nested || self.shared.nested_comments.contains(&(start, end)) { self.stack.push(end); if log_enabled!(Trace) && self.shared.allows_nested { trace!("Start nested {:?}", start); } else { trace!("Start {:?}", start); } } return Some(start.len()); } } None } #[inline] pub(crate) fn parse_end_of_multi_line(&mut self, window: &[u8]) -> Option { if self .stack .last() .map_or(false, |l| window.starts_with(l.as_bytes())) { let last = self.stack.pop().unwrap(); if log_enabled!(Trace) { if self.stack.is_empty() { trace!("End {:?}", last); } else { trace!("End {:?}. Still in comments.", last); } } Some(last.len()) } else { None } } } ================================================ FILE: src/lib.rs ================================================ //! # Tokei: Count your code quickly. //! //! A simple, efficient library for counting code in directories. This //! functionality is also provided as a //! [CLI utility](//github.com/XAMPPRocky/tokei). Tokei uses a small state //! machine rather than regular expressions found in other code counters. Tokei //! can accurately count a lot more edge cases such as nested comments, or //! comment syntax inside string literals. //! //! # Examples //! //! Gets the total lines of code from all rust files in current directory, //! and all subdirectories. //! //! ```no_run //! use std::collections::BTreeMap; //! use std::fs::File; //! use std::io::Read; //! //! use tokei::{Config, Languages, LanguageType}; //! //! // The paths to search. Accepts absolute, relative, and glob paths. //! let paths = &["src", "tests"]; //! // Exclude any path that contains any of these strings. //! let excluded = &["target"]; //! // `Config` allows you to configure what is searched and counted. //! let config = Config::default(); //! //! let mut languages = Languages::new(); //! languages.get_statistics(paths, excluded, &config); //! let rust = &languages[&LanguageType::Rust]; //! //! println!("Lines of code: {}", rust.code); //! ``` #![deny( trivial_casts, trivial_numeric_casts, unused_variables, unstable_features, unused_import_braces, missing_docs )] #[macro_use] extern crate log; #[macro_use] extern crate serde; #[macro_use] mod utils; mod config; mod consts; mod language; mod sort; mod stats; pub use self::{ config::Config, consts::*, language::{Language, LanguageType, Languages}, sort::Sort, stats::{find_char_boundary, CodeStats, Report}, }; ================================================ FILE: src/main.rs ================================================ #[macro_use] extern crate log; mod cli; mod cli_utils; mod consts; mod input; use std::{error::Error, io, process}; use tokei::{Config, Languages, Sort}; use crate::{ cli::Cli, cli_utils::Printer, consts::{ BLANKS_COLUMN_WIDTH, CODE_COLUMN_WIDTH, COMMENTS_COLUMN_WIDTH, FALLBACK_ROW_LEN, LANGUAGE_COLUMN_WIDTH, LINES_COLUMN_WIDTH, PATH_COLUMN_WIDTH, }, input::add_input, }; fn main() -> Result<(), Box> { let mut cli = Cli::from_args(); if cli.print_languages { Cli::print_supported_languages()?; process::exit(0); } let config = cli.override_config(Config::from_config_files()); let mut languages = Languages::new(); if let Some(input) = cli.file_input() { if !add_input(input, &mut languages) { Cli::print_input_parse_failure(input); process::exit(1); } } let input = cli.input(); for path in &input { if ::std::fs::metadata(path).is_err() { eprintln!("Error: '{}' not found.", path); process::exit(1); } } let columns = cli .columns .or(config.columns) .or_else(|| { if cli.files { term_size::dimensions().map(|(w, _)| w) } else { None } }) .unwrap_or(FALLBACK_ROW_LEN) .max(FALLBACK_ROW_LEN); if cli.streaming == Some(crate::cli::Streaming::Simple) { println!( "#{:^LANGUAGE_COLUMN_WIDTH$} {:^PATH_COLUMN_WIDTH$} {:^LINES_COLUMN_WIDTH$} {:^CODE_COLUMN_WIDTH$} {:^COMMENTS_COLUMN_WIDTH$} {:^BLANKS_COLUMN_WIDTH$}", "language", "path", "lines", "code", "comments", "blanks" ); println!( "{:>LANGUAGE_COLUMN_WIDTH$} {:LINES_COLUMN_WIDTH$} {:>CODE_COLUMN_WIDTH$} {:>COMMENTS_COLUMN_WIDTH$} {:>BLANKS_COLUMN_WIDTH$}", (0..10).map(|_| "#").collect::(), (0..80).map(|_| "#").collect::(), (0..12).map(|_| "#").collect::(), (0..12).map(|_| "#").collect::(), (0..12).map(|_| "#").collect::(), (0..12).map(|_| "#").collect::() ); } languages.get_statistics(&input, &cli.ignored_directories(), &config); if config.for_each_fn.is_some() { process::exit(0); } if let Some(format) = cli.output { print!("{}", format.print(&languages).unwrap()); process::exit(0); } let mut printer = Printer::new( columns, cli.files, io::BufWriter::new(io::stdout()), cli.number_format, ); if languages.iter().any(|(_, lang)| lang.inaccurate) { printer.print_inaccuracy_warning()?; } printer.print_header()?; let mut is_sorted = false; if let Some(sort_category) = cli.sort.or(config.sort) { for (_, ref mut language) in &mut languages { language.sort_by(sort_category); } let mut languages: Vec<_> = languages.iter().collect(); match sort_category { Sort::Blanks => languages.sort_by(|a, b| b.1.blanks.cmp(&a.1.blanks)), Sort::Comments => languages.sort_by(|a, b| b.1.comments.cmp(&a.1.comments)), Sort::Code => languages.sort_by(|a, b| b.1.code.cmp(&a.1.code)), Sort::Files => languages.sort_by(|a, b| b.1.reports.len().cmp(&a.1.reports.len())), Sort::Lines => languages.sort_by(|a, b| b.1.lines().cmp(&a.1.lines())), } is_sorted = true; if cli.sort_reverse { printer.print_results(languages.into_iter().rev(), cli.compact, is_sorted)?; } else { printer.print_results(languages.into_iter(), cli.compact, is_sorted)?; } } else { printer.print_results(languages.iter(), cli.compact, is_sorted)?; } printer.print_total(&languages)?; Ok(()) } ================================================ FILE: src/sort.rs ================================================ use std::{borrow::Cow, str::FromStr}; use serde::de::{self, Deserialize, Deserializer}; /// Used for sorting languages. #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] pub enum Sort { /// Sort by number blank lines. Blanks, /// Sort by number comments lines. Comments, /// Sort by number code lines. Code, /// Sort by number files lines. Files, /// Sort by number of lines. Lines, } impl FromStr for Sort { type Err = String; fn from_str(s: &str) -> Result { Ok(if s.eq_ignore_ascii_case("blanks") { Sort::Blanks } else if s.eq_ignore_ascii_case("comments") { Sort::Comments } else if s.eq_ignore_ascii_case("code") { Sort::Code } else if s.eq_ignore_ascii_case("files") { Sort::Files } else if s.eq_ignore_ascii_case("lines") { Sort::Lines } else { return Err(format!("Unsupported sorting option: {}", s)); }) } } impl<'de> Deserialize<'de> for Sort { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { String::deserialize(deserializer)? .parse() .map_err(de::Error::custom) } } impl<'a> From for Cow<'a, Sort> { fn from(from: Sort) -> Self { Cow::Owned(from) } } impl<'a> From<&'a Sort> for Cow<'a, Sort> { fn from(from: &'a Sort) -> Self { Cow::Borrowed(from) } } ================================================ FILE: src/stats.rs ================================================ use crate::consts::{ BLANKS_COLUMN_WIDTH, CODE_COLUMN_WIDTH, COMMENTS_COLUMN_WIDTH, LINES_COLUMN_WIDTH, }; use crate::LanguageType; use std::{collections::BTreeMap, fmt, ops, path::PathBuf}; /// A struct representing stats about a single blob of code. #[derive(Clone, Debug, Default, PartialEq, serde::Deserialize, serde::Serialize)] #[non_exhaustive] pub struct CodeStats { /// The blank lines in the blob. pub blanks: usize, /// The lines of code in the blob. pub code: usize, /// The lines of comments in the blob. pub comments: usize, /// Language blobs that were contained inside this blob. pub blobs: BTreeMap, } impl CodeStats { /// Creates a new blank `CodeStats`. #[must_use] pub fn new() -> Self { Self::default() } /// Get the total lines in a blob of code. #[must_use] pub fn lines(&self) -> usize { self.blanks + self.code + self.comments } /// Creates a new `CodeStats` from an existing one with all of the child /// blobs merged. #[must_use] pub fn summarise(&self) -> Self { let mut summary = self.clone(); for (_, stats) in std::mem::take(&mut summary.blobs) { let child_summary = stats.summarise(); summary.blanks += child_summary.blanks; summary.comments += child_summary.comments; summary.code += child_summary.code; } summary } } impl ops::AddAssign for CodeStats { fn add_assign(&mut self, rhs: Self) { self.add_assign(&rhs); } } impl ops::AddAssign<&'_ CodeStats> for CodeStats { fn add_assign(&mut self, rhs: &'_ CodeStats) { self.blanks += rhs.blanks; self.code += rhs.code; self.comments += rhs.comments; for (language, stats) in &rhs.blobs { *self.blobs.entry(*language).or_default() += stats; } } } /// A struct representing the statistics of a file. #[derive(Deserialize, Serialize, Clone, Debug, Default, PartialEq)] #[non_exhaustive] pub struct Report { /// The code statistics found in the file. pub stats: CodeStats, /// File name. pub name: PathBuf, } impl Report { /// Create a new `Report` from a [`PathBuf`]. /// /// [`PathBuf`]: //doc.rust-lang.org/std/path/struct.PathBuf.html #[must_use] pub fn new(name: PathBuf) -> Self { Self { name, ..Self::default() } } } impl ops::AddAssign for Report { fn add_assign(&mut self, rhs: CodeStats) { self.stats += rhs; } } #[doc(hidden)] #[must_use] pub fn find_char_boundary(s: &str, index: usize) -> usize { for i in 0..4 { if s.is_char_boundary(index + i) { return index + i; } } unreachable!(); } macro_rules! display_stats { ($f:expr, $this:expr, $name:expr, $max:expr) => { write!( $f, " {: LINES_COLUMN_WIDTH$} {:>CODE_COLUMN_WIDTH$} {:>COMMENTS_COLUMN_WIDTH$} {:>BLANKS_COLUMN_WIDTH$}", $name, $this.stats.lines(), $this.stats.code, $this.stats.comments, $this.stats.blanks, max = $max ) }; } impl fmt::Display for Report { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let name = self.name.to_string_lossy(); let name_length = name.len(); // Added 2 to max length to cover wider Files column (see https://github.com/XAMPPRocky/tokei/issues/891). let max_len = f.width().unwrap_or(27) + 2; if name_length <= max_len { display_stats!(f, self, name, max_len) } else { let mut formatted = String::from("|"); // Add 1 to the index to account for the '|' we add to the output string let from = find_char_boundary(&name, name_length + 1 - max_len); formatted.push_str(&name[from..]); display_stats!(f, self, formatted, max_len) } } } ================================================ FILE: src/utils/ext.rs ================================================ //! Various extensions to Rust std types. pub(crate) trait AsciiExt { fn is_whitespace(&self) -> bool; fn is_line_ending_whitespace(&self) -> bool; } impl AsciiExt for u8 { fn is_whitespace(&self) -> bool { *self == b' ' || (b'\x09'..=b'\x0d').contains(self) } fn is_line_ending_whitespace(&self) -> bool { *self == b'\n' } } pub(crate) trait SliceExt { fn trim_first_and_last_line_of_whitespace(&self) -> &Self; fn trim_start(&self) -> &Self; fn trim(&self) -> &Self; fn contains_slice(&self, needle: &Self) -> bool; } impl SliceExt for [u8] { fn trim_first_and_last_line_of_whitespace(&self) -> &Self { let start = self .iter() .position(|c| c.is_line_ending_whitespace() || !c.is_whitespace()) .map_or(0, |i| (i + 1).min(self.len().saturating_sub(1))); let end = self .iter() .rposition(|c| c.is_line_ending_whitespace() || !c.is_whitespace()) .map_or_else( || self.len().saturating_sub(1), |i| { // Remove the entire `\r\n` in the case that it was the line ending whitespace if self[i.saturating_sub(1)] == b'\r' && self[i] == b'\n' { i - 1 } else { i } }, ); if self[start..].is_empty() { return &[]; } &self[start..=end] } fn trim_start(&self) -> &Self { let length = self.len(); if length == 0 { return self; } let start = match self.iter().position(|c| !c.is_whitespace()) { Some(start) => start, None => return &[], }; &self[start..] } fn trim(&self) -> &Self { let length = self.len(); if length == 0 { return self; } let start = match self.iter().position(|c| !c.is_whitespace()) { Some(start) => start, None => return &[], }; let end = match self.iter().rposition(|c| !c.is_whitespace()) { Some(end) => end.max(start), _ => length, }; &self[start..=end] } fn contains_slice(&self, needle: &Self) -> bool { let self_length = self.len(); let needle_length = needle.len(); if needle_length == 0 || needle_length > self_length { return false; } else if needle_length == self_length { return self == needle; } for window in self.windows(needle_length) { if needle == window { return true; } } false } } #[cfg(test)] mod tests { use super::*; use proptest::prelude::*; #[test] fn is_whitespace() { assert!(b' '.is_whitespace()); assert!(b'\r'.is_whitespace()); assert!(b'\n'.is_whitespace()); } #[test] fn trim() { assert!([b' ', b' ', b' '].trim().is_empty()); assert!([b' ', b'\r', b'\n'].trim().is_empty()); assert!([b'\n'].trim().is_empty()); assert!([].trim().is_empty()); assert_eq!([b'a', b'b'], [b'a', b'b'].trim()); assert_eq!([b'h', b'i'], [b' ', b'h', b'i'].trim()); assert_eq!([b'h', b'i'], [b'h', b'i', b' '].trim()); assert_eq!([b'h', b'i'], [b' ', b'h', b'i', b' '].trim()); } #[test] fn contains() { assert!([1, 2, 3, 4, 5].contains_slice(&[1, 2, 3, 4, 5])); assert!([1, 2, 3, 4, 5].contains_slice(&[1, 2, 3])); assert!([1, 2, 3, 4, 5].contains_slice(&[3, 4, 5])); assert!([1, 2, 3, 4, 5].contains_slice(&[2, 3, 4])); assert!(![1, 2, 3, 4, 5].contains_slice(&[])); } #[test] fn trim_first_and_last_line_of_whitespace_edge_cases() { assert_eq!(b"", b"\ra ".trim_first_and_last_line_of_whitespace()); assert_eq!(b"a", b"\r\na ".trim_first_and_last_line_of_whitespace()); assert_eq!(b" ", b" ".trim_first_and_last_line_of_whitespace()); } proptest! { #[test] fn trim_first_and_last_line_of_whitespace_doesnt_panic(input: Vec) { let _ = &input.trim_first_and_last_line_of_whitespace(); } } } ================================================ FILE: src/utils/fs.rs ================================================ use std::{collections::BTreeMap, path::Path}; use ignore::{overrides::OverrideBuilder, DirEntry, WalkBuilder, WalkState::Continue}; use rayon::prelude::*; use crate::{ config::Config, language::{Language, LanguageType}, }; const IGNORE_FILE: &str = ".tokeignore"; pub fn get_all_files>( paths: &[A], ignored_directories: &[&str], languages: &mut BTreeMap, config: &Config, ) { let languages = parking_lot::Mutex::new(languages); let (tx, rx) = crossbeam_channel::unbounded(); let mut paths = paths.iter(); let mut walker = WalkBuilder::new(paths.next().unwrap()); for path in paths { walker.add(path); } if !ignored_directories.is_empty() { let mut overrides = OverrideBuilder::new("."); for ignored in ignored_directories { rs_error!(overrides.add(&format!("!{}", ignored))); } walker.overrides(overrides.build().expect("Excludes provided were invalid")); } let ignore = config.no_ignore.map(|b| !b).unwrap_or(true); let ignore_dot = ignore && config.no_ignore_dot.map(|b| !b).unwrap_or(true); let ignore_vcs = ignore && config.no_ignore_vcs.map(|b| !b).unwrap_or(true); // Custom ignore files always work even if the `ignore` option is false, // so we only add if that option is not present. if ignore_dot { walker.add_custom_ignore_filename(IGNORE_FILE); } walker .git_exclude(ignore_vcs) .git_global(ignore_vcs) .git_ignore(ignore_vcs) .hidden(config.hidden.map(|b| !b).unwrap_or(true)) .ignore(ignore_dot) .parents(ignore && config.no_ignore_parent.map(|b| !b).unwrap_or(true)); walker.build_parallel().run(move || { let tx = tx.clone(); Box::new(move |entry| { let entry = match entry { Ok(entry) => entry, Err(error) => { use ignore::Error; if let Error::WithDepth { err: ref error, .. } = error { if let Error::WithPath { ref path, err: ref error, } = **error { error!("{} reading {}", error, path.display()); return Continue; } } error!("{}", error); return Continue; } }; if entry.file_type().map_or(false, |ft| ft.is_file()) { tx.send(entry).unwrap(); } Continue }) }); let rx_iter = rx .into_iter() .par_bridge() .filter_map(|e| LanguageType::from_path(e.path(), config).map(|l| (e, l))); let process = |(entry, language): (DirEntry, LanguageType)| { let result = language.parse(entry.into_path(), config); let mut lock = languages.lock(); let entry = lock.entry(language).or_insert_with(Language::new); match result { Ok(stats) => { let func = config.for_each_fn; if let Some(f) = func { f(language, stats.clone()) }; entry.add_report(stats) } Err((error, path)) => { entry.mark_inaccurate(); error!("Error reading {}:\n{}", path.display(), error); } } }; if let Some(types) = config.types.as_deref() { rx_iter.filter(|(_, l)| types.contains(l)).for_each(process) } else { rx_iter.for_each(process) } } pub(crate) fn get_extension(path: &Path) -> Option { path.extension().map(|e| e.to_string_lossy().to_lowercase()) } pub(crate) fn get_filename(path: &Path) -> Option { path.file_name().map(|e| e.to_string_lossy().to_lowercase()) } #[cfg(test)] mod tests { use std::fs; use tempfile::TempDir; use super::IGNORE_FILE; use crate::{ config::Config, language::{languages::Languages, LanguageType}, }; const FILE_CONTENTS: &[u8] = b"fn main() {}"; const FILE_NAME: &str = "main.rs"; const IGNORE_PATTERN: &str = "*.rs"; const LANGUAGE: &LanguageType = &LanguageType::Rust; #[test] fn ignore_directory_with_extension() { let mut languages = Languages::new(); let tmp_dir = TempDir::new().expect("Couldn't create temp dir"); let path_name = tmp_dir.path().join("directory.rs"); fs::create_dir(path_name).expect("Couldn't create directory.rs within temp"); super::get_all_files( &[tmp_dir.into_path().to_str().unwrap()], &[], &mut languages, &Config::default(), ); assert!(languages.get(LANGUAGE).is_none()); } #[test] fn hidden() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); fs::write(dir.path().join(".hidden.rs"), FILE_CONTENTS).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); config.hidden = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } #[test] fn no_ignore_implies_dot() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); fs::write(dir.path().join(".ignore"), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); config.no_ignore = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } #[test] fn no_ignore_implies_vcs_gitignore() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); git2::Repository::init(dir.path()).expect("Couldn't create git repo."); fs::write(dir.path().join(".gitignore"), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); config.no_ignore = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } #[test] fn no_ignore_parent() { let parent_dir = TempDir::new().expect("Couldn't create temp dir."); let child_dir = parent_dir.path().join("child/"); let mut config = Config::default(); let mut languages = Languages::new(); fs::create_dir_all(&child_dir) .unwrap_or_else(|_| panic!("Couldn't create {:?}", child_dir)); fs::write(parent_dir.path().join(".ignore"), IGNORE_PATTERN) .expect("Couldn't create .gitignore."); fs::write(child_dir.join(FILE_NAME), FILE_CONTENTS).expect("Couldn't create child.rs"); super::get_all_files( &[child_dir.as_path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); config.no_ignore_parent = Some(true); super::get_all_files( &[child_dir.as_path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } #[test] fn no_ignore_dot() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); fs::write(dir.path().join(".ignore"), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); config.no_ignore_dot = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } #[test] fn no_ignore_dot_still_vcs_gitignore() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); git2::Repository::init(dir.path()).expect("Couldn't create git repo."); fs::write(dir.path().join(".gitignore"), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); config.no_ignore_dot = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); } #[test] fn no_ignore_dot_includes_custom_ignore() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); fs::write(dir.path().join(IGNORE_FILE), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); config.no_ignore_dot = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } #[test] fn no_ignore_vcs_gitignore() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); git2::Repository::init(dir.path()).expect("Couldn't create git repo."); fs::write(dir.path().join(".gitignore"), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); config.no_ignore_vcs = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } #[test] fn no_ignore_vcs_gitignore_still_dot() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); fs::write(dir.path().join(".ignore"), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); config.no_ignore_vcs = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); } #[test] fn no_ignore_vcs_gitexclude() { let dir = TempDir::new().expect("Couldn't create temp dir."); let mut config = Config::default(); let mut languages = Languages::new(); git2::Repository::init(dir.path()).expect("Couldn't create git repo."); fs::write(dir.path().join(".git/info/exclude"), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); config.no_ignore_vcs = Some(true); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } #[test] fn custom_ignore() { let dir = TempDir::new().expect("Couldn't create temp dir."); let config = Config::default(); let mut languages = Languages::new(); git2::Repository::init(dir.path()).expect("Couldn't create git repo."); fs::write(dir.path().join(IGNORE_FILE), IGNORE_PATTERN).unwrap(); fs::write(dir.path().join(FILE_NAME), FILE_CONTENTS).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_none()); fs::remove_file(dir.path().join(IGNORE_FILE)).unwrap(); super::get_all_files( &[dir.path().to_str().unwrap()], &[], &mut languages, &config, ); assert!(languages.get(LANGUAGE).is_some()); } } ================================================ FILE: src/utils/macros.rs ================================================ #![allow(unused_macros)] macro_rules! opt_warn { ($option:expr, $message:expr) => { match $option { Some(result) => result, None => { warn!($message); continue; } } }; } macro_rules! rs_warn { ($result:expr, $message: expr) => { match $result { Ok(result) => result, Err(error) => { warn!("{}", error); continue; } } }; } macro_rules! opt_error { ($option:expr, $message:expr) => { match $option { Some(result) => result, None => { error!($message); continue; } } }; } macro_rules! rs_error { ($result:expr) => { match $result { Ok(result) => result, Err(error) => { error!("{}", error); continue; } } }; } macro_rules! opt_ret_warn { ($option:expr, $message:expr) => { match $option { Some(result) => result, None => { warn!($message); return None; } } }; } macro_rules! rs_ret_warn { ($result:expr, $message: expr) => { match $result { Ok(result) => result, Err(error) => { warn!("{}", error); return None; } } }; } macro_rules! opt_ret_error { ($option:expr, $message:expr) => { match $option { Some(result) => result, None => { error!($message); return None; } } }; } macro_rules! rs_ret_error { ($result:expr) => { match $result { Ok(result) => result, Err(error) => { error!("{}", error); return None; } } }; } macro_rules! debug { ($fmt:expr) => (if cfg!(debug_assertions) {println!($fmt)}); ($fmt:expr, $($arg:tt)*) => (if cfg!(debug_assertions) {println!($fmt, $($arg)*)}); } ================================================ FILE: src/utils/mod.rs ================================================ #[macro_use] mod macros; pub(crate) mod ext; pub mod fs; ================================================ FILE: tests/accuracy.rs ================================================ extern crate ignore; extern crate regex; extern crate tokei; use std::fs; use once_cell::sync::Lazy; use regex::Regex; use tokei::{Config, Languages}; static LINES: Lazy = Lazy::new(|| Regex::new(r"\d+ lines").unwrap()); static CODE: Lazy = Lazy::new(|| Regex::new(r"\d+ code").unwrap()); static COMMENTS: Lazy = Lazy::new(|| Regex::new(r"\d+ comments").unwrap()); static BLANKS: Lazy = Lazy::new(|| Regex::new(r"\d+ blanks").unwrap()); macro_rules! get_digit { ($regex:expr, $text:expr) => {{ let matched = $regex.find(&$text).expect("Couldn't find category"); matched .as_str() .split_whitespace() .next() .unwrap() .parse::() .unwrap() }}; } mod config { use tokei::*; /* #[test] fn extension_change() { use std::collections::HashMap; let mut languages = Languages::new(); let config = Config { languages: { let mut map = HashMap::new(); let mut config = LanguageConfig::new(); config.extensions(vec![String::from("cpp")]); map.insert(LanguageType::C, config); Some(map) }, ..Config::default() }; languages.get_statistics(&["tests/data/cpp.cpp"], &[], &config); if languages.len() != 1 { panic!("wrong languages detected: expected just C, found {:?}", languages.into_iter().collect::>()); } let (name, _) = languages.into_iter().next().unwrap(); assert_eq!(LanguageType::C, name); } */ #[test] fn treating_comments_as_code() { let mut languages = Languages::new(); let config = Config { treat_doc_strings_as_comments: Some(true), ..Config::default() }; languages.get_statistics(&["tests/data/python.py"], &[], &config); if languages.len() != 1 { panic!( "wrong languages detected: expected just Python, found {:?}", languages.into_iter().collect::>() ); } let (_, language) = languages.into_iter().next().unwrap(); assert_eq!(language.lines(), 15); assert_eq!(language.blanks, 3); assert_eq!(language.comments, 7); assert_eq!(language.code, 5); } } include!(concat!(env!("OUT_DIR"), "/tests.rs")); ================================================ FILE: tests/data/Daml.daml ================================================ -- ! 42 lines 24 code 9 comments 9 blanks {- This code is derived from https://github.com/digital-asset/ex-secure-daml-infra/blob/master/daml/BobTrigger.daml -- Haskell/DAML support nested comments {- including nested block comments -} -} module BobTrigger where -- import DA.Action import DA.Foldable import DA.Next.Map (Map) import Daml.Trigger import Main (--$) :: a -> b -> String a --$ b = "Not a comment" rejectTrigger : Trigger () = Trigger with initialize = \_ -> () updateState = \_ _ () -> () rule = rejectRule registeredTemplates = AllInDar heartbeat = None testNotAComment = 1 --$ 2 == "Not a comment" rejectRule : Party -> ACS -> Time -> Map CommandId [Command] -> () -> TriggerA () rejectRule p acs _ _ _ = do let assets = getContracts @Asset acs let bobAssets = filter (\(_,a) -> a.owner == p) assets let configs = getContracts @DonorConfig acs let Some (_,bobConfig) = find (\(_,c) -> c.owner == p) configs forA_ bobAssets $ \(_cid, c) -> do debug "Ran rejectRule" emitCommands [exerciseCmd _cid Give with newOwner = bobConfig.donateTo] [toAnyContractId _cid] ================================================ FILE: tests/data/Dockerfile ================================================ # 17 lines 7 code 3 comments 7 blanks FROM netbsd:7.0.2 MAINTAINER Somebody version: 2.2 RUN curl -sSf https://static.rust-lang.org/rustup.sh | sh -s -- -y # this part is important VOLUME ["/project"] WORKDIR "/project" RUN sh -c 'echo "Hello World" > /dev/null' RUN cargo install tokei # not counted # now you do your part ================================================ FILE: tests/data/MSBuild.csproj ================================================ Exe netcoreapp2.0 Latest ================================================ FILE: tests/data/Makefile ================================================ # 24 lines 11 code 5 comments 8 blanks ## ## ## IMPORTANT COMMENT ## ## ## all: hello hello: main.o factorial.o hello.o g++ main.o factorial.o hello.o -o hello # main.o is my favorite main.o: main.cpp g++ -c main.cpp factorial.o: factorial.cpp g++ -c factorial.cpp hello.o: hello.cpp g++ -c hello.cpp clean: rm *o hello #not counted ================================================ FILE: tests/data/Modelica.mo ================================================ // 21 lines 13 code 5 comments 3 blanks block Add "Output the sum of the two inputs" extends Interfaces.SI2SO; /* parameter section */ parameter Real k1=+1 "Gain of input signal 1"; parameter Real k2=+1 "Gain of input signal 2"; // equation section equation y = k1*u1 + k2*u2; annotation ( Documentation(info="

Some documentation.

")); end Add; ================================================ FILE: tests/data/NuGet.Config ================================================ ================================================ FILE: tests/data/PKGBUILD ================================================ # 24 lines 19 code 3 comments 2 blanks # Maintainer: Andy 'Blocktronics' Herbert # Aur Maintainer: Wanesty pkgname=moebius-bin pkgver=1.0.29 pkgrel=1 epoch=1 pkgdesc="Modern ANSI & ASCII Art Editor" arch=('x86_64') url="https://github.com/blocktronics/moebius" license=('Apache') depends=('gtk3' 'libnotify' 'libxss' 'libxtst' 'xdg-utils' 'libappindicator-gtk3') makedepends=('libarchive') conflicts=('moebius') source=("https://github.com/blocktronics/moebius/releases/download/$pkgver/Moebius.rpm") sha256sums=(69aaa1e42e287ed78c8e73971dae3df23ae4fa00e3416ea0fc262b7d147fefec) noextract=("Moebius.rpm") package() { bsdtar -C "${pkgdir}" -xvf "$srcdir/Moebius.rpm" mkdir "$pkgdir/usr/bin" ln -s "/opt/Moebius/moebius" "$pkgdir/usr/bin/moebius" } ================================================ FILE: tests/data/Rakefile ================================================ # 10 lines 4 code 2 comments 4 blanks # this is a rakefile task default: %w[test] task :test do # not counted ruby "test/unittest.rb" end ================================================ FILE: tests/data/SConstruct ================================================ #!python # 10 lines 3 code 3 comments 4 blanks # this is a comment Program('cpp.cpp') # this is a line-ending comment env = Environment(CCFLAGS='-O3') env.Append(CCFLAGS='-O3') ================================================ FILE: tests/data/Snakefile ================================================ # 67 lines 50 code 4 comments 13 blanks """ A sample Snakefile for testing line counting """ SAMPLES = ["A", "B"] # This is a # multiline # comment rule all: input: "plots/quals.svg" '''Sometimes even some comments in single quote fences.''' rule bwa_map: input: "data/genome.fa", # Inline comments are also supported "data/samples/{sample}.fastq" output: "mapped_reads/{sample}.bam" shell: "bwa mem {input} | samtools view -Sb - > {output}" rule samtools_sort: input: "mapped_reads/{sample}.bam" output: "sorted_reads/{sample}.bam" shell: "samtools sort -T sorted_reads/{wildcards.sample} " "-O bam {input} > {output}" rule samtools_index: input: "sorted_reads/{sample}.bam" output: "sorted_reads/{sample}.bam.bai" shell: "samtools index {input}" rule bcftools_call: input: fa="data/genome.fa", bam=expand("sorted_reads/{sample}.bam", sample=SAMPLES), bai=expand("sorted_reads/{sample}.bam.bai", sample=SAMPLES) output: "calls/all.vcf" shell: "bcftools mpileup -f {input.fa} {input.bam} | " "bcftools call -mv - > {output}" rule plot_quals: input: "calls/all.vcf" output: "plots/quals.svg" script: "scripts/plot-quals.py" ================================================ FILE: tests/data/Tera.tera ================================================ {# 42 lines 26 code 11 comments 5 blanks #} link {# #} {# comment #} body {# multi-line-comment #} {% if error %}
{{ error.content }}
{# comment #} {% endif %} ================================================ FILE: tests/data/abnf.abnf ================================================ ; 11 lines 3 code 5 comments 3 blanks ; comment line 0 ; comment line 1 ALPHA = %x41-5A / %x61-7A ; A-Z / a-z BIT = "0" / "1" CHAR = %x01-7F ; any 7-bit US-ASCII character, ; excluding NUL ================================================ FILE: tests/data/alloy.als ================================================ // 18 lines 10 code 3 comments 5 blanks sig Node { edge: set Node } ------------------------------------------------------------------------ pred self_loop[n: Node] { n in n.edge } pred all_self_loop { all n: Node | self_loop[n] } /* Comments started by /* don't nest */ run all_self_loop ================================================ FILE: tests/data/apl.apl ================================================ ⍝ 10 lines 3 code 3 comments 4 blanks 256=2*8 ⍝ Comment ⊃¨ ' '(≠⊆⊢) 'A Programming Language' ⍝ A magic square of length ⊢ MS ← (⍳-∘⌈÷∘2)(⊣⊖⌽),⍨⍴∘⍳×⍨ ================================================ FILE: tests/data/arduino.ino ================================================ // 23 lines 13 code 6 comments 4 blanks int led = 13; void setup() { // setup println() Serial.begin(155200); // Init LED pin pinMode(led, OUTPUT); } /** * Blink the LED */ void loop() { Serial.println("LED ON!"); digitalWrite(led, HIGH); delay(1000); Serial.println("LED OFF!"); digitalWrite(led, LOW); delay(1000); } ================================================ FILE: tests/data/arturo.art ================================================ ; 8 lines 3 code 3 comments 2 blanks ; this is a comment ; this is another comment a1: 2 a2: 3.14 ; pi a3: 213213 ; another number ================================================ FILE: tests/data/asciidoc.adoc ================================================ // 20 lines 5 code 8 comments 7 blanks = AsciiDoc title A simple paragraph. // single line comment That should end before a paragraph. //// multi = Line comment //// == Nested title Nested titles and paragraphs are fine, too. ================================================ FILE: tests/data/asn1.asn1 ================================================ -- 34 lines 16 code 11 comments 7 blanks PKCS-12 { iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) pkcs-12(12) modules(0) pkcs-12(1) } -- PKCS #12 v1.1 ASN.1 Module -- Revised October 27, 2012 -- This module has been checked for conformance with the ASN.1 standard -- by the OSS ASN.1 Tools DEFINITIONS IMPLICIT TAGS ::= BEGIN PFX ::= SEQUENCE { version INTEGER {v3(3)}(v3,...), authSafe OCTET STRING, macData MacData /* " " */ OPTIONAL } /* * Multi line * */ MacData ::= SEQUENCE { mac OBJECT IDENTIFIER, macSalt OCTET STRING, iterations INTEGER DEFAULT 1 -- Note: The default is for historical reasons and its use is -- deprecated. } END ================================================ FILE: tests/data/ats.dats ================================================ //! 42 lines 25 code 9 comments 8 blanks (************* Reference: https://github.com/ats-lang/ats-lang.github.io/blob/master/DOCUMENT/ATS2TUTORIAL/CODE/chap_stream_vt.dats **************) #include "share/atspre_staload.hats" /* Lazy-evaluated integer iterator */ fun from (n: int): stream_vt(int) = $ldelay(stream_vt_cons(n, from(n + 1))) // Lazy-evaluated prime finder fun sieve (ns: stream_vt(int)) : stream_vt(int) = $ldelay( let val ns_con = !ns val- @stream_vt_cons(n0, ns1) = ns_con val n0_val = n0 val ns1_val = ns1 val () = (ns1 := sieve(stream_vt_filter_cloptr(ns1_val, lam x => x mod n0_val > 0))) prval () = fold@(ns_con) in ns_con end , ~ns ) // Test run for finding the 1000-th prime number val thePrimes = sieve(from(2)) val p1000 = (steam_vt_drop_exn(thePrimes, 1000)).head() val () = println!("p1000 = ", p1000) implement main0 () = {} (* End of file *) ================================================ FILE: tests/data/awk.awk ================================================ #!/bin/awk -f # 5 lines 1 code 3 comments 1 blanks # This is a comment { print $0 } ================================================ FILE: tests/data/ballerina.bal ================================================ // 40 lines 29 code 6 comments 5 blanks # Returns Bob's response to someone talking to him. # # + input - whatever is said to Bob # + return - Bob's response public function hey(string input) returns string { string trimmed = input.trim(); boolean silent = isSilence(trimmed); boolean asking = isQuestion(trimmed); boolean yelling = isYelling(trimmed); match [silent, yelling, asking] { [true, _, _] => { return "Fine. Be that way!"; } [_, true, true] => { return "Calm down, I know what I'm doing!"; } [_, true, false] => { return "Whoa, chill out!"; } [_, false, true] => { return "Sure."; } _ => { return "Whatever."; } } } isolated function isSilence(string input) returns boolean => input.length() == 0; isolated function isQuestion(string input) returns boolean => input.endsWith("?"); function isYelling(string input) returns boolean { // contains an uppercase letter and does not contain a lowercase letter return input.includesMatch(re `\p{Lu}`) && !input.includesMatch(re `\p{Ll}`); } ================================================ FILE: tests/data/bazel.bzl ================================================ # 18 lines 13 code 3 comments 2 blanks # build hello-greet cc_library( name = "hello-greet", srcs = ["hello-greet.cc"], hdrs = ["hello-greet.h"], ) # build hello-world cc_binary( name = "hello-world", srcs = ["hello-world.cc"], deps = [ ":hello-greet", "//lib:hello-time", ], ) ================================================ FILE: tests/data/bean.bean ================================================ ; 27 lines 13 code 6 comments 8 blanks option "operating_currency" "EUR" 2002-01-01 commodity EUR name: "Euro" asset-class: "cash" ; open accounts initially 2020-09-01 open Equity:Opening-Balances 2020-09-01 open Assets:Cash EUR 2020-09-01 open Expenses:Food EUR ; put initial money on account 2020-09-01 pad Assets:Cash Equity:Opening-Balances ; verifying starting balance 2020-09-02 balance Assets:Cash 81.7 EUR ; transferring money 2020-09-03 * "transfer of money" Assets:Cash -17.7 EUR Expenses:Food ; validating changed balance 2020-09-04 balance Assets:Cash 64 EUR ================================================ FILE: tests/data/bicep.bicep ================================================ //! 50 lines 35 code 8 comments 7 blanks /* Bicep is a declarative language, which means the elements can appear in any order. Unlike imperative languages, the order of elements doesn't affect how deployment is processed. This means you can define resources, variables, and parameters in any order you like. */ metadata description = 'Creates a storage account and a web app' @description('The prefix to use for the storage account name.') @minLength(3) @maxLength(11) param storagePrefix string param storageSKU string = 'Standard_LRS' param location string = resourceGroup().location var uniqueStorageName = '${storagePrefix}${uniqueString(resourceGroup().id)}' var objectExmaple = { name: 'John' age: 30 address: ''' 1 Microsoft Way Redmond, WA 98052 ''' } // Create a storage account resource stg 'Microsoft.Storage/storageAccounts@2022-09-01' = { name: uniqueStorageName location: location sku: { name: storageSKU } kind: 'StorageV2' properties: { supportsHttpsTrafficOnly: true } } // Use a module to deploy a web app // Modules are a way to encapsulate and reuse resources in Bicep module webModule './webApp.bicep' = { name: 'webDeploy' params: { skuName: 'S1' location: location personalInfo: objectExmaple } } ================================================ FILE: tests/data/bitbake.bb ================================================ # 23 lines 13 code 5 comments 5 blanks # # This file was derived from the 'Hello World!' example recipe in the # Yocto Project Development Manual. # SUMMARY = "Simple helloworld application" SECTION = "examples" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" SRC_URI = "file://helloworld.c" S = "${WORKDIR}" do_compile() { ${CC} helloworld.c -o helloworld } do_install() { install -d ${D}${bindir} install -m 0755 helloworld ${D}${bindir} } ================================================ FILE: tests/data/bqn.bqn ================================================ # 10 lines 3 code 3 comments 4 blanks 256=2⋆8 # Comment <⟜'a'⊸/"Big Questions Notation" # A magic square of length ⊢ MS ← (↕-⌈∘÷⟜2)(⊣⌽˘⌾⍉⌽˘)⋈˜⥊⟜↕ט ================================================ FILE: tests/data/brightscript.brs ================================================ ' 26 lines 10 code 13 comments 3 blanks ' /** ' * @member difference ' * @memberof module:rodash ' * @instance ' * @description Return a new array of items from the first which are not in the second. ' * @param {Array} first ' * @param {Array} second ' * @example REM * difference = _.difference([1,2], [2]) REM * ' => [1] REM * REM */ Function rodash_difference_(first, second) result = [] for each f in first result.push(f) 'Push array for each s in second if m.equal(s,f) then result.pop() end for end for return result End Function ================================================ FILE: tests/data/c.c ================================================ // 50 lines 33 code 8 comments 9 blanks /* /* we can't nest block comments in c, but we can start one */ int main(void) { char *start = "/*"; int x = 1; x += 2; // end of line comment */ } void foo() { char *esc = "\"/*escaped quotes in a string and block comment*/\""; func1(); func2(); char *next_line = "*/ /*string on new line\ continued to another line\ bar();\ */"; char *next_line2 = "line1\ // not a real comment\ line3*/"; /* Block comment // line comment in a block comment end block comment*/ char *late_start = // " "wow\ that's pretty neat"; char *late_start2 = /* " */ "*/ still just a string"; // but this is a line comment } void foobar() { int a = 4; // /* int b = 5; int c = 6; // */ } /*\ / comment \*/ struct Point { int x; int y; int z; }; ================================================ FILE: tests/data/c3.c3 ================================================ /* 8 lines 6 code 1 comments 1 blanks */ fn int main() { int x = 5; /* Multline line comment */ io::printn("Hello, world!"); int y = 4; // Single line comment. return 0; } ================================================ FILE: tests/data/cairo.cairo ================================================ //! 51 lines 32 code 13 comments 6 blanks //! ```rust //! fn main () { //! // Comment //! //! println!("Hello World!"); //! } //! ``` /// The main function fn main() { let x: ByteArray = "\"/*##\"\"##\'\'"; // comment loop { if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */ break; } } } fn foo>(name: T) { let this_ends = 'a "\'test/"*.'; call1(); call2(); let this_does_not = // a // nested // comment " // ///"*/another /*test call3(); //*/"; } fn call1() {} fn call2() {} fn call3() {} fn foobar() { let does_not_start: ByteArray = // " "until here, test/* test"; // a quote: " let also_doesnt_start = /// " */ 'until here, test,'; // another quote: " } fn foo2() { let a = 4; // /// let b = '5'; let c = 6; // /// } ================================================ FILE: tests/data/cangjie.cj ================================================ /* 50 lines 34 code 8 comments 8 blanks */ /* /* we can nest block*/ comments in cangjie */ main(): Unit { let start: String = "/*" var x: Int64 = 1 x += 2 // end of line comment */ } func foo(): Unit { let esc: String = "\"/*escaped quotes in a string and block comment*/\""; let next_line: String = """ */ /*string on new line continued to another line bar(); */" """ /* Block comment // line comment in a block comment end block comment*/ let late_start: String = // " ##"wow\ that's pretty neat"## let late_start2: String = /* " */ "*/ still just a string" // but this is a line comment } func foobar(): Unit { let a: Int64 = 4 // /* let b: Int64 = 5 let c: Int64 = 6 // */ } /* \ / comment \*/ struct Point { let x: Int64 let y: Int64 let z: Int64 public init(x: Int64, y: Int64, z: Int64) { this.x = x this.y = y this.z = z } } ================================================ FILE: tests/data/chapel.chpl ================================================ // chapel 24 lines 9 code 9 comments 6 blanks // Tidy line comment /* Tidy block comment. */ // Cheeky line comments /* // */ /* Cheeky // block comments */ // Calculate a factorial proc factorial(n: int): int { var x = 1; // this will eventually be returned for i in 1..n { x *= i; } return x; } writeln("// this isn't a comment"); writeln('/* this is also not a comment */'); ================================================ FILE: tests/data/cil.cil ================================================ ; 20 lines 15 code 3 comments 2 blanks ;============= etcd_t ============== (allow etcd_t proc_sysctl_t (dir (search))) (allow etcd_t proc_sysctl_t (file (read open))) (allow etcd_t procfs_t (dir (search getattr))) (allow etcd_t procfs_t (lnk_file (read))) (allow etcd_t self (dir (read open search))) (allow etcd_t self (fifo_file (write read))) ;============= kernel_t ============== (allow kernel_t bin_t (dir (search))) (allow kernel_t bin_t (file (read execute_no_trans open map execute))) (allow kernel_t debugfs_t (dir (search))) (allow kernel_t device_t (blk_file (create setattr))) (allow kernel_t device_t (chr_file (write create setattr))) (allow kernel_t self (capability (dac_override mknod))) (allow kernel_t self (dir (write add_name search))) (allow kernel_t self (file (write create open))) (filecon "/.extra(/.*)?" any (system_u object_r extra_t (systemLow systemLow))) ================================================ FILE: tests/data/circom.circom ================================================ // 34 lines 23 code 7 comments 4 blanks pragma circom 2.0.8; /* * Sum an array of non-zero values. */ function sum(values, size) { var sum = 0; for (var i = 0; i < size; i++) { assert(values[i] != 0); sum += values[i]; } log("sum = ", sum); return sum; } /* * Ensure x is a solution to x^5 - 2x^4 + 5x - 4 = 0. */ template Polynomial() { signal input x; signal x2; signal x4; signal x5; signal output y; x2 <== x * x; x4 <== x2 * x2; x5 <== x4 * x; y <== x5 - 2 * x4 + 5 * x - 4; // y = x^5 - 2 * x^4 + 5x - 4. y === 0; // Ensure that y = 0. } component main = Polynomial(); ================================================ FILE: tests/data/clojure.clj ================================================ ; 19 lines 13 code 3 comments 3 blanks (ns clojure) ; Below is a function (defn a-fn "Docstring with a column ;" [a b] (+ 1 1)) (defn a-fn2 ;"Not a doc" "Doc doc again" [a b] ; a and b right? (let [multiline "I'm a multline ; string "] (str multline a b))) ================================================ FILE: tests/data/clojurec.cljc ================================================ ; 19 lines 13 code 3 comments 3 blanks (ns clojure) ; Below is a function (defn a-fn "Docstring with a column ;" [a b] (+ 1 1)) (defn a-fn2 ;"Not a doc" "Doc doc again" [a b] ; a and b right? (let [multiline "I'm a multline ; string "] (str multline a b))) ================================================ FILE: tests/data/clojurescript.cljs ================================================ ; 19 lines 13 code 3 comments 3 blanks (ns clojure) ; Below is a function (defn a-fn "Docstring with a column ;" [a b] (+ 1 1)) (defn a-fn2 ;"Not a doc" "Doc doc again" [a b] ; a and b right? (let [multiline "I'm a multline ; string "] (str multline a b))) ================================================ FILE: tests/data/cmake.cmake ================================================ # 25 lines 16 code 3 comments 6 blanks SET(_POSSIBLE_XYZ_INCLUDE include include/xyz) SET(_POSSIBLE_XYZ_EXECUTABLE xyz) SET(_POSSIBLE_XYZ_LIBRARY XYZ) # this is a comment IF(XYZ_FIND_VERSION_MAJOR AND XYZ_FIND_VERSION_MINOR) SET(_POSSIBLE_SUFFIXES "${XYZ_FIND_VERSION_MAJOR}${XYZ_FIND_VERSION_MINOR}" "${XYZ_FIND_VERSION_MAJOR}.${XYZ_FIND_VERSION_MINOR}" "-${XYZ_FIND_VERSION_MAJOR}.${XYZ_FIND_VERSION_MINOR}") # not counted ELSE(XYZ_FIND_VERSION_MAJOR AND XYZ_FIND_VERSION_MINOR) SET(_POSSIBLE_SUFFIXES "67" "92" "352.9" "0.0.8z") ENDIF(XYZ_FIND_VERSION_MAJOR AND XYZ_FIND_VERSION_MINOR) FOREACH(_SUFFIX ${_POSSIBLE_SUFFIXES}) LIST(APPEND _POSSIBLE_XYZ_INCLUDE "include/XYZ${_SUFFIX}") LIST(APPEND _POSSIBLE_XYZ_EXECUTABLE "XYZ${_SUFFIX}") LIST(APPEND _POSSIBLE_XYZ_LIBRARY "XYZ${_SUFFIX}") ENDFOREACH(_SUFFIX) # not counted FIND_PROGRAM(XYZ_EXECUTABLE NAMES ${_POSSIBLE_XYZ_EXECUTABLE} ) # this is also a comment ================================================ FILE: tests/data/codeql.ql ================================================ //! 40 lines 17 code 15 comments 8 blanks /** * @name fu * @description bar * * Rerum similique consequatur non dolor sit. Autem doloribus sed in sint * ratione sit voluptates at. Nihil ut fugiat ab ut aliquid consequatur sunt * ullam. Adipisci voluptatem hic dicta. */ // asdf import cpp private import test.foo.bar.baz /** * Another comment. */ class C extends Expr { C () { // single comment not this.test() and not this.what() } private predicate what() { /* TODO */ this.isAbstract() } predicate test() { this = "what" } } from Function f where f.getName() = "function" and /* inline comment */ f.getArgument(0).asExpr() instanceof FooBar select f, "function" ================================================ FILE: tests/data/cogent.cogent ================================================ -- 7 lines 2 code 2 comments 3 blanks type A -- uncounted comment -- comment type B ================================================ FILE: tests/data/cpp.cpp ================================================ /* 46 lines 37 code 3 comments 6 blanks */ #include // bubble_sort_function void bubble_sort(int a[10], int n) { int t; int j = n; int s = 1; while (s > 0) { s = 0; int i = 1; while (i < j) { if (a[i] < a[i - 1]) { t = a[i]; a[i] = a[i - 1]; a[i - 1] = t; s = 1; } i++; } j--; } } int main() { int a[] = {4, 65, 2, -31, 0, 99, 2, 83, 782, 1}; int n = 10; int i = 0; printf(R"(Before sorting:\n\n" )"); // Single line comment while (i < n) { printf("%d ", a[i]); i++; } bubble_sort(a, n); printf("\n\nAfter sorting:\n\n"); i = 0; while (i < n) { printf("%d ", a[i]); i++; } } ================================================ FILE: tests/data/cppm.cppm ================================================ /* 46 lines 37 code 3 comments 6 blanks */ #include // bubble_sort_function void bubble_sort(int a[10], int n) { int t; int j = n; int s = 1; while (s > 0) { s = 0; int i = 1; while (i < j) { if (a[i] < a[i - 1]) { t = a[i]; a[i] = a[i - 1]; a[i - 1] = t; s = 1; } i++; } j--; } } int main() { int a[] = {4, 65, 2, -31, 0, 99, 2, 83, 782, 1}; int n = 10; int i = 0; printf(R"(Before sorting:\n\n" )"); // Single line comment while (i < n) { printf("%d ", a[i]); i++; } bubble_sort(a, n); printf("\n\nAfter sorting:\n\n"); i = 0; while (i < n) { printf("%d ", a[i]); i++; } } ================================================ FILE: tests/data/crystal.cr ================================================ # 20 lines 14 code 2 comments 4 blanks x = 3 if x < 2 p = "Smaller" else p = "Bigger" end multiline_string = "first line second line" heredoc = <<-SOME hello SOME # testing. while x > 2 && x < 10 x += 1 end ================================================ FILE: tests/data/csharp.cs ================================================ // 26 lines 14 code 9 comments 3 blanks namespace Ns { /* multi-line comment */ public class Cls { private const string BasePath = @"a:\"; [Fact] public void MyTest() { // Arrange. Foo(); // Act. Bar(); // Assert. Baz(); } } } ================================================ FILE: tests/data/cuda.cu ================================================ /* 7 lines 4 code 2 comments 1 blanks */ // add vector __host__ void add(const int* a, const int* b, int* c) { int i = threadIdx.x; c[i] = a[i] + b[i]; } ================================================ FILE: tests/data/cue.cue ================================================ // 12 lines 8 code 2 comments 2 blanks // A documentation comment map: { normal: "normal string" // inline comment (not counted) content: """ Multi-line string """ raw: #"A newline is \#n written as "\n"."# byte: '\U0001F604' } ================================================ FILE: tests/data/cython.pyx ================================================ # 29 lines, 21 code, 3 comments, 5 blanks def add(x, y): ''' Hello World # Real Second line Second line ''' string = "Hello World #\ " y += len(string) # Add the two numbers. x + y cdef add2(x, y): """ Hello World # Real Second line Second line Note that docstring lines are counted as code """ string = "Hello World" y += len(string) # Add the two numbers. x + y ================================================ FILE: tests/data/d.d ================================================ /* 8 lines 5 code 1 comments 2 blanks */ void main() { auto x = 5; /+ a /+ nested +/ comment /* +/ writefln("hello"); auto y = 4; // */ } ================================================ FILE: tests/data/d2.d2 ================================================ # 15 lines 4 code 6 comments 5 blanks # Comments start with a hash character and continue until the next newline or EOF. x -> y x -> y # I am at the end '#x' -> "#y" """ This is a block comment """ y -> z ================================================ FILE: tests/data/dhall.dhall ================================================ -- 16 lines 9 code 5 comments 2 blanks {- A comment within the interior of a multi-line literal counts as part of the literal -} '' -- Hello {- world -} '' { some = "thing" , keys = ["can" , "have", -- wait for it "lists"] } ================================================ FILE: tests/data/dreammaker.dm ================================================ // 17 lines 7 code 6 comments 4 blanks /* * /* Hello! */ */ /mob // I can rely on this file to exist on disk. var/some_file = './/dreammaker.dm' /mob/Login() // Not counted. /* */ world << "// Say hello to [src]!" src << browse({" /*Link*/ "}, "window=google") ================================================ FILE: tests/data/dust.dust ================================================ {! 10 lines 2 code 5 comments 3 blanks !} {! All Dust comments are multiline comments. And there's no quoting comment openers and closers. Instead there are escape sequences for producing literal brackets in template output: {~lb} outputs a left-bracket.!}

Hello world!

{! More comments !}

~{lb}Goodbye, world~{rb}

================================================ FILE: tests/data/ebuild.ebuild ================================================ # 16 lines 9 code 2 comments 5 blanks # test comment EAPI=8 DESCRIPTION="ebuild file" HOMEPAGE="https://foo.example.org/" SRC_URI="ftp://foo.example.org/${P}.tar.gz" LICENSE="MIT" SLOT="0" src_compile() { : } ================================================ FILE: tests/data/edgeql.edgeql ================================================ # 28 lines 21 code 3 comments 4 blanks select User { name, friends: { name }, has_i := .friends.name ilike '%i%', has_o := .friends.name ilike '%o%', } filter .has_i or .has_o; select {} ?? User {name}; # update the user with the name 'Alice Smith' with module example update User filter .name = 'Alice Smith' set { name := 'Alice J. Smith' }; # update all users whose name is 'Bob' with module example update User filter .name like 'Bob%' set { name := User.name ++ '*' }; ================================================ FILE: tests/data/edn.edn ================================================ ; 11 lines 6 code 2 comments 3 blanks ; Just some random data {:a [ 1] :b 1 ; this doesn't count as a comment :c {1 1 2 2} :d [1 2 3]} ================================================ FILE: tests/data/eight.8th ================================================ \ 22 lines 9 code 8 comments 5 blanks (* multiline comments (* a nested comment *) * *) -- here's a single line comment "Hello, " var, foo -- line ending comment \ here's another single line comment "!" var, bar \ a different line ending comment : hello \ s -- foo @ s:<+ bar @ s:+ . cr ; "World" hello bye ================================================ FILE: tests/data/elvish.elv ================================================ # 16 lines, 9 code, 5 blanks, 2 comments echo "This is a multiline string # with a hash in it." echo 'This is a single-quoted string.' # This is a comment. use re edit:after-readline = [ [line]{ print "\e]2;"$line"\a" > /dev/tty } ] ================================================ FILE: tests/data/emacs_dev_env.ede ================================================ ;; 16 lines 6 code 7 comments 3 blanks ;; This is an EDE Project file ;; Object ede-proj-project ;; EDE project file. (ede-proj-project "ede-proj-project" :name "my-proj" :version "1.0.0" :file "Project.ede" :targets (list )) ;; Local Variables: ;; mode: emacs-lisp ;; End: ================================================ FILE: tests/data/emacs_lisp.el ================================================ ;; 21 lines 11 code 6 comments 4 blanks ; This is a comment line ;; This too! ;;; This 3! ;;;; This 4! (setq some-global-var nil) ;Comment ;;;###autoload (defun some-fn () "Some function." (interactive) (message "I am some function")) (defun fundamental-mode () "Major mode not specialized for anything in particular. Other major modes are defined by comparison with this one." (interactive) (kill-all-local-variables) (run-mode-hooks)) ================================================ FILE: tests/data/emojicode.🍇 ================================================ 💭 24 lines 10 code 10 comments 4 blanks 📘 This package is neat. 📘 💭🔜 Comment! 💭🔜 nested? 🔚💭 📗 Simple docstring with a quote 🔤 📗 🐇👨‍🚀🍇 💭 More quotes! 🔤 🐇❗👋 name🔡 🍇 😀 🍪 🔤Hello there, 🔤 name 🔤! ❌🔤 💭 no comment here🔤 🍪❗ 🍉 🍉 🏁 🍇 👋🐇👨‍🚀 🔤Karen🔤❗ 🍉 ================================================ FILE: tests/data/esdl.esdl ================================================ # 20 lines 13 code 4 comments 3 blanks # no module block type default::Movie { required property title -> str; # the year of release property year -> int64; required link director -> default::Person; required multi link actors -> default::Person; } type default::Person { required property first_name -> str; required property last_name -> str; } abstract link friends_base { # declare a specific title for the link annotation title := 'Close contacts'; } ================================================ FILE: tests/data/example.umpl ================================================ ! 68 lines 58 code 2 comments 8 blanks create yum with 5 create num with ((plus 7 6 6 yum))> (num)> ((plus 1 num))> ! prints 17, the greater than signifies that this should be the printed, the bang symbol signifies a comment, numbers must be in hexadecimal but will be printed in decimal potato 😂3 ⧼ ! to create a function we use the potato keyword, functions must be nameed with a single emoji, we put the number of args and then ⧼⧽ is where the code goes loop ⧼ (` `)> if {$3} ⧼ break ⧽ else ⧼ continue ⧽ ⧽ potato 😂 1 ⧼ ((minus 1 10))> ! prints -15 ⧽ create final with ((add $1 $2))< ! ! we use the create keyword to create a variable and the with keyword to assign a value to the variable, to refrence the args from a function use $1 for the first arg and so on ! adds both arg1 and arg2, the less than signifies that this should not be the printed and sets it to the final variable return: ! we use (:) colon whe returning nothing ⧽ ((new 😂 10 1 0x1A))> ! we use the new keyword to call a function followed by function names, prints 32, when the first digit of number is a letre prefix with 0x list it with [((not true))> 0] ! creates a list with the values 1,2 (in hex), list can only have two elements and you cant declare recursive lists like: [0 [1 1]], but if you have an identiifer (d) that points to a list you can do: [9 d] ((set-with it.first 2)) ! prints false ((set-with other-bool true))< ! we set other-bool to true if {other-bool} ⧼ ! the {} must evalte to a boolean, we can also use eq, ne lt, le etc (`hello`)> ! the bactick (`) is used to encapsulate a string ⧽ else ⧼ (`by`)> ⧽ if {true} ⧼ ! there is now else, use nested if elses, condotins must be in {} and we can use nested () within the ((minus 1 10))> ! prints -15 ⧽ else ⧼ ((plus 1 10))> ! prints 17 ⧽ loop ⧼ potato 😂 1 ⧼ ((minus 1 10))> ! prints -15 ⧽ create inputs with ((input `continue?`))< if {((eq inputs `y`))<} ⧼ ! there is now else, use nested if elses, condotins must be in {} and we can use nested () within the potato 😂 1 ⧼ ((minus 1 10))> ! prints -15 ⧽ continue ⧽ else ⧼ potato 😂 1 ⧼ ((minus 1 10))> ! prints -15 ⧽ break ! what the if/esle do can be nothing if you dont pu anything in the parenthesis ⧽ ((inputs `continue?`))< ⧽ ((new 😂 10 190 0x1A))> ! we use the new keyword to call a function followed by function names, prints 32, when the first digit of number is a letre prefix with 0x ((set-with bool (input `ads`)))> create inputs with ((input `continue?`))< ================================================ FILE: tests/data/factor.factor ================================================ ! 14 lines, 5 code, 6 comments, 3 blanks /* we can use some dependencies */ USING: math multiline sequences ; ! this is a vocabulary IN: my-vocab ! this comment describes this function : add ( x y -- z ) "Hello World !\ " length /* Add the three numbers. */ + + ; ================================================ FILE: tests/data/fennel.fnl ================================================ ;; 18 lines 8 code 5 comments 5 blanks ; this is a ; single comment ;;;; this is also a single comment ;;;;;; ; "this is a comment too!" (local variable "I ;am a ;variable!") ; (print "\"I am commented out!\"") (print "\"Hello world!\"") ; this is an ; end of line comment (print "This is not a comment: ;") (print "This is a multiline string") (fn somefn [x] (print "I am some function.") (print "My parameter is " (string.format "\"%s\"" x))) ================================================ FILE: tests/data/flatbuffers.fbs ================================================ // 34 lines 21 code 6 comments 7 blanks include "another_schema.fbs"; namespace Example; // one line comment enum PhoneType: byte { MOBILE, HOME, WORK } /* block comment another line end */ table PhoneNumber { number: string; type: PhoneType; } /// documentation comment table Person { name: string; id: int32; email: string; phones: [PhoneNumber]; // a stray quote " } table AddressBook { people: /* a block comment inside code */ [Person]; } root_type AddressBook; /* block /* comments cannot be nested (except the start comment) */ ================================================ FILE: tests/data/forgecfg.cfg ================================================ # 79 lines 20 code 40 comments 19 blanks # Configuration file ~CONFIG_VERSION: 0.4.0 ########################################################################################################## # advanced #--------------------------------------------------------------------------------------------------------# # Advanced config options to change the way JEI functions. ########################################################################################################## advanced { # Move the JEI search bar to the bottom center of the screen. [default: false] B:centerSearchBarEnabled=true B:debugModeEnabled=false # Choose if JEI should give ingredients direct to the inventory (inventory) or pick them up with the mouse (mouse_pickup). # [Default: mouse_pickup] # [Valid: [inventory, mouse_pickup]] S:giveMode=inventory # The maximum width of the ingredient list. [range: 4 ~ 100, default: 100] I:maxColumns=100 # The maximum height of the recipe GUI. [range: 175 ~ 5000, default: 350] I:maxRecipeGuiHeight=350 # How the mod name should be formatted in the tooltip for JEI GUIs. Leave blank to disable. [Default: blue italic] [Valid: [black, dark_blue, dark_green, dark_aqua, dark_red, dark_purple, gold, gray, dark_gray, blue, green, aqua, red, light_purple, yellow, white, obfuscated, bold, strikethrough, underline, italic]] S:modNameFormat=blue italic # Enable JEI memory usage optimizations. [default: true] B:optimizeMemoryUsage=true } ########################################################################################################## # search #--------------------------------------------------------------------------------------------------------# # Options relating to the search bar. ########################################################################################################## search { # Search mode for Colors (prefix: ^) # [Default: disabled] # [Valid: [enabled, require_prefix, disabled]] S:colorSearchMode=DISABLED # Search mode for Creative Tab Names (prefix: %) # [Default: disabled] # [Valid: [enabled, require_prefix, disabled]] S:creativeTabSearchMode=DISABLED # Search mode for Mod Names (prefix: @) # [Default: require_prefix] # [Valid: [enabled, require_prefix, disabled]] S:modNameSearchMode=require_prefix # Search mode for Ore Dictionary Names (prefix: $) # [Default: disabled] # [Valid: [enabled, require_prefix, disabled]] S:oreDictSearchMode=require_prefix # Search mode for resources ids (prefix: &) # [Default: disabled] # [Valid: [enabled, require_prefix, disabled]] S:resourceIdSearchMode=enabled # Search mode for Tooltips (prefix: #) # [Default: enabled] # [Valid: [enabled, require_prefix, disabled]] S:tooltipSearchMode=enabled } searchadvancedtooltips { # config.jei.searchAdvancedTooltips.search.comment [default: false] B:search=false } ================================================ FILE: tests/data/fsharp.fs ================================================ (* 15 lines 6 code 5 comments 4 blanks *) // Comment let foo = (* Comment *) 5 let bar = "(* Code *)" let baz = @"a:\" // Comment ================================================ FILE: tests/data/fstar.fst ================================================ (* 11 lines 3 code 5 comments 3 blanks *) module Hello (* multi line comment *) open FStar.IO // uncounted comment // single line comment let main = print_string "Hello, F*!\n" (* uncounted comment *) ================================================ FILE: tests/data/ftl.ftl ================================================ <#-- 10 lines 5 code 3 comments 2 blanks --> <#ftl output_format="plainText"/> <#-- Define the print macro --> <#macro print text> ${text} <#-- Print "Hello world" --> <@print "Hello world"/> ================================================ FILE: tests/data/futhark.fut ================================================ -- 8 lines 2 code 3 comments 3 blanks -- this is a test file -- add two let f (x: i32) = x + 2 let main = f -- eta expand ================================================ FILE: tests/data/gas.S ================================================ // 67 lines 46 code 10 comments 11 blanks #include "config.h" /* BIG FAT WARNING GOES HERE */ #define dbg(char) \ mov \char, %ax ;\ out %ax, $0xe9 ;\ #endif .align 16 .code16 .section .resettext, "xa", @progbits reset_vector: cli jmp switch_modes .section .text switch_modes: /* * The developer manual builds up the GDT, but since it should get * cached by the CPU, we can just have it in the flash. */ mov %cs, %ax mov %ax, %ds /* Enable protected mode (PE) */ mov %cr0, %eax or $1, %al mov %eax, %cr0 ljmpl $0x8,$protected_mode .code32 protected_mode: mov $0x10, %ax mov %ax, %ds mov %ax, %es mov %ax, %fs mov %ax, %gs mov %ax, %ss gdt_desc: .word egdt - gdt - 1 .long gdt .align 8 gdt: // 0 segment .long 0, 0 // code (0x8) .word 0xffff // limit 15:0 .word 0x0 // base 15:0 .byte 0x0 // base 23:16 .byte 0x9b // present, ring 0, executable, !conforming, readable, accessed .byte 0xcf // granularity size, limit[19:16] = f .byte 0x0 // base 31:24 // data (0x10) .word 0xffff // limit 15:0 .word 0x0 // base 15:0 .byte 0x0 // base 23:16 .byte 0x93 // present, priv=0, !executable, stack down, writable, accessed .byte 0xcf // granularity=1 size=1, limit 19:16 = f .byte 0x0 // base 31:24 egdt: ================================================ FILE: tests/data/gdb.gdb ================================================ # 15 lines 7 code 5 comments 3 blanks # # This is a comment line. We don't have multi-comment lines # macro define offsetof(_type, _memb) ((long)(&((_type *)0)->_memb)) break foo continue # Let's have something print when a breakpoint is hit. commands 2 p i continue end ================================================ FILE: tests/data/gdshader.gdshader ================================================ // 24 lines 6 code 13 comments 5 blanks /* test test test */ /* /** */ void fragment() { // there is not any string in gdshader int x, b, c; float p, q, r; if(b == c) { } /** //sdsadasdasdasd /* **/ /* // odpwopdw // */ } ================================================ FILE: tests/data/gherkin.feature ================================================ # 13 lines 8 code 3 comments 2 blanks Feature: Guess the word # The first example has two steps Scenario: Maker starts a game When the Maker starts a game Then the Maker waits for a Breaker to join # The second example has three steps Scenario: Breaker joins a game Given the Maker has started a game with the word "silky" When the Breaker joins the Maker's game Then the Breaker must guess a word with 5 characters ================================================ FILE: tests/data/gleam.gleam ================================================ //// 34 lines 24 code 4 comments 6 blanks import gleam/option.{Option, None} import gleam/io pub type LoadedBool { Yup AlsoYup } pub external type Person pub opaque type Cat { Cat( name: String, age: Int, is_cute: LoadedBool, owner: Some(Person), ) } pub fn main() { let jane = // Here is a quote " new_kitten(called: "Jane") let kira = new_kitten(called: "Kira") io.println("Two kitties!") } /// A new baby kitten /// fn new_kitten(called name: String) -> Cat { // No owner yet! Cat(name: name, age: 0, is_cute: Yup, owner: None) } ================================================ FILE: tests/data/glimmer_js.gjs ================================================ // 27 lines, 18 code, 6 comments, 3 blanks import { helper } from '@ember/component/helper'; import { modifier } from 'ember-modifier'; // A single-line comment const plusOne = helper(([num]) => num + 1); /** * A multi-line comment */ const setScrollPosition = modifier((element, [position]) => { element.scrollTop = position }); ================================================ FILE: tests/data/glimmer_ts.gts ================================================ // 18 lines, 10 code, 6 comments, 2 blanks import type { TemplateOnlyComponent } from '@glimmer/component'; // A single-line comment const localVariable = 'foo'; /** * A multi-line comment */ const Greet: TemplateOnlyComponent<{ name: string }> = ================================================ FILE: tests/data/gml.gml ================================================ /* 17 lines 5 code 9 comments 3 blanks */ /* here's a comment */ /* this is also ...a comment! */ var a = 0; // @function b(c) // hi! function b(c) { d = 5; // how are you? } e = "good how r u"; ================================================ FILE: tests/data/go.go ================================================ // 37 lines 24 code 6 comments 7 blanks // Package main is a test file. package main import ( "errors" ) /* /**/ func main() { start := "/*" for { if len(start) >= 2 && start[1] == '*' && start[0] == '/' { // found the */ break } } if err := Foo(42, start); err != nil { panic(err) } } // Foo is a function. /* nested comment */ func Foo( // first a int, s string, /* second */ ) (err error) { m := `a multiline string` return errors.New(m) } // end of file ================================================ FILE: tests/data/gohtml.gohtml ================================================ {{ .title }}

{{ .title }}

{{ .text }}

{{ template "footer" . }} {{/* GoHTML multi line comment */}} ================================================ FILE: tests/data/graphql.gql ================================================ # 89 lines 71 code 3 comments 15 blanks """ A simple GraphQL schema which is well described. This is not a comment. See: https://facebook.github.io/graphql/June2018/#sec-Descriptions """ type Query { """ Translates a string from a given language into a different language. """ translate( "The original language that `text` is provided in." fromLanguage: Language "The translated language to be returned." toLanguage: Language "The text to be translated." text: String ): String } """ The set of languages supported by `translate`. """ enum Language { "English" EN "French" FR "Chinese" CH } # Comment the query and use "quotes" inside the comment query withFragments($expandedInfo: Boolean) { user(id: "3bd5a1cbed10e") { id # Insignificant comment ... @include(if: $expandedInfo) { firstName lastName birthday } friends(first: 10) { ...friendFields } profiles( handles: [ "zuck", "cocacola", "#hashed#hash#inside" ] ) { handle ... on User { friends { count } } ... on Page { likers { count } } } } } fragment friendFields on User { id firstName profilePic(size: 50) } # A simple GraphQL type definition type User { id: ID firstName: String lastName: String birthday: Date } ================================================ FILE: tests/data/gwion.gw ================================================ #! 10 lines 8 code 1 comments 1 blanks class C { var int i; var float f; var Object o; operator void @dtor () { <<< "dtor" >>>; } } var C c; <<< c >>>; ================================================ FILE: tests/data/haml.haml ================================================ -# 18 lines 11 code 2 comments 5 blanks %section.container - @posts.each do |post| -# Ruby comment %h1= post.title %h2= post.subtitle .content = post.content / HTML comment. Not detected as of now. %div %span This is all wrapped in a comment ================================================ FILE: tests/data/hcl.tf ================================================ # 22 lines 11 code 7 comments 4 blanks variable "foo" "bar" { default = "yes" } /* We like multiple lines */ locals { // this this = "that" # list more = ["foo", "bar"] /* map */ map = { yep = "nope" # bad comment nope = "yep" } } ================================================ FILE: tests/data/headache.ha ================================================ // 13 lines 9 code 3 comments 1 blanks /* //The Headache cat program */ // Implementation of Unix Cat in Headache void main() { char x; read x; //read from stdin while(x){ @x; //print char x read x; //read again from stdin if(x - (255 as char)){} else x = 0; //necessary for terminal emulation } } ================================================ FILE: tests/data/hex0.hex0 ================================================ # 154 lines 89 code 34 comments 31 blanks # SPDX-FileCopyrightText: 2017 Jeremiah Orians # SPDX-FileCopyrightText: 2023 Andrius Štikonas # # SPDX-License-Identifier: GPL-3.0-or-later ## ELF Header #:ELF_base 7F 45 4C 46 ## e_ident[EI_MAG0-3] ELF's magic number 02 ## e_ident[EI_CLASS] Indicating 64 bit 01 ## e_ident[EI_DATA] Indicating little endianness 01 ## e_ident[EI_VERSION] Indicating original elf 03 ## e_ident[EI_OSABI] Set at 3 because FreeBSD is strict 00 ## e_ident[EI_ABIVERSION] Set at 0 because none cares 00 00 00 00 00 00 00 ## e_ident[EI_PAD] 02 00 ## e_type Indicating Executable 3E 00 ## e_machine Indicating AMD64 01 00 00 00 ## e_version Indicating original elf 78 00 60 00 00 00 00 00 ## e_entry Address of the entry point (Number of bytes this header is + Base Address) 40 00 00 00 00 00 00 00 ## e_phoff Address of program header table 00 00 00 00 00 00 00 00 ## e_shoff Address of section header table 00 00 00 00 ## e_flags 40 00 ## e_ehsize Indicating our 64 Byte header 38 00 ## e_phentsize size of a program header table 01 00 ## e_phnum number of entries in program table 00 00 ## e_shentsize size of a section header table 00 00 ## e_shnum number of entries in section table 00 00 ## e_shstrndx index of the section names ## Program Header #:ELF_program_headers 01 00 00 00 ## p_type 01 00 00 00 ## p_flags: PF-X = 1 00 00 00 00 00 00 00 00 ## p_offset 00 00 60 00 00 00 00 00 ## p_vaddr 00 00 60 00 00 00 00 00 ## p_physaddr E5 00 00 00 00 00 00 00 ## p_filesz E5 00 00 00 00 00 00 00 ## p_memsz 01 00 00 00 00 00 00 00 ## Required alignment #:ELF_text # Where the ELF Header is going to hit # Simply jump to _start # Our main function #:_start (0x600078) 58 ; pop_rax # Get the number of arguments 5F ; pop_rdi # Get the program name 5F ; pop_rdi # Get the actual input name 31F6 ; xor_esi,esi # prepare read_only, rsi = 0 6A 02 ; push !2 # prepare syscall number 58 ; pop_rax # the syscall number for open() 99 ; cdq # Extra sure, rdx = 0 0F05 ; syscall # Now open that damn file 5F ; pop_rdi # Get the actual output name 50 ; push_rax # Preserve the file pointer we were given 66BE 4102 ; mov_si, @577 # Prepare file as O_WRONLY|O_CREAT|O_TRUNC 66BA C001 ; mov_dx, @448 # Prepare file as RWX for owner only (700 in octal) 6A 02 ; push !2 # prepare syscall number 58 ; pop_rax # the syscall number for open() 0F05 ; syscall # Now open that damn file 93 ; xchg_ebx,eax # Preserve the file pointer we were given 99 ; cdq # rdx = 0 since file descriptor is nonnegative FFC2 ; inc_edx # rdx = 1 (count for read/write) #:loop_reset_all (0x600096) 31ED ; xor_ebp,ebp # ebp = 0 (no prior hex val) # Comment tracking is done with ecx. # ecx is decremented if we hit a # comment (';' or '#') and reset # if we hit a new-line. #:loop_reset_comment (0x600098) 52 ; push_rdx 59 ; pop_rcx # Set no current comment #:loop_add_comment (0x60009A) FFC9 ; dec_ecx #:loop (0x60009C) # Read a byte 5F ; pop_rdi # Get infile 54 ; push_rsp 5E ; pop_rsi # Set buffer # rdx is already set to 1. 31C0 ; xor_eax,eax # Set read syscall in rax 51 ; push_rcx # Save comment tracking 0F05 ; syscall # Do the actual read 59 ; pop_rcx # Restore comment tracking 57 ; push_rdi # Re-save infile 85C0 ; test_eax,eax # Check what we got 75 06 ; jne !cont # No EOF # Exit successfully B0 3C ; mov_al, !60 # Set exit syscall in rax 31FF ; xor_edi,edi # Set return success (rdi = 0) 0F05 ; syscall # Exit #:cont (0x6000B0) 8A06 ; mov_al,[rsi] # Move prog byte in eax # New line check 3C 0A ; cmp_al, !10 # Check new-line 74 E2 ; je !loop_reset_comment # If new-line, end comment handling # In comment check 85C9 ; test_ecx,ecx # Skip byte if we are in a comment 75 E2 ; jne !loop # Start comment check 3C 23 ; cmp_al, !35 # Start of '#' comment 74 DC ; je !loop_add_comment 3C 3B ; cmp_al, !59 # Start of ';' comment 74 D8 ; je !loop_add_comment # Start of hex str to int 2C 30 ; sub_al, !48 # Subtract ascii '0' from al 2C 0A ; sub_al, !10 # Check for value in '0'-'9' 72 08 ; jb !write # We have hex value, write it 2C 07 ; sub_al, !7 # Subtract ('A'-'0') from al 24 DF ; and_al, !0xDF # Remove lower case bit 3C 07 ; cmp_al, !7 # Check for value 'A'-'F' 73 CC ; jae !loop # We have hex value, write it #:write (0x6000D0) C1E5 04 ; shl_ebp, !4 # Shift up existing hex digit 04 0A ; add_al, !10 # Finish converting ascii to raw value 01C5 ; add_ebp,eax # Combine the hex digits # Check if this is first digit in hex val F7DB ; neg_ebx # Flip sign of r10 to indicate we got a digit 7C C1 ; jl !loop # Negative -> first digit, get another one # We have both digits in low byte of ebp, good to write 892E ; mov_[rsi],ebp # Move edge to buffer 89DF ; mov_edi,ebx # Move outfile to rdi B0 01 ; mov_al, !1 # Set write syscall in rax 0F05 ; syscall # Do the write EB B1 ; jmp !loop_reset_all # Start a fresh byte #:ELF_end (0x6000E5) ================================================ FILE: tests/data/hex1.hex1 ================================================ # 589 lines 387 code 91 comments 111 blanks # SPDX-FileCopyrightText: 2016 Jeremiah Orians # SPDX-FileCopyrightText: 2017 Jan Nieuwenhuizen # # SPDX-License-Identifier: GPL-3.0-or-later ## ELF Header # :ELF_base 7F 45 4C 46 ## e_ident[EI_MAG0-3] ELF's magic number 02 ## e_ident[EI_CLASS] Indicating 64 bit 01 ## e_ident[EI_DATA] Indicating little endianness 01 ## e_ident[EI_VERSION] Indicating original elf 03 ## e_ident[EI_OSABI] Set at 3 because FreeBSD is strict 00 ## e_ident[EI_ABIVERSION] Set at 0 because none cares 00 00 00 00 00 00 00 ## e_ident[EI_PAD] 02 00 ## e_type Indicating Executable 3E 00 ## e_machine Indicating AMD64 01 00 00 00 ## e_version Indicating original elf 78 00 60 00 00 00 00 00 ## e_entry Address of the entry point (Number of bytes this header is + Base Address) 40 00 00 00 00 00 00 00 ## e_phoff Address of program header table 00 00 00 00 00 00 00 00 ## e_shoff Address of section header table 00 00 00 00 ## e_flags 40 00 ## e_ehsize Indicating our 64 Byte header 38 00 ## e_phentsize size of a program header table 01 00 ## e_phnum number of entries in program table 00 00 ## e_shentsize size of a section header table 00 00 ## e_shnum number of entries in section table 00 00 ## e_shstrndx index of the section names ## Program Header # :ELF_program_headers 01 00 00 00 ## p_type 07 00 00 00 ## ph_flags: PF-X|PF-W|PF-R = 7 00 00 00 00 00 00 00 00 ## p_offset 00 00 60 00 00 00 00 00 ## p_vaddr 00 00 60 00 00 00 00 00 ## p_physaddr EF 05 00 00 00 00 00 00 ## p_filesz EF 05 00 00 00 00 00 00 ## p_memsz 01 00 00 00 00 00 00 00 ## Required alignment # :ELF_text # Where the ELF Header is going to hit # Simply jump to _start # Our main function # Register usage: # RAX, RDX, RSI, RDI => Temps # R15 => Flag # R14 => High bits # R13 => IP # R12 => MALLOC # R11 => HEAD # Struct format: (size 24) # NEXT => 0 # TARGET => 8 # NAME => 16 # :_start 48C7C7 00000000 ; mov_rdi, %0 # Get current pointer E8 %w ; call %malloc # Get current HEAP 4889C7 ; mov_rdi,rax # Using current 4989C4 ; mov_r12,rax # Setup MALLOC 4881C7 00008000 ; add_rdi, %8388608 # Create space for temp [8MB] E8 %w ; call %malloc # Give ourselves 8192000 bytes to work with 4C8925 %T ; mov_[rip+DWORD],r12 %scratch # Allocate space for scratch area 4981C4 00080000 ; add_r12, %0x800 # 2 KiB of scratch 58 ; pop_rax # Get the number of arguments 5F ; pop_rdi # Get the program name 5F ; pop_rdi # Get the actual input name 48C7C6 00000000 ; mov_rsi, %0 # prepare read_only 48C7C0 02000000 ; mov_rax, %2 # the syscall number for open() 0F05 ; syscall # Now open that damn file 4989C1 ; mov_r9,rax # Preserve the file pointer we were given 5F ; pop_rdi # Get the actual output name 48C7C6 41020000 ; mov_rsi, %577 # Prepare file as O_WRONLY|O_CREAT|O_TRUNC 48C7C2 C0010000 ; mov_rdx, %448 # Prepare file as RWX for owner only (700 in octal) 48C7C0 02000000 ; mov_rax, %2 # the syscall number for open() 0F05 ; syscall # Now open that damn file 4883F8 00 ; cmp_rax, !0 # Check for missing output 0F8F %R ; jg %_start_out # Have real input 48C7C0 01000000 ; mov_rax, %1 # Use stdout :R # :_start_out 4989C2 ; mov_r10,rax # Preserve the file pointer we were given E8 %H ; call %ClearScratch # Zero scratch 49C7C7 FFFFFFFF ; mov_r15, %-1 # Our flag for byte processing 49C7C6 00000000 ; mov_r14, %0 # temp storage for the sum 49C7C5 00006000 ; mov_r13, %0x00600000 # Our starting IP 49C7C3 00000000 ; mov_r11, %0 # HEAD = NULL E8 %a ; call %First_pass # Process it # rewind input file 4C89CF ; mov_rdi,r9 # Using our input file 48C7C6 00000000 ; mov_rsi, %0 # Offset Zero 48C7C2 00000000 ; mov_rdx, %0 # Whence Zero 48C7C0 08000000 ; mov_rax, %8 # lseek 4153 ; push_r11 # Protect HEAD 0F05 ; syscall 415B ; pop_r11 # Restore HEAD 49C7C7 FFFFFFFF ; mov_r15, %-1 # Our flag for byte processing 49C7C6 00000000 ; mov_r14, %0 # temp storage for the sum 49C7C5 00006000 ; mov_r13, %0x00600000 # Our starting IP E8 %k ; call %Second_pass # Process it E9 %v ; jmp %Done :a # :First_pass E8 %x ; call %Read_byte # Deal with EOF 4883F8 FC ; cmp_rax, !-4 0F84 %i ; je %First_pass_done # Check for : 4883F8 3A ; cmp_rax, !0x3A 0F85 %b ; jne %First_pass_0 # Deal with label E9 %C ; jmp %StoreLabel :b # :First_pass_0 # Check for ! 4883F8 21 ; cmp_rax, !0x21 0F84 %h ; je %First_pass_pointer # Check for @ 4883F8 40 ; cmp_rax, !0x40 0F84 %h ; je %First_pass_pointer # Check for $ 4883F8 24 ; cmp_rax, !0x24 0F84 %h ; je %First_pass_pointer # Check for % 4883F8 25 ; cmp_rax, !0x25 0F84 %h ; je %First_pass_pointer # Check for & 4883F8 26 ; cmp_rax, !0x26 0F84 %h ; je %First_pass_pointer # Deal with everything else E8 %j ; call %hex # Process our char # Deal with EOF 4883F8 FC ; cmp_rax, !-4 0F84 %i ; je %First_pass_done # deal with -1 values 4883F8 00 ; cmp_rax, !0 0F8C %a ; jl %First_pass # deal with toggle 4983FF 00 ; cmp_r15, !0 0F84 %c ; je %First_pass_1 4983C5 01 ; add_r13, !1 # Increment IP :c # :First_pass_1 49F7D7 ; not_r15 E9 %a ; jmp %First_pass :d # :Update_Pointer # Check for ! 4883F8 21 ; cmp_rax, !0x21 0F84 %g ; je %Update_Pointer_1 # Check for @ 4883F8 40 ; cmp_rax, !0x40 0F84 %f ; je %Update_Pointer_2 # Check for $ 4883F8 24 ; cmp_rax, !0x24 0F84 %f ; je %Update_Pointer_2 # Check for % 4883F8 25 ; cmp_rax, !0x25 0F84 %e ; je %Update_Pointer_4 # Check for & 4883F8 26 ; cmp_rax, !0x26 0F84 %e ; je %Update_Pointer_4 # deal with bad input E8 %Q # call %fail :e # :Update_Pointer_4 4983C5 02 ; add_r13, !2 # Increment IP :f # :Update_Pointer_2 4983C5 01 ; add_r13, !1 # Increment IP :g # :Update_Pointer_1 4983C5 01 ; add_r13, !1 # Increment IP C3 ; ret :h # :First_pass_pointer # Deal with Pointer to label E8 %d ; call %Update_Pointer # Increment IP 488B1D %T ; mov_rbx,[rip+DWORD] %scratch # Using scratch E8 %A ; call %consume_token # Read token E8 %H ; call %ClearScratch # Throw away token 4883F8 3E ; cmp_rax, !0x3E # check for '>' 0F85 %a ; jne %First_pass # Loop again # Deal with %label>label case 488B1D %T ; mov_rbx,[rip+DWORD] %scratch # Write to scratch E8 %A ; call %consume_token # get token E8 %H ; call %ClearScratch # Clean up after ourselves E9 %a ; jmp %First_pass # Loop again :i # :First_pass_done C3 ; ret :j # :hex # deal with EOF 4883F8 FC ; cmp_rax, !-4 0F84 %n ; je %EOF # deal with line comments starting with # 4883F8 23 ; cmp_rax, !0x23 0F84 %s ; je %ascii_comment # deal with line comments starting with ; 4883F8 3B ; cmp_rax, !0x3B 0F84 %s ; je %ascii_comment # deal all ascii less than 0 4883F8 30 ; cmp_rax, !0x30 0F8C %r ; jl %ascii_other # deal with 0-9 4883F8 3A ; cmp_rax, !0x3A 0F8C %o ; jl %ascii_num # deal with all ascii less than A 4883F8 41 ; cmp_rax, !0x41 0F8C %r ; jl %ascii_other # deal with A-F 4883F8 47 ; cmp_rax, !0x47 0F8C %q ; jl %ascii_high # deal with all ascii less than a 4883F8 61 ; cmp_rax, !0x61 0F8C %r ; jl %ascii_other # deal with a-f 4883F8 67 ; cmp_rax, !0x67 0F8C %p ; jl %ascii_low # The rest that remains needs to be ignored E9 %r ; jmp %ascii_other :k # :Second_pass E8 %x ; call %Read_byte # Deal with EOF 4883F8 FC ; cmp_rax, !-4 0F84 %m ; je %Second_pass_done # Simply drop the label 4883F8 3A ; cmp_rax, !0x3A 0F85 %l ; jne %Second_pass_0 488B1D %T ; mov_rbx,[rip+DWORD] %scratch # Using scratch E8 %A ; call %consume_token # Read token E8 %H ; call %ClearScratch # Throw away token E9 %k ; jmp %Second_pass :l # :Second_pass_0 # Deal with % pointer 4883F8 25 ; cmp_rax, !0x25 0F84 %L ; je %StorePointer_rel4 # Deal with @ pointer 4883F8 40 ; cmp_rax, !0x40 0F84 %M ; je %StorePointer_rel2 # Deal with ! pointer 4883F8 21 ; cmp_rax, !0x21 0F84 %N ; je %StorePointer_rel1 # Deal with & pointer 4883F8 26 ; cmp_rax, !0x26 0F84 %O ; je %StorePointer_abs4 # Deal with $ pointer 4883F8 24 ; cmp_rax, !0x24 0F84 %P ; je %StorePointer_abs2 # :Second_pass_1 # Deal with everything else E8 %j ; call %hex # Process our char # Deal with EOF 4883F8 FC ; cmp_rax, !-4 0F84 %m ; je %Second_pass_done # deal with -1 values 4883F8 00 ; cmp_rax, !0 0F8C %k ; jl %Second_pass # deal with toggle 4983FF 00 ; cmp_r15, !0 0F84 %u ; je %print # process first byte of pair 4989C6 ; mov_r14,rax 49C7C7 00000000 ; mov_r15, %0 E9 %k ; jmp %Second_pass :m # :Second_pass_done :n # :EOF C3 ; ret :o # :ascii_num 83E8 30 ; sub_rax, !0x30 C3 ; ret :p # :ascii_low 83E8 57 ; sub_rax, !0x57 C3 ; ret :q # :ascii_high 83E8 37 ; sub_rax, !0x37 C3 ; ret :r # :ascii_other 48C7C0 FFFFFFFF ; mov_rax, %-1 C3 ; ret :s # :ascii_comment E8 %x ; call %Read_byte 4883F8 0D ; cmp_rax, !0x0D 0F84 %t ; je %ascii_comment_cr 4883F8 0A ; cmp_rax, !0x0A 0F85 %s ; jne %ascii_comment :t # :ascii_comment_cr 48C7C0 FFFFFFFF ; mov_rax, %-1 C3 ; ret # process second byte of pair :u # :print # update the sum and store in output 49C1E6 04 ; shl_r14, !4 4C01F0 ; add_rax,r14 # flip the toggle 49F7D7 ; not_r15 # Print our first Hex 48C7C2 01000000 ; mov_rdx, %1 # set the size of chars we want E8 %z ; call %print_chars 4983C5 01 ; add_r13, !1 # Increment IP E9 %k ; jmp %Second_pass :v # :Done # program completed Successfully 48C7C7 00000000 ; mov_rdi, %0 # All is well 48C7C0 3C000000 ; mov_rax, %0x3C # put the exit syscall number in eax 0F05 ; syscall # Call it a good day # Malloc isn't actually required if the program being built fits in the initial memory # However, it doesn't take much to add it. # Requires a value in RDI :w # :malloc 48C7C0 0C000000 ; mov_rax, %12 # the Syscall # for SYS_BRK 4153 ; push_r11 # Protect r11 0F05 ; syscall # call the Kernel 415B ; pop_r11 # Restore r11 C3 ; ret :x # :Read_byte # Attempt to read 1 byte from STDIN 48C7C2 01000000 ; mov_rdx, %1 # set the size of chars we want 488D35 %S ; lea_rsi,[rip+DWORD] %write # Where to put it 4C89CF ; mov_rdi,r9 # Where are we reading from 48C7C0 00000000 ; mov_rax, %0 # the syscall number for read 4153 ; push_r11 # Protect r11 0F05 ; syscall # call the Kernel 415B ; pop_r11 # Restore r11 4885C0 ; test_rax,rax # check what we got 0F84 %y ; je %Read_byte_1 # Got EOF call it done # load byte 8A05 %S ; mov_al,[rip+DWORD] %write # load char 480FB6C0 ; movzx_rax,al # We have to zero extend it to use it C3 ; ret # Deal with EOF :y # :Read_byte_1 48C7C0 FCFFFFFF ; mov_rax, %-4 # Put EOF in rax C3 ; ret :z # :print_chars 50 ; push_rax # Push address of chars onto stack 4889E6 ; mov_rsi,rsp # What we are writing 4C89D7 ; mov_rdi,r10 # Write to target file 48C7C0 01000000 ; mov_rax, %1 # the syscall number for write 4153 ; push_r11 # Protect HEAD 0F05 ; syscall # call the Kernel 415B ; pop_r11 # Restore HEAD 58 ; pop_rax # deallocate stack C3 ; ret # Receives pointer in RBX # Writes out char and updates RBX :A # :consume_token E8 %x ; call %Read_byte # Consume_token # Check for \t 4883F8 09 ; cmp_rax, !0x09 0F84 %B ; je %consume_token_done # Check for \n 4883F8 0A ; cmp_rax, !0x0A 0F84 %B ; je %consume_token_done # Check for ' ' 4883F8 20 ; cmp_rax, !0x20 0F84 %B ; je %consume_token_done # Check for '>' 4883F8 3E ; cmp_rax, !0x3E 0F84 %B ; je %consume_token_done # Looks like we are still reading token 8803 ; mov_[rbx],al # Store char 4883C3 01 ; add_rbx, !1 # Point to next spot E9 %A ; jmp %consume_token # loop until done :B # :consume_token_done 48C7C1 00000000 ; mov_rcx, %0 # Pad with nulls 48890B ; mov_[rbx],rcx 4883C3 08 ; add_rbx, !8 C3 ; ret :C # :StoreLabel 4C89E0 ; mov_rax,r12 # ENTRY 4981C4 18000000 ; add_r12, %24 # CALLOC 4C8968 08 ; mov_[rax+BYTE],r13 !8 # ENTRY->TARGET = IP 4C8918 ; mov_[rax],r11 # ENTRY->NEXT = JUMP_TABLE 4989C3 ; mov_r11,rax # JUMP_TABLE = ENTRY 4D8963 10 ; mov_[r11+BYTE],r12 !16 # ENTRY->NAME = TOKEN 4C89E3 ; mov_rbx,r12 # Write Starting after struct E8 %A ; call %consume_token # Collect whole string 4989DC ; mov_r12,rbx # Update HEAP E9 %a ; jmp %First_pass :D # :GetTarget 488B3D %T ; mov_rdi,[rip+DWORD] %scratch # Reset scratch 4C89D9 ; mov_rcx,r11 # Grab JUMP_TABLE 488B71 10 ; mov_rsi,[rcx+BYTE] !16 # I->NAME :E # :GetTarget_loop 8A06 ; mov_al,[rsi] # I->NAME[0] 8A1F ; mov_bl,[rdi] # scratch[0] 480FB6DB ; movzx_rbx,bl # Zero extend 480FB6C0 ; movzx_rax,al # Zero extend 38D8 ; cmp_al,bl # IF TOKEN == I->NAME 0F85 %F ; jne %GetTarget_miss # Oops 4883C6 01 ; add_rsi, !1 4881C7 01000000 ; add_rdi, %1 3C 00 ; cmp_al, !0 0F85 %E ; jne %GetTarget_loop # Loop until E9 %G ; jmp %GetTarget_done # Match # Miss :F # :GetTarget_miss 488B09 ; mov_rcx,[rcx] # I = I->NEXT 4883F9 00 ; cmp_rcx, !0 # IF NULL == I 0F84 %Q ; je %fail # Abort hard 488B71 10 ; mov_rsi,[rcx+BYTE] !16 # I->NAME 488B3D %T ; mov_rdi,[rip+DWORD] %scratch # Reset scratch E9 %E ; jmp %GetTarget_loop :G # :GetTarget_done 488B41 08 ; mov_rax,[rcx+BYTE] !8 # Get address C3 ; ret :H # :ClearScratch 50 ; push_rax # Protect against changes 53 ; push_rbx # And overwrites 51 ; push_rcx # While we work 488B1D %T ; mov_rbx,[rip+DWORD] %scratch # Where our scratch is 48C7C0 00000000 ; mov_rax, %0 # Using null :I # :ClearScratch_loop 488B0B ; mov_rcx,[rbx] # Get current value 8803 ; mov_[rbx],al # Because we want null 4883C3 01 ; add_rbx, !1 # Increment 4883F9 00 ; cmp_rcx, !0 # Check if we hit null 0F85 %I ; jne %ClearScratch_loop # Keep looping 59 ; pop_rcx # Don't Forget to 5B ; pop_rbx # Restore Damage 58 ; pop_rax # Entirely C3 ; ret :J # :StorePointer E8 %d ; call %Update_Pointer # Increment IP 488B1D %T ; mov_rbx,[rip+DWORD] %scratch # Write to scratch E8 %A ; call %consume_token # get token 50 ; push_rax # Protect base_sep_p 488B05 %T ; mov_rax,[rip+DWORD] %scratch # Pointer to scratch E8 %D ; call %GetTarget # Get address of pointer E8 %H ; call %ClearScratch # Clean up after ourselves 4C89EA ; mov_rdx,r13 # base = IP 5B ; pop_rbx # Restore base_sep_p 4883FB 3E ; cmp_rbx, !0x3E # If base_sep_p == '>' 0F85 %K ; jne %StorePointer_done # If not # Deal with %label>label case 50 ; push_rax # We need to preserve main target 488B1D %T ; mov_rbx,[rip+DWORD] %scratch # Write to scratch E8 %A ; call %consume_token # get token 488B05 %T ; mov_rax,[rip+DWORD] %scratch # Pointer to scratch E8 %D ; call %GetTarget # Get address of pointer E8 %H ; call %ClearScratch # Clean up after ourselves 4889C2 ; mov_rdx,rax # Use our new base 58 ; pop_rax # Restore main target :K # :StorePointer_done C3 ; ret :L # :StorePointer_rel4 E8 %J ; call %StorePointer # Do Common 4829D0 ; sub_rax,rdx # target - ip 48C7C2 04000000 ; mov_rdx, %4 # set the size of chars we want E8 %z ; call %print_chars E8 %H ; call %ClearScratch # Clean up after ourselves E9 %k ; jmp %Second_pass :M # :StorePointer_rel2 E8 %J ; call %StorePointer # Do Common 4829D0 ; sub_rax,rdx # target - ip 48C7C2 02000000 ; mov_rdx, %2 # set the size of chars we want E8 %z ; call %print_chars E8 %H ; call %ClearScratch # Clean up after ourselves E9 %k ; jmp %Second_pass :N # :StorePointer_rel1 E8 %J ; call %StorePointer # Do Common 4829D0 ; sub_rax,rdx # target - ip 48C7C2 01000000 ; mov_rdx, %1 # set the size of chars we want E8 %z ; call %print_chars E8 %H ; call %ClearScratch # Clean up after ourselves E9 %k ; jmp %Second_pass :O # :StorePointer_abs4 E8 %J ; call %StorePointer # Do Common 48C7C2 04000000 ; mov_rdx, %4 # set the size of chars we want E8 %z ; call %print_chars E8 %H ; call %ClearScratch # Clean up after ourselves E9 %k ; jmp %Second_pass :P # :StorePointer_abs2 E8 %J ; call %StorePointer # Do Common 48C7C2 02000000 ; mov_rdx, %2 # set the size of chars we want E8 %z ; call %print_chars E8 %H ; call %ClearScratch # Clean up after ourselves E9 %k ; jmp %Second_pass :Q # :fail # Some shit went wrong 48C7C7 01000000 ; mov_rdi, %1 # All is wrong 48C7C0 3C000000 ; mov_rax, %0x3C # put the exit syscall number in eax 0F05 ; syscall # Call it a good day :S # :write 00000000 ; NULL 00000000 ; NULL :T # :scratch 00000000 ; NULL 00000000 ; NULL # :ELF_end ================================================ FILE: tests/data/hex2.hex2 ================================================ # 110 lines 78 code 7 comments 25 blanks # SPDX-FileCopyrightText: 2019 Jeremiah Orians # # SPDX-License-Identifier: GPL-3.0-or-later ## ELF Header :ELF_base 7F 45 4C 46 ## e_ident[EI_MAG0-3] ELF's magic number 02 ## e_ident[EI_CLASS] Indicating 64 bit 01 ## e_ident[EI_DATA] Indicating little endianness 01 ## e_ident[EI_VERSION] Indicating original elf 03 ## e_ident[EI_OSABI] Set at 3 because FreeBSD is strict 00 ## e_ident[EI_ABIVERSION] Set at 0 because none cares 00 00 00 00 00 00 00 ## e_ident[EI_PAD] 02 00 ## e_type Indicating Executable 3E 00 ## e_machine Indicating AMD64 01 00 00 00 ## e_version Indicating original elf &_start 00 00 00 00 ## e_entry Address of the entry point (Number of bytes this header is + Base Address) %ELF_program_headers>ELF_base 00 00 00 00 ## e_phoff Address of program header table 00 00 00 00 00 00 00 00 ## e_shoff Address of section header table 00 00 00 00 ## e_flags 40 00 ## e_ehsize Indicating our 64 Byte header 38 00 ## e_phentsize size of a program header table 01 00 ## e_phnum number of entries in program table 00 00 ## e_shentsize size of a section header table 00 00 ## e_shnum number of entries in section table 00 00 ## e_shstrndx index of the section names ## Program Header :ELF_program_headers 01 00 00 00 ## p_type 07 00 00 00 ## ph_flags: PF-X|PF-W|PF-R = 7 00 00 00 00 00 00 00 00 ## p_offset &ELF_base 00 00 00 00 ## p_vaddr &ELF_base 00 00 00 00 ## p_physaddr %ELF_end>ELF_base 00 00 00 00 ## p_filesz %ELF_end>ELF_base 00 00 00 00 ## p_memsz 01 00 00 00 00 00 00 00 ## Required alignment :ELF_text :_start 58 ; pop_rax # Get the number of arguments 5F ; pop_rdi # Get the program name 5F ; pop_rdi # Get the actual output name 48C7C6 41020000 ; mov_rsi, %577 # Prepare file as O_WRONLY|O_CREAT|O_TRUNC 48C7C2 80010000 ; mov_rdx, %384 # Prepare file as RW for owner only (600 in octal) 48C7C0 02000000 ; mov_rax, %2 # the syscall number for open() 0F05 ; syscall # Now open that file 4989C7 ; mov_r15,rax # Preserve the file pointer we were given 48C7C0 0C000000 ; mov_rax, %12 # the Syscall # for SYS_BRK 48C7C7 00000000 ; mov_rdi, %0 # Get current brk 0F05 ; syscall # Let the kernel do the work 4989C6 ; mov_r14,rax # Set our malloc pointer 48C7C0 0C000000 ; mov_rax, %12 # the Syscall # for SYS_BRK 4C89F7 ; mov_r14,rax # Using current pointer 4881C7 00001000 ; add_rdi, %0x100000 # Allocate 1MB 0F05 ; syscall # Let the kernel do the work :core 5F ; pop_rdi # Get the actual input name 4883FF 00 ; cmp_rdi, !0 # Check for null string 0F84 %done ; je %done # Hit null be done 48C7C6 00000000 ; mov_rsi, %0 # prepare read_only 48C7C2 00000000 ; mov_rdx, %0 # prevent any interactions 48C7C0 02000000 ; mov_rax, %2 # the syscall number for open() 0F05 ; syscall # Now open that damn file 4989C5 ; mov_r13,rax # Protect INPUT :keep 48C7C2 00001000 ; mov_rdx, %0x100000 # set the size of chars we want 4C89F6 ; mov_rsi,r14 # Where to put it 4C89EF ; mov_rdi,r13 # Where are we reading from 48C7C0 00000000 ; mov_rax, %0 # the syscall number for read 0F05 ; syscall # call the Kernel 50 ; push_rax # Protect the number of bytes read 4889C2 ; mov_rdx,rax # Number of bytes to write 4C89F6 ; mov_rsi,r14 # What we are writing 4C89FF ; mov_rdi,r15 # Write to target file 48C7C0 01000000 ; mov_rax, %1 # the syscall number for write 0F05 ; syscall # call the Kernel 58 ; pop_rax # Get bytes read 483D 00001000 ; cmp_rax, %0x100000 # Check if buffer was fully used 0F84 %keep ; je %keep # Keep looping if was full E9 %core ; jmp %core # Otherwise move to next file :done # program completed Successfully 48C7C7 00000000 ; mov_rdi, %0 # All is well 48C7C0 3C000000 ; mov_rax, %0x3C # put the exit syscall number in eax 0F05 ; syscall # Call it a good day :ELF_end ================================================ FILE: tests/data/hicad.mac ================================================ REM 10 lines 4 code 3 comments 3 blanks START 59 REM Comment on a line %XY:=42 rem This is also a comment IF FOO= "foo" GOTO 10 END ================================================ FILE: tests/data/hledger.hledger ================================================ # 18 lines 6 code 10 comments 2 blanks # a comment ; another comment ; ^ a blank line comment account assets ; Declare valid account names and display order. a block comment end comment account assets:savings ; A subaccount. This one represents a bank account. account assets:checking ; Another. Note, 2+ spaces after the account name. account assets:receivable ; Accounting type is inferred from english names, account passifs ; or declared with a "type" tag, type:L account expenses ; type:X ; A follow-on comment line, indented. account expenses:rent ; Expense and revenue categories are also accounts. ; Subaccounts inherit their parent's type. ================================================ FILE: tests/data/hpp.hpp ================================================ /* 21 lines 11 code 5 comments 5 blanks */ #ifndef TEST_H #define TEST_H #include //Some definitions extern int out; void foo(); /* * Templated function */ template void print_value(T& t) { std::cout< link body ================================================ FILE: tests/data/janet.janet ================================================ # 17 lines 12 code 3 comments 2 blanks # Below is a function (defn a-fn "Docstring with a hash #" [a b] (+ 1 1)) (defn a-fn2 #"Not a doc" "String" [a b] # a and b right? (let [multiline "I'm a multline # string "] (str multline a b))) ================================================ FILE: tests/data/java.java ================================================ /* 37 lines 23 code 5 comments 9 blanks */ /* * Simple test class */ public class Test { int j = 0; // Not counted public static void main(String[] args) { Foo f = new Foo(); f.bar(); } } class Foo { public void bar() { System.out.println("FooBar"); //Not counted } } // issues/915 public class BackSlash { public void run() { "\\"; // 1 code + 2 blanks "\\"; // 1 code + 3 blanks } } ================================================ FILE: tests/data/javascript.js ================================================ // 33 lines, 14 code, 12 comments, 7 blanks /* * /* Nested comment * // single line comment * */ /* function add(a, b) { return a + b; } */ class Rectangle { constructor(width, height) { this.width = width; this.height = height; } get area() { return this.calcArea(); } calcArea() { return this.width * this.height; } } let rect = new Rectangle(20, 20); console.log(rect.area); // 400 // Comment ================================================ FILE: tests/data/jinja2.j2 ================================================ {# 5 lines 1 code 2 comments 2 blanks #} {# test comment #} {{ testvar }} ================================================ FILE: tests/data/jq.jq ================================================ # 11 lines 3 code 5 comments 3 blanks # A function to perform arithmetic def add_mul(adder; multiplier): # comment character in quotes "# Result: " + ((. + adder) * multiplier | tostring); # and demonstrate it 10 | add_mul(5; 4) # => "# Result: 60" # end of file ================================================ FILE: tests/data/jslt.jslt ================================================ // 126 lines 80 code 20 comments 26 blanks // https://github.com/schibsted/jslt/blob/master/examples/queens.jslt // =========================================================================== // N-Queens problem solution in JSLT // board is n lists of length n // 0 => no queen // 1 => queen // queens(8) produces // [ // [ 1, 0, 0, 0, 0, 0, 0, 0 ], // [ 0, 0, 0, 0, 1, 0, 0, 0 ], // [ 0, 0, 0, 0, 0, 0, 0, 1 ], // [ 0, 0, 0, 0, 0, 1, 0, 0 ], // [ 0, 0, 1, 0, 0, 0, 0, 0 ], // [ 0, 0, 0, 0, 0, 0, 1, 0 ], // [ 0, 1, 0, 0, 0, 0, 0, 0 ], // [ 0, 0, 0, 1, 0, 0, 0, 0 ] // ] def queens(n) solve(0, make-board($n)) def range(length, list) if (size($list) < $length) range($length, $list + [size($list)]) else $list def zeroes(length) [for (range($length, [])) 0] def make-board(n) [for (range($n, [])) zeroes($n)] def solve(row, board) let n = size($board) if ($row == $n) $board else let tries = [for (range($n, [])) let newboard = place-queen($row, ., $board) if (is-ok($newboard)) solve($row + 1, $newboard) else null] filter($tries)[0] def is-ok(board) rows-ok($board) and cols-ok($board) and diagonals-ok($board) def rows-ok(board) all-ok([for ($board) sum(.) <= 1]) def cols-ok(board) // 0, 1, 2, 3, ... let indexes = range(size($board), []) // list of columns instead of list of rows let columns = [for ($indexes) let col = (.) [for ($board) .[$col]] ] rows-ok($columns) def diagonals-ok(board) let n = size($board) let offsets = range($n - 1, [])[1 : ] // starts with 1 let diagonals-right = ( [diagonal-right($board, 0, 0)] + [for ($offsets) diagonal-right($board, 0, .)] + [for ($offsets) diagonal-right($board, ., 0)] ) let diagonals-left = ( [diagonal-left($board, 0, $n - 1)] + [for ($offsets) diagonal-left($board, ., $n - 1)] + [for ($offsets) diagonal-left($board, 0, .)] ) rows-ok($diagonals-right + $diagonals-left) def diagonal-right(board, rowoff, coloff) if ($rowoff >= size($board) or $coloff >= size($board)) [] else [$board[$rowoff][$coloff]] + diagonal-right($board, $rowoff+1, $coloff+1) def diagonal-left(board, rowoff, coloff) if ($rowoff >= size($board) or $coloff < 0) [] else diagonal-left($board, $rowoff + 1, $coloff - 1) + [$board[$rowoff][$coloff]] def sum(numbers) if (not($numbers)) 0 else $numbers[0] + sum($numbers[1 : ]) def all-ok(booleans) if (not($booleans)) true else $booleans[0] and all-ok($booleans[1 : ]) def place-queen(row, col, board) let changerow = $board[$row] let newrow = $changerow[ : $col] + [1] + $changerow[$col + 1 : ] $board[ : $row] + [$newrow] + $board[$row + 1 : ] def filter(array) if (not($array)) [] else if ($array[0]) [$array[0]] + filter($array[1 : ]) else filter($array[1 : ]) queens(8) ================================================ FILE: tests/data/jsonnet.jsonnet ================================================ // 13 lines 7 code 4 comments 2 blanks /* /**/ */ local func(a, b) = { // very useful a: a, b: b, # you forgot about me! c: " /* comment in a string! */ ", e: ' // in another string ', // another after f: ' # in a final string ', # comment after though } ================================================ FILE: tests/data/jupyter.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Cheat Sheet: Writing Python 2-3 compatible code" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- **Copyright (c):** 2013-2015 Python Charmers Pty Ltd, Australia.\n", "- **Author:** Ed Schofield.\n", "- **Licence:** Creative Commons Attribution.\n", "\n", "A PDF version is here: http://python-future.org/compatible_idioms.pdf\n", "\n", "This notebook shows you idioms for writing future-proof code that is compatible with both versions of Python: 2 and 3. It accompanies Ed Schofield's talk at PyCon AU 2014, \"Writing 2/3 compatible code\". (The video is here: .)\n", "\n", "Minimum versions:\n", "\n", " - Python 2: 2.6+\n", " - Python 3: 3.3+" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Setup" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The imports below refer to these ``pip``-installable packages on PyPI:\n", "\n", " import future # pip install future\n", " import builtins # pip install future\n", " import past # pip install future\n", " import six # pip install six\n", "\n", "The following scripts are also ``pip``-installable:\n", "\n", " futurize # pip install future\n", " pasteurize # pip install future\n", "\n", "See http://python-future.org and https://pythonhosted.org/six/ for more information." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Essential syntax differences" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### print" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "print 'Hello'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "print('Hello')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To print multiple strings, import ``print_function`` to prevent Py2 from interpreting it as a tuple:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "print 'Hello', 'Guido'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from __future__ import print_function # (at top of module)\n", "\n", "print('Hello', 'Guido')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "print >> sys.stderr, 'Hello'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from __future__ import print_function\n", "\n", "print('Hello', file=sys.stderr)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "print 'Hello'," ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from __future__ import print_function\n", "\n", "print('Hello', end='')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Raising exceptions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "raise ValueError, \"dodgy value\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "raise ValueError(\"dodgy value\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Raising exceptions with a traceback:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "traceback = sys.exc_info()[2]\n", "raise ValueError, \"dodgy value\", traceback" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only:\n", "raise ValueError(\"dodgy value\").with_traceback()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "from six import reraise as raise_\n", "# or\n", "from future.utils import raise_\n", "\n", "traceback = sys.exc_info()[2]\n", "raise_(ValueError, \"dodgy value\", traceback)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "from future.utils import raise_with_traceback\n", "\n", "raise_with_traceback(ValueError(\"dodgy value\"))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Exception chaining (PEP 3134):" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Setup:\n", "class DatabaseError(Exception):\n", " pass" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only\n", "class FileDatabase:\n", " def __init__(self, filename):\n", " try:\n", " self.file = open(filename)\n", " except IOError as exc:\n", " raise DatabaseError('failed to open') from exc" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from future.utils import raise_from\n", "\n", "class FileDatabase:\n", " def __init__(self, filename):\n", " try:\n", " self.file = open(filename)\n", " except IOError as exc:\n", " raise_from(DatabaseError('failed to open'), exc)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Testing the above:\n", "try:\n", " fd = FileDatabase('non_existent_file.txt')\n", "except Exception as e:\n", " assert isinstance(e.__cause__, IOError) # FileNotFoundError on Py3.3+ inherits from IOError" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Catching exceptions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "try:\n", " ...\n", "except ValueError, e:\n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "try:\n", " ...\n", "except ValueError as e:\n", " ..." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Division" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Integer division (rounding down):" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "assert 2 / 3 == 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "assert 2 // 3 == 0" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "\"True division\" (float division):" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only:\n", "assert 3 / 2 == 1.5" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from __future__ import division # (at top of module)\n", "\n", "assert 3 / 2 == 1.5" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "\"Old division\" (i.e. compatible with Py2 behaviour):" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "a = b / c # with any types" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from past.utils import old_div\n", "\n", "a = old_div(b, c) # always same as / on Py2" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Long integers" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Short integers are gone in Python 3 and ``long`` has become ``int`` (without the trailing ``L`` in the ``repr``)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "k = 9223372036854775808L\n", "\n", "# Python 2 and 3:\n", "k = 9223372036854775808" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "bigint = 1L\n", "\n", "# Python 2 and 3\n", "from builtins import int\n", "bigint = int(1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To test whether a value is an integer (of any kind):" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "if isinstance(x, (int, long)):\n", " ...\n", "\n", "# Python 3 only:\n", "if isinstance(x, int):\n", " ...\n", "\n", "# Python 2 and 3: option 1\n", "from builtins import int # subclass of long on Py2\n", "\n", "if isinstance(x, int): # matches both int and long on Py2\n", " ...\n", "\n", "# Python 2 and 3: option 2\n", "from past.builtins import long\n", "\n", "if isinstance(x, (int, long)):\n", " ..." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Octal constants" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "0644 # Python 2 only" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "0o644 # Python 2 and 3" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Backtick repr" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "`x` # Python 2 only" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "repr(x) # Python 2 and 3" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Metaclasses" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "class BaseForm(object):\n", " pass\n", "\n", "class FormType(type):\n", " pass" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "class Form(BaseForm):\n", " __metaclass__ = FormType\n", " pass" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only:\n", "class Form(BaseForm, metaclass=FormType):\n", " pass" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from six import with_metaclass\n", "# or\n", "from future.utils import with_metaclass\n", "\n", "class Form(with_metaclass(FormType, BaseForm)):\n", " pass" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Strings and bytes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Unicode (text) string literals" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If you are upgrading an existing Python 2 codebase, it may be preferable to mark up all string literals as unicode explicitly with ``u`` prefixes:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "s1 = 'The Zen of Python'\n", "s2 = u'きたないのよりきれいな方がいい\\n'\n", "\n", "# Python 2 and 3\n", "s1 = u'The Zen of Python'\n", "s2 = u'きたないのよりきれいな方がいい\\n'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The ``futurize`` and ``python-modernize`` tools do not currently offer an option to do this automatically." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If you are writing code for a new project or new codebase, you can use this idiom to make all string literals in a module unicode strings:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3\n", "from __future__ import unicode_literals # at top of module\n", "\n", "s1 = 'The Zen of Python'\n", "s2 = 'きたないのよりきれいな方がいい\\n'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "See http://python-future.org/unicode_literals.html for more discussion on which style to use." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Byte-string literals" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "s = 'This must be a byte-string'\n", "\n", "# Python 2 and 3\n", "s = b'This must be a byte-string'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To loop over a byte-string with possible high-bit characters, obtaining each character as a byte-string of length 1:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "for bytechar in 'byte-string with high-bit chars like \\xf9':\n", " ...\n", "\n", "# Python 3 only:\n", "for myint in b'byte-string with high-bit chars like \\xf9':\n", " bytechar = bytes([myint])\n", "\n", "# Python 2 and 3:\n", "from builtins import bytes\n", "for myint in bytes(b'byte-string with high-bit chars like \\xf9'):\n", " bytechar = bytes([myint])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As an alternative, ``chr()`` and ``.encode('latin-1')`` can be used to convert an int into a 1-char byte string:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only:\n", "for myint in b'byte-string with high-bit chars like \\xf9':\n", " char = chr(myint) # returns a unicode string\n", " bytechar = char.encode('latin-1')\n", "\n", "# Python 2 and 3:\n", "from builtins import bytes, chr\n", "for myint in bytes(b'byte-string with high-bit chars like \\xf9'):\n", " char = chr(myint) # returns a unicode string\n", " bytechar = char.encode('latin-1') # forces returning a byte str" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### basestring" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "a = u'abc'\n", "b = 'def'\n", "assert (isinstance(a, basestring) and isinstance(b, basestring))\n", "\n", "# Python 2 and 3: alternative 1\n", "from past.builtins import basestring # pip install future\n", "\n", "a = u'abc'\n", "b = b'def'\n", "assert (isinstance(a, basestring) and isinstance(b, basestring))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 2: refactor the code to avoid considering\n", "# byte-strings as strings.\n", "\n", "from builtins import str\n", "a = u'abc'\n", "b = b'def'\n", "c = b.decode()\n", "assert isinstance(a, str) and isinstance(c, str)\n", "# ..." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### unicode" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "templates = [u\"blog/blog_post_detail_%s.html\" % unicode(slug)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 1\n", "from builtins import str\n", "templates = [u\"blog/blog_post_detail_%s.html\" % str(slug)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 2\n", "from builtins import str as text\n", "templates = [u\"blog/blog_post_detail_%s.html\" % text(slug)]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### StringIO" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from StringIO import StringIO\n", "# or:\n", "from cStringIO import StringIO\n", "\n", "# Python 2 and 3:\n", "from io import BytesIO # for handling byte strings\n", "from io import StringIO # for handling unicode strings" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Imports relative to a package" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Suppose the package is:\n", "\n", " mypackage/\n", " __init__.py\n", " submodule1.py\n", " submodule2.py\n", " \n", "and the code below is in ``submodule1.py``:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only: \n", "import submodule2" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from . import submodule2" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "# To make Py2 code safer (more like Py3) by preventing\n", "# implicit relative imports, you can also add this to the top:\n", "from __future__ import absolute_import" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Dictionaries" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "heights = {'Fred': 175, 'Anne': 166, 'Joe': 192}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Iterating through ``dict`` keys/values/items" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Iterable dict keys:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "for key in heights.iterkeys():\n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "for key in heights:\n", " ..." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Iterable dict values:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "for value in heights.itervalues():\n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Idiomatic Python 3\n", "for value in heights.values(): # extra memory overhead on Py2\n", " ..." ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "from builtins import dict\n", "\n", "heights = dict(Fred=175, Anne=166, Joe=192)\n", "for key in heights.values(): # efficient on Py2 and Py3\n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "from builtins import itervalues\n", "# or\n", "from six import itervalues\n", "\n", "for key in itervalues(heights):\n", " ..." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Iterable dict items:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "for (key, value) in heights.iteritems():\n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "for (key, value) in heights.items(): # inefficient on Py2 \n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "from future.utils import viewitems\n", "\n", "for (key, value) in viewitems(heights): # also behaves like a set\n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 3\n", "from future.utils import iteritems\n", "# or\n", "from six import iteritems\n", "\n", "for (key, value) in iteritems(heights):\n", " ..." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### dict keys/values/items as a list" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "dict keys as a list:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "keylist = heights.keys()\n", "assert isinstance(keylist, list)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "keylist = list(heights)\n", "assert isinstance(keylist, list)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "dict values as a list:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "heights = {'Fred': 175, 'Anne': 166, 'Joe': 192}\n", "valuelist = heights.values()\n", "assert isinstance(valuelist, list)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "valuelist = list(heights.values()) # inefficient on Py2" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "from builtins import dict\n", "\n", "heights = dict(Fred=175, Anne=166, Joe=192)\n", "valuelist = list(heights.values())" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 3\n", "from future.utils import listvalues\n", "\n", "valuelist = listvalues(heights)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 4\n", "from future.utils import itervalues\n", "# or\n", "from six import itervalues\n", "\n", "valuelist = list(itervalues(heights))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "dict items as a list:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "itemlist = list(heights.items()) # inefficient on Py2" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "from future.utils import listitems\n", "\n", "itemlist = listitems(heights)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 3\n", "from future.utils import iteritems\n", "# or\n", "from six import iteritems\n", "\n", "itemlist = list(iteritems(heights))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Custom class behaviour" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Custom iterators" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "class Upper(object):\n", " def __init__(self, iterable):\n", " self._iter = iter(iterable)\n", " def next(self): # Py2-style\n", " return self._iter.next().upper()\n", " def __iter__(self):\n", " return self\n", "\n", "itr = Upper('hello')\n", "assert itr.next() == 'H' # Py2-style\n", "assert list(itr) == list('ELLO')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "from builtins import object\n", "\n", "class Upper(object):\n", " def __init__(self, iterable):\n", " self._iter = iter(iterable)\n", " def __next__(self): # Py3-style iterator interface\n", " return next(self._iter).upper() # builtin next() function calls\n", " def __iter__(self):\n", " return self\n", "\n", "itr = Upper('hello')\n", "assert next(itr) == 'H' # compatible style\n", "assert list(itr) == list('ELLO')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "from future.utils import implements_iterator\n", "\n", "@implements_iterator\n", "class Upper(object):\n", " def __init__(self, iterable):\n", " self._iter = iter(iterable)\n", " def __next__(self): # Py3-style iterator interface\n", " return next(self._iter).upper() # builtin next() function calls\n", " def __iter__(self):\n", " return self\n", "\n", "itr = Upper('hello')\n", "assert next(itr) == 'H'\n", "assert list(itr) == list('ELLO')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Custom ``__str__`` methods" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "class MyClass(object):\n", " def __unicode__(self):\n", " return 'Unicode string: \\u5b54\\u5b50'\n", " def __str__(self):\n", " return unicode(self).encode('utf-8')\n", "\n", "a = MyClass()\n", "print(a) # prints encoded string" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Unicode string: 孔子\n" ] } ], "source": [ "# Python 2 and 3:\n", "from future.utils import python_2_unicode_compatible\n", "\n", "@python_2_unicode_compatible\n", "class MyClass(object):\n", " def __str__(self):\n", " return u'Unicode string: \\u5b54\\u5b50'\n", "\n", "a = MyClass()\n", "print(a) # prints string encoded as utf-8 on Py2" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Custom ``__nonzero__`` vs ``__bool__`` method:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "class AllOrNothing(object):\n", " def __init__(self, l):\n", " self.l = l\n", " def __nonzero__(self):\n", " return all(self.l)\n", "\n", "container = AllOrNothing([0, 100, 200])\n", "assert not bool(container)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from builtins import object\n", "\n", "class AllOrNothing(object):\n", " def __init__(self, l):\n", " self.l = l\n", " def __bool__(self):\n", " return all(self.l)\n", "\n", "container = AllOrNothing([0, 100, 200])\n", "assert not bool(container)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Lists versus iterators" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### xrange" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "for i in xrange(10**8):\n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: forward-compatible\n", "from builtins import range\n", "for i in range(10**8):\n", " ..." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: backward-compatible\n", "from past.builtins import xrange\n", "for i in xrange(10**8):\n", " ..." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### range" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "mylist = range(5)\n", "assert mylist == [0, 1, 2, 3, 4]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: forward-compatible: option 1\n", "mylist = list(range(5)) # copies memory on Py2\n", "assert mylist == [0, 1, 2, 3, 4]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: forward-compatible: option 2\n", "from builtins import range\n", "\n", "mylist = list(range(5))\n", "assert mylist == [0, 1, 2, 3, 4]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 3\n", "from future.utils import lrange\n", "\n", "mylist = lrange(5)\n", "assert mylist == [0, 1, 2, 3, 4]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: backward compatible\n", "from past.builtins import range\n", "\n", "mylist = range(5)\n", "assert mylist == [0, 1, 2, 3, 4]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### map" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "mynewlist = map(f, myoldlist)\n", "assert mynewlist == [f(x) for x in myoldlist]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "# Idiomatic Py3, but inefficient on Py2\n", "mynewlist = list(map(f, myoldlist))\n", "assert mynewlist == [f(x) for x in myoldlist]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "from builtins import map\n", "\n", "mynewlist = list(map(f, myoldlist))\n", "assert mynewlist == [f(x) for x in myoldlist]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 3\n", "try:\n", " import itertools.imap as map\n", "except ImportError:\n", " pass\n", "\n", "mynewlist = list(map(f, myoldlist)) # inefficient on Py2\n", "assert mynewlist == [f(x) for x in myoldlist]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 4\n", "from future.utils import lmap\n", "\n", "mynewlist = lmap(f, myoldlist)\n", "assert mynewlist == [f(x) for x in myoldlist]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 5\n", "from past.builtins import map\n", "\n", "mynewlist = map(f, myoldlist)\n", "assert mynewlist == [f(x) for x in myoldlist]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### imap" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from itertools import imap\n", "\n", "myiter = imap(func, myoldlist)\n", "assert isinstance(myiter, iter)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only:\n", "myiter = map(func, myoldlist)\n", "assert isinstance(myiter, iter)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "from builtins import map\n", "\n", "myiter = map(func, myoldlist)\n", "assert isinstance(myiter, iter)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "try:\n", " import itertools.imap as map\n", "except ImportError:\n", " pass\n", "\n", "myiter = map(func, myoldlist)\n", "assert isinstance(myiter, iter)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### zip, izip" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As above with ``zip`` and ``itertools.izip``." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### filter, ifilter" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As above with ``filter`` and ``itertools.ifilter`` too." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Other builtins" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### File IO with open()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Python 2 only\n", "f = open('myfile.txt')\n", "data = f.read() # as a byte string\n", "text = data.decode('utf-8')\n", "\n", "# Python 2 and 3: alternative 1\n", "from io import open\n", "f = open('myfile.txt', 'rb')\n", "data = f.read() # as bytes\n", "text = data.decode('utf-8') # unicode, not bytes\n", "\n", "# Python 2 and 3: alternative 2\n", "from io import open\n", "f = open('myfile.txt', encoding='utf-8')\n", "text = f.read() # unicode, not bytes" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### reduce()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "assert reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) == 1+2+3+4+5" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from functools import reduce\n", "\n", "assert reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) == 1+2+3+4+5" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### raw_input()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "name = raw_input('What is your name? ')\n", "assert isinstance(name, str) # native str" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from builtins import input\n", "\n", "name = input('What is your name? ')\n", "assert isinstance(name, str) # native str on Py2 and Py3" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### input()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "input(\"Type something safe please: \")" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3\n", "from builtins import input\n", "eval(input(\"Type something safe please: \"))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Warning: using either of these is **unsafe** with untrusted input." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### file()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "f = file(pathname)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "f = open(pathname)\n", "\n", "# But preferably, use this:\n", "from io import open\n", "f = open(pathname, 'rb') # if f.read() should return bytes\n", "# or\n", "f = open(pathname, 'rt') # if f.read() should return unicode text" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### exec" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Python 2 only:\n", "exec 'x = 10'\n", "\n", "# Python 2 and 3:\n", "exec('x = 10')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Python 2 only:\n", "g = globals()\n", "exec 'x = 10' in g\n", "\n", "# Python 2 and 3:\n", "g = globals()\n", "exec('x = 10', g)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Python 2 only:\n", "l = locals()\n", "exec 'x = 10' in g, l\n", "\n", "# Python 2 and 3:\n", "exec('x = 10', g, l)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "But note that Py3's `exec()` is less powerful (and less dangerous) than Py2's `exec` statement." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### execfile()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "execfile('myfile.py')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 1\n", "from past.builtins import execfile\n", "\n", "execfile('myfile.py')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 2\n", "exec(compile(open('myfile.py').read()))\n", "\n", "# This can sometimes cause this:\n", "# SyntaxError: function ... uses import * and bare exec ...\n", "# See https://github.com/PythonCharmers/python-future/issues/37" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### unichr()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "assert unichr(8364) == '€'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only:\n", "assert chr(8364) == '€'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from builtins import chr\n", "assert chr(8364) == '€'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### intern()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "intern('mystring')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only:\n", "from sys import intern\n", "intern('mystring')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 1\n", "from past.builtins import intern\n", "intern('mystring')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 2\n", "from six.moves import intern\n", "intern('mystring')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 3\n", "from future.standard_library import install_aliases\n", "install_aliases()\n", "from sys import intern\n", "intern('mystring')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 2\n", "try:\n", " from sys import intern\n", "except ImportError:\n", " pass\n", "intern('mystring')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### apply()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "args = ('a', 'b')\n", "kwargs = {'kwarg1': True}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "apply(f, args, kwargs)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 1\n", "f(*args, **kwargs)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 2\n", "from past.builtins import apply\n", "apply(f, args, kwargs)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### chr()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "assert chr(64) == b'@'\n", "assert chr(200) == b'\\xc8'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only: option 1\n", "assert chr(64).encode('latin-1') == b'@'\n", "assert chr(0xc8).encode('latin-1') == b'\\xc8'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 1\n", "from builtins import chr\n", "\n", "assert chr(64).encode('latin-1') == b'@'\n", "assert chr(0xc8).encode('latin-1') == b'\\xc8'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only: option 2\n", "assert bytes([64]) == b'@'\n", "assert bytes([0xc8]) == b'\\xc8'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: option 2\n", "from builtins import bytes\n", "\n", "assert bytes([64]) == b'@'\n", "assert bytes([0xc8]) == b'\\xc8'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### cmp()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "assert cmp('a', 'b') < 0 and cmp('b', 'a') > 0 and cmp('c', 'c') == 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 1\n", "from past.builtins import cmp\n", "assert cmp('a', 'b') < 0 and cmp('b', 'a') > 0 and cmp('c', 'c') == 0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 2\n", "cmp = lambda(x, y): (x > y) - (x < y)\n", "assert cmp('a', 'b') < 0 and cmp('b', 'a') > 0 and cmp('c', 'c') == 0" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### reload()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "reload(mymodule)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3\n", "from imp import reload\n", "reload(mymodule)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Standard library" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### dbm modules" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "import anydbm\n", "import whichdb\n", "import dbm\n", "import dumbdbm\n", "import gdbm\n", "\n", "# Python 2 and 3: alternative 1\n", "from future import standard_library\n", "standard_library.install_aliases()\n", "\n", "import dbm\n", "import dbm.ndbm\n", "import dbm.dumb\n", "import dbm.gnu\n", "\n", "# Python 2 and 3: alternative 2\n", "from future.moves import dbm\n", "from future.moves.dbm import dumb\n", "from future.moves.dbm import ndbm\n", "from future.moves.dbm import gnu\n", "\n", "# Python 2 and 3: alternative 3\n", "from six.moves import dbm_gnu\n", "# (others not supported)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### commands / subprocess modules" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "from commands import getoutput, getstatusoutput\n", "\n", "# Python 2 and 3\n", "from future import standard_library\n", "standard_library.install_aliases()\n", "\n", "from subprocess import getoutput, getstatusoutput" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### subprocess.check_output()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2.7 and above\n", "from subprocess import check_output\n", "\n", "# Python 2.6 and above: alternative 1\n", "from future.moves.subprocess import check_output\n", "\n", "# Python 2.6 and above: alternative 2\n", "from future import standard_library\n", "standard_library.install_aliases()\n", "\n", "from subprocess import check_output" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### collections: Counter, OrderedDict, ChainMap" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2.7 and above\n", "from collections import Counter, OrderedDict, ChainMap\n", "\n", "# Python 2.6 and above: alternative 1\n", "from future.backports import Counter, OrderedDict, ChainMap\n", "\n", "# Python 2.6 and above: alternative 2\n", "from future import standard_library\n", "standard_library.install_aliases()\n", "\n", "from collections import Counter, OrderedDict, ChainMap" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### StringIO module" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only\n", "from StringIO import StringIO\n", "from cStringIO import StringIO" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3\n", "from io import BytesIO\n", "# and refactor StringIO() calls to BytesIO() if passing byte-strings" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### http module" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "import httplib\n", "import Cookie\n", "import cookielib\n", "import BaseHTTPServer\n", "import SimpleHTTPServer\n", "import CGIHttpServer\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "import http.client\n", "import http.cookies\n", "import http.cookiejar\n", "import http.server" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### xmlrpc module" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "import DocXMLRPCServer\n", "import SimpleXMLRPCServer\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "import xmlrpc.server" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "import xmlrpclib\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "import xmlrpc.client" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### html escaping and entities" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3:\n", "from cgi import escape\n", "\n", "# Safer (Python 2 and 3, after ``pip install future``):\n", "from html import escape\n", "\n", "# Python 2 only:\n", "from htmlentitydefs import codepoint2name, entitydefs, name2codepoint\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "from html.entities import codepoint2name, entitydefs, name2codepoint" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### html parsing" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from HTMLParser import HTMLParser\n", "\n", "# Python 2 and 3 (after ``pip install future``)\n", "from html.parser import HTMLParser\n", "\n", "# Python 2 and 3 (alternative 2):\n", "from future.moves.html.parser import HTMLParser" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### urllib module" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "``urllib`` is the hardest module to use from Python 2/3 compatible code. You may like to use Requests (http://python-requests.org) instead." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from urlparse import urlparse\n", "from urllib import urlencode\n", "from urllib2 import urlopen, Request, HTTPError" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 3 only:\n", "from urllib.parse import urlparse, urlencode\n", "from urllib.request import urlopen, Request\n", "from urllib.error import HTTPError" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: easiest option\n", "from future.standard_library import install_aliases\n", "install_aliases()\n", "\n", "from urllib.parse import urlparse, urlencode\n", "from urllib.request import urlopen, Request\n", "from urllib.error import HTTPError" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 2\n", "from future.standard_library import hooks\n", "\n", "with hooks():\n", " from urllib.parse import urlparse, urlencode\n", " from urllib.request import urlopen, Request\n", " from urllib.error import HTTPError" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 3\n", "from future.moves.urllib.parse import urlparse, urlencode\n", "from future.moves.urllib.request import urlopen, Request\n", "from future.moves.urllib.error import HTTPError\n", "# or\n", "from six.moves.urllib.parse import urlparse, urlencode\n", "from six.moves.urllib.request import urlopen\n", "from six.moves.urllib.error import HTTPError" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 and 3: alternative 4\n", "try:\n", " from urllib.parse import urlparse, urlencode\n", " from urllib.request import urlopen, Request\n", " from urllib.error import HTTPError\n", "except ImportError:\n", " from urlparse import urlparse\n", " from urllib import urlencode\n", " from urllib2 import urlopen, Request, HTTPError" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Tkinter" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "import Tkinter\n", "import Dialog\n", "import FileDialog\n", "import ScrolledText\n", "import SimpleDialog\n", "import Tix \n", "import Tkconstants\n", "import Tkdnd \n", "import tkColorChooser\n", "import tkCommonDialog\n", "import tkFileDialog\n", "import tkFont\n", "import tkMessageBox\n", "import tkSimpleDialog\n", "import ttk\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "import tkinter\n", "import tkinter.dialog\n", "import tkinter.filedialog\n", "import tkinter.scrolledtext\n", "import tkinter.simpledialog\n", "import tkinter.tix\n", "import tkinter.constants\n", "import tkinter.dnd\n", "import tkinter.colorchooser\n", "import tkinter.commondialog\n", "import tkinter.filedialog\n", "import tkinter.font\n", "import tkinter.messagebox\n", "import tkinter.simpledialog\n", "import tkinter.ttk" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### socketserver" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "import SocketServer\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "import socketserver" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### copy_reg, copyreg" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "import copy_reg\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "import copyreg" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### configparser" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from ConfigParser import ConfigParser\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "from configparser import ConfigParser" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### queue" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from Queue import Queue, heapq, deque\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "from queue import Queue, heapq, deque" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### repr, reprlib" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from repr import aRepr, repr\n", "\n", "# Python 2 and 3 (after ``pip install future``):\n", "from reprlib import aRepr, repr" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### UserDict, UserList, UserString" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from UserDict import UserDict\n", "from UserList import UserList\n", "from UserString import UserString\n", "\n", "# Python 3 only:\n", "from collections import UserDict, UserList, UserString\n", "\n", "# Python 2 and 3: alternative 1\n", "from future.moves.collections import UserDict, UserList, UserString\n", "\n", "# Python 2 and 3: alternative 2\n", "from six.moves import UserDict, UserList, UserString\n", "\n", "# Python 2 and 3: alternative 3\n", "from future.standard_library import install_aliases\n", "install_aliases()\n", "from collections import UserDict, UserList, UserString" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### itertools: filterfalse, zip_longest" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Python 2 only:\n", "from itertools import ifilterfalse, izip_longest\n", "\n", "# Python 3 only:\n", "from itertools import filterfalse, zip_longest\n", "\n", "# Python 2 and 3: alternative 1\n", "from future.moves.itertools import filterfalse, zip_longest\n", "\n", "# Python 2 and 3: alternative 2\n", "from six.moves import filterfalse, zip_longest\n", "\n", "# Python 2 and 3: alternative 3\n", "from future.standard_library import install_aliases\n", "install_aliases()\n", "from itertools import filterfalse, zip_longest" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.4.3" } }, "nbformat": 4, "nbformat_minor": 0 } ================================================ FILE: tests/data/justfile ================================================ # 215 lines 154 code 11 comments 50 blanks set shell := ["sh", "-c"] set windows-shell := ["powershell.exe", "-NoLogo", "-Command"] set allow-duplicate-recipes set positional-arguments set dotenv-load set export alias s := serve bt := '0' export RUST_BACKTRACE_1 := bt log := "warn" export JUST_LOG := (log + "ing" + `grep loop /etc/networks | cut -f2`) tmpdir := `mktemp` version := "0.2.7" tardir := tmpdir / "awesomesauce-" + version foo1 := / "tmp" foo2_3 := "a/" tarball := tardir + ".tar.gz" export RUST_BACKTRACE_2 := "1" string-with-tab := "\t" string-with-newline := "\n" string-with-carriage-return := "\r" string-with-double-quote := "\"" string-with-slash := "\\" string-with-no-newline := "\ " # Newlines in variables single := ' hello ' double := " goodbye " escapes := '\t\n\r\"\\' # this string will evaluate to `foo\nbar\n` x := ''' foo bar ''' # this string will evaluate to `abc\n wuv\nbar\n` y := """ abc wuv xyz """ for: for file in `ls .`; do \ echo $file; \ done serve: touch {{tmpdir}}/file # This backtick evaluates the command `echo foo\necho bar\n`, which produces the value `foo\nbar\n`. stuff := ``` echo foo echo bar ``` an_arch := trim(lowercase(justfile())) + arch() trim_end := trim_end("99.99954% ") home_dir := replace(env_var('HOME') / "yep", 'yep', '') quoted := quote("some things beyond\"$()^%#@!|-+=_*&'`") smartphone := trim_end_match('blah.txt', 'txt') museum := trim_start_match(trim_start(trim_end_matches(' yep_blah.txt.txt', '.txt')), 'yep_') water := trim_start_matches('ssssssoup.txt', 's') congress := uppercase(os()) fam := os_family() path_1 := absolute_path('test') path_2 := '/tmp/subcommittee.txt' ext_z := extension(path_2) exe_name := file_name(just_executable()) a_stem := file_stem(path_2) a_parent := parent_directory(path_2) sans_ext := without_extension(path_2) camera := join('tmp', 'dir1', 'dir2', path_2) cleaned := clean('/tmp/blah/..///thing.txt') id__path := '/tmp' / sha256('blah') / sha256_file(justfile()) _another_var := env_var_or_default("HOME", justfile_directory()) python := `which python` exists := if path_exists(just_executable()) =~ '^/User' { uuid() } else { 'yeah' } foo := if env_var("_") == "/usr/bin/env" { `touch /tmp/a_file` } else { "dummy-value" } foo_b := if "hello" == "goodbye" { "xyz" } else { if "no" == "no" { "yep"} else { error("123") } } foo_c := if "hello" == "goodbye" { "xyz" } else if "a" == "a" { "abc" } else { "123" } bar: @echo {{foo}} bar2 foo_stuff: echo {{ if foo_stuff == "bar" { "hello" } else { "goodbye" } }} executable: @echo The executable is at: {{just_executable()}} rustfmt: find {{invocation_directory()}} -name \*.rs -exec rustfmt {} \; test: echo "{{home_dir}}" linewise: Write-Host "Hello, world!" serve2: @echo "Starting server with database $DATABASE_ADDRESS on port $SERVER_PORT…" shebang := if os() == 'windows' { 'powershell.exe' } else { '/usr/bin/env pwsh' } shebang: #!{{shebang}} $PSV = $PSVersionTable.PSVersion | % {"$_" -split "\." } $psver = $PSV[0] + "." + $PSV[1] if ($PSV[2].Length -lt 4) { $psver += "." + $PSV[2] + " Core" } else { $psver += " Desktop" } echo "PowerShell $psver" @foo: echo bar @test5 *args='': bash -c 'while (( "$#" )); do echo - $1; shift; done' -- "$@" test2 $RUST_BACKTRACE="1": # will print a stack trace if it crashes cargo test notify m="": keybase chat send --topic-type "chat" --channel "upd(): {{m}}" # Sample project script 2 script2 *ARGS: {{ python }} script2.py {{ ARGS }} braces: echo 'I {{{{LOVE}} curly braces!' _braces2: echo '{{'I {{LOVE}} curly braces!'}}' _braces3: echo 'I {{ "{{" }}LOVE}} curly braces!' foo2: -@cat foo echo 'Done!' test3 target tests=path_1: @echo 'Testing {{target}}:{{tests}}…' ./test --tests {{tests}} {{target}} test4 triple=(an_arch + "-unknown-unknown") input=(an_arch / "input.dat"): ./test {{triple}} variadic $VAR1_1 VAR2 VAR3 VAR4=("a") +$FLAGS='-q': foo2 braces cargo test {{FLAGS}} time: @-date +"%H:%S" -cat /tmp/nonexistent_file.txt @echo "finished" justwords: grep just \ --text /usr/share/dict/words \ > /tmp/justwords # Subsequent dependencies # https://just.systems/man/en/chapter_37.html # To test, run `$ just -f test-suite.just b` a: echo 'A!' b: a && d echo 'B start!' just -f {{justfile()}} c echo 'B end!' c: echo 'C!' d: echo 'D!' ================================================ FILE: tests/data/k.k ================================================ // 8 lines 2 code 4 comments 2 blanks /suduko solver / initial state x:.:'"200370009009200007001004002050000800008000900006000040900100500800007600400089001" / breadth search all solutions (p:row col box for each position) *(,x)(,/{@[x;y;:;]'&~in[!10]x*|/p[;y]=p,:3/:_(p:9\:!81)%3}')/&~x ================================================ FILE: tests/data/kaem.kaem ================================================ # 43 lines 2 code 33 comments 8 blanks #! /usr/bin/env bash # Mes --- Maxwell Equations of Software # Copyright © 2017 Jan Nieuwenhuizen # Copyright © 2017 Jeremiah Orians # # This file is part of Mes. # # Mes is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 3 of the License, or (at # your option) any later version. # # Mes is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Mes. If not, see . # Can also be run by kaem or any other shell of your personal choice # To run in kaem simply: kaem --verbose --strict # Warning all binaries prior to the use of blood-elf will not be readable by # Objdump, you may need to use ndism or gdb to view the assembly in the binary. ############################################### # Phase-0 Build hex0 from bootstrapped binary # ############################################### ./bootstrap-seeds/POSIX/AMD64/hex0-seed ./AMD64/hex0_AMD64.hex0 ./AMD64/artifact/hex0 # hex0 should have the exact same checksum as hex0-seed as they are both supposed # to be built from hex0_amd64.hex0 and by definition must be identical ######################################### # Phase-0b Build minimal kaem from hex0 # ######################################### ./AMD64/artifact/hex0 ./AMD64/kaem-minimal.hex0 ./AMD64/artifact/kaem-0 # for checksum validation reasons ================================================ FILE: tests/data/kakoune_script.kak ================================================ # 13 lines, 8 code, 2 comments, 3 blanks hook global BufCreate (.*/)?(kakrc|.*.kak) %{ set-option buffer filetype kak } echo "This is a string". echo 'This is a multiline string # with a hash in it.' # This is a comment. ================================================ FILE: tests/data/koka.kk ================================================ // 29 lines 11 code 12 comments 6 blanks /*--------------------------------------------------------------------------- Copyright 2020-2021, Microsoft Research, Daan Leijen. This is free software; you can redistribute it and/or modify it under the terms of the Apache License, Version 2.0. A copy of the License can be found in the LICENSE file at the root of this distribution. ---------------------------------------------------------------------------*/ /* Run processes. */ module std/os/process import std/os/path extern import { c file "process-inline.c" } // Run a command in the shell and return its output as a string. pub extern run-system-read( cmd : string ) : io error { c "kk_os_run_command_error" } // Run a command in the shell pub extern run-system( cmd : string ) : io int { c "kk_os_run_system_prim" } ================================================ FILE: tests/data/ksh.ksh ================================================ #!/bin/ksh # 17 lines, 11 code, 4 comments, 2 blanks # first comment files="/etc/passwd /etc/group /etc/hosts" for f in $files; do if [ ! -f $f ] then echo "$f file missing!" fi done # second comment for f in $(ls /tmp/*) do print "Full file path in /tmp dir : $f" done ================================================ FILE: tests/data/kvlanguage.kv ================================================ # 22 lines 17 code 3 comments 2 blanks #:kivy 2.0.0 #:import C kivy.utils.get_color_from_hex #:import KeypadButton keypadbutton #:include keypadbutton.kv #:set name value : color: C('#27272A') # general comment : width: self.minimum_width height: self.minimum_height size_hint: None, None cols: 3 spacing: 6 KeypadButton: text: '1' disabled: root.disabled on_press: root.dispatch('on_key_pressed', self.key_val) # a final comment ================================================ FILE: tests/data/lalrpop.lalrpop ================================================ // 37 lines 26 code 3 comments 8 blanks use crate::ast::{ExprSymbol, Opcode}; use crate::tok9::Tok; grammar<'input>(input: &'input str); // line comment pub Expr: Box> = { // comment 1 Expr r##"verbatim2"## Factor => Box::new(ExprSymbol::Op(<>)), Factor, // comment 2 }; Factor: Box> = { // comment 3 Factor "FactorOp" Term => Box::new(ExprSymbol::Op(<>)), Term, }; // comment 4 Term: Box> = { r#"verbatim"# => Box::new(ExprSymbol::NumSymbol(<>)), "(" ")" }; extern { type Location = usize; type Error = (); enum Tok<'input> { r#"verbatim"# => Tok::NumSymbol(<&'input str>), "FactorOp" => Tok::FactorOp(), r##"verbatim2"## => Tok::ExprOp(), "(" => Tok::ParenOpen, ")" => Tok::ParenClose, } } ================================================ FILE: tests/data/linguafranca.lf ================================================ // 36 lines 16 code 9 comments 11 blanks target Rust; // A C style comment import KeyboardEvents from "KeyboardEvents.lf"; /* A block comment */ # a python like comment main reactor Snake(grid_side: usize(32), food_limit: u32(2)) { // counts as 2 lines of Rust code and one blank preamble {= use crate::snakes::*; use rand::prelude::*; =} /// rust doc comment keyboard = new KeyboardEvents(); // T state snake: CircularSnake ({= CircularSnake::new(grid_side) =}); state grid: SnakeGrid ({= SnakeGrid::new(grid_side, &snake) =}); state food_on_grid: u32(0); // 1 line of rust code reaction(shutdown) {= // comment in Rust println!("New high score: {}", self.snake.len()); =} } ================================================ FILE: tests/data/liquid.liquid ================================================ {% comment %} 24 lines 19 code 1 comments 4 blanks {% endcomment %} {% paginate collection.products by 20 %}
    {% for product in collection.products %}

  • {{product.title}}

    {{ product.description | strip_html | truncatewords: 35 }}

    {{ product.price_min | money }}{% if product.price_varies %} - {{ product.price_max | money }}{% endif %}

    • {% endfor %}
{% endpaginate %} ================================================ FILE: tests/data/livescript.ls ================================================ # 28 lines, 10 code, 12 comments, 6 blanks /* * /* Nested comment * # single line comment * */ /* add = (a, b) -> return a + b */ hello = -> console.log 'hello, world!' "hello!" |> capitalize |> console.log # Easy listing of implicit objects table1 = * id: 1 name: 'george' * id: 2 name: 'mike' # comment * id: 3 name: 'donald' # Comment ================================================ FILE: tests/data/llvm.ll ================================================ ; 21 lines 17 code 1 comments 3 blanks define i32 @add1(i32 %a, i32 %b) { entry: %tmp1 = add i32 %a, %b ret i32 %tmp1 } define i32 @add2(i32 %a, i32 %b) { entry: %tmp1 = icmp eq i32 %a, 0 br i1 %tmp1, label %done, label %recurse recurse: %tmp2 = sub i32 %a, 1 %tmp3 = add i32 %b, 1 %tmp4 = call i32 @add2(i32 %tmp2, i32 %tmp3) ret i32 %tmp4 done: ret i32 %b } ================================================ FILE: tests/data/logtalk.lgt ================================================ /* Test file for the Logtalk programming language (copied by the author from a Logtalk distribution example) 65 lines 27 code 18 comments 20 blanks */ % Alf believes he is the only survivor of his species; no point in % defining a class if there is only going to be a single instance: % a prototype, which is also a stand-alone object :- object(alf). % prototypes declare predicates for themselves (and derived prototypes) :- public([ name/1, planet/1, stomachs/1, favorite_food/1, chases/1, motto/1 ]). name('Gordon Shumway'). planet('Melmac'). stomachs(8). favorite_food(cats). chases('Lucky'). motto('Are you going to finish that sandwich?'). :- end_object. % later on, Alf finds out that his best friend, Skip, and his % girlfriend, Rhonda, also survived Melmac's explosion; as they % are all melmacians, they share most attributes (and add some % of their own): % "skip", a derived prototype from "alf", its parent prototype :- object(skip, extends(alf)). :- public(best_friend/1). best_friend(alf). name('Skip'). % still longing for a nice cat to eat since Melmac exploded chases(_) :- fail. :- end_object. % "rhonda" is also a prototype derived from "alf" :- object(rhonda, extends(alf)). :- public(boyfriend/1). boyfriend(alf). name('Rhonda'). % still longing for a nice cat to eat since Melmac exploded chases(_) :- fail. :- end_object. ================================================ FILE: tests/data/lolcode.lol ================================================ BTW 26 lines 11 code 9 comments 6 blanks HAI 1.3 BTW TEST! I HAS A MSG ITZ "BYE! OBTW" HOW IZ I PRINT_HELLO I HAS A MSG ITZ "BTW Hello, World!" BTW OBTW BTW MORE COMMENTS! VISIBLE MSG BTW TLDR IF U SAY SO I HAS A MSG ITZ "Hello OBTW BTW TLDR" I IZ PRINT_HELLO MKAY OBTW a longer test asd TLDR OBTW TLDR I IZ PRINT_HELLO MKAY OBTW should be valid foo bar TLDR VISIBLE MSG KTHXBYE ================================================ FILE: tests/data/m1.m1 ================================================ # 280 lines 197 code 47 comments 36 blanks ## Copyright (C) 2016 Jeremiah Orians ## This file is part of stage0. ## ## stage0 is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 3 of the License, or ## (at your option) any later version. ## ## stage0 is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with stage0. If not, see . # M2-Planet standards DEFINE NULL 00000000 # Registers DEFINE R0 0 DEFINE R1 1 DEFINE R2 2 DEFINE R3 3 DEFINE R4 4 DEFINE R5 5 DEFINE R6 6 DEFINE R7 7 DEFINE R8 8 DEFINE R9 9 DEFINE R10 A DEFINE R11 B DEFINE R12 C DEFINE BP C DEFINE R13 D DEFINE SP D DEFINE R14 E DEFINE LR E DEFINE R15 F DEFINE PC F # Register masks for push/pop16 DEFINE {R0} 0100 DEFINE {R1} 0200 DEFINE {R2} 0400 DEFINE {R3} 0800 DEFINE {R4} 1000 DEFINE {R8} 0001 DEFINE {R9} 0002 DEFINE {R10} 0004 DEFINE {R11} 0008 DEFINE {BP} 0010 DEFINE {LR} 0040 # Bitshift constants DEFINE NO_SHIFT 0 DEFINE LEFT 1 DEFINE RIGHT 3 DEFINE ARITH_RIGHT 5 # LOAD/STORE DEFINE HALF_MEMORY E1 DEFINE MEMORY E5 DEFINE NO_OFFSET B0 DEFINE STORE32 08 DEFINE STORE16 0C DEFINE STORE8 0C DEFINE LOAD32 09 DEFINE LOADU8 0 DEFINE LOADS8 D0 DEFINE LOADS16 F0 DEFINE LOAD 0D DEFINE LOADI8_ALWAYS 0A0E3 DEFINE LOADI8_G 0A0C3 DEFINE LOADI8_GE 0A0A3 DEFINE LOADI8_EQUAL 0A003 DEFINE LOADI8_NE 0A013 DEFINE LOADI8_LE 0A0D3 DEFINE LOADI8_L 0A0B3 DEFINE LOADI8_HI 0A083 DEFINE LOADI8_HS 0A023 DEFINE LOADI8_LS 0A093 DEFINE LOADI8_LO 0A033 # JUMP/BRANCH DEFINE JUMP_ALWAYS EA DEFINE JUMP_EQUAL 0A DEFINE JUMP_NE 1A DEFINE CALL_ALWAYS EB DEFINE CALL_REG_ALWAYS FF2FE1 DEFINE RETURN FF2FE1 # Data movement DEFINE MOVE_ALWAYS A0E1 DEFINE MVN_ALWAYS 0E0E1 DEFINE MVN_LT 0E0B1 DEFINE MVNI8_EQUAL 0E003 DEFINE PUSH_ALWAYS 2DE9 DEFINE POP_ALWAYS BDE8 # Arithmetic/logic DEFINE AUX_ALWAYS E1 DEFINE IMM_ALWAYS E3 DEFINE ARITH_ALWAYS E2 DEFINE ARITH_GE A2 DEFINE ARITH_LT B2 DEFINE ARITH_NE 12 DEFINE ARITH2_ALWAYS E0 DEFINE ARITH2_GE A0 DEFINE ADC 0A DEFINE ADCS 0B DEFINE ADD 08 DEFINE ADDS 09 DEFINE AND 00 DEFINE CMP 005 DEFINE CMPI8 005 DEFINE MUL 0 DEFINE MULS 1 DEFINE OR 08 DEFINE SHIFT A0 DEFINE SUB 04 DEFINE RSUB 06 DEFINE XOR 02 # SYSCALL DEFINE SYSCALL_ALWAYS 000000EF ## Copyright (C) 2016 Jeremiah Orians ## This file is part of M2-Planet. ## ## M2-Planet is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 3 of the License, or ## (at your option) any later version. ## ## M2-Planet is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with M2-Planet. If not, see . :_start '0' SP BP NO_SHIFT MOVE_ALWAYS ; Setup Base Pointer ;; Prepare argv !4 R0 ADD BP ARITH_ALWAYS ; ARGV_address = BP + 4 {R0} PUSH_ALWAYS ; Put argv on the stack ;; Prepare envp '0' BP R0 NO_SHIFT MOVE_ALWAYS ; Address we need to load from !0 R0 LOAD32 R0 MEMORY ; Get ARGC !2 R0 ADD R0 ARITH_ALWAYS ; OFFSET = ARGC + 2 '0' R0 R0 '1' MOVE_ALWAYS ; OFFSET = OFFSET * WORDSIZE '0' R0 R0 ADD BP ARITH2_ALWAYS ; ENVP_address = BP + OFFSET {R0} PUSH_ALWAYS ; Put envp on the stack ;; Stack offset !4 BP ADD BP ARITH_ALWAYS ; Fix BP ^~FUNCTION___init_malloc CALL_ALWAYS ; Setup for malloc ^~FUNCTION___init_io CALL_ALWAYS ; Setup for FILE* ^~FUNCTION_main CALL_ALWAYS ; Jump right into main {R1} POP_ALWAYS ; Fix stack {R1} POP_ALWAYS ; Fix stack {R1} POP_ALWAYS ; Fix stack {R0} PUSH_ALWAYS ; put return on the stack {R0} PUSH_ALWAYS ; So that _exit will have it {R0} PUSH_ALWAYS ; So that _exit will have it :FUNCTION_exit ^~FUNCTION___kill_io CALL_ALWAYS :FUNCTION__exit !4 R0 SUB R12 ARITH_ALWAYS !0 R0 LOAD32 R0 MEMORY !1 R7 LOADI8_ALWAYS SYSCALL_ALWAYS ; exit # Unsigned Divide :divide {R4} PUSH_ALWAYS ; Protect R4 {R3} PUSH_ALWAYS ; Protect R3 {R2} PUSH_ALWAYS ; Protect R2 '0' R0 R3 NO_SHIFT MOVE_ALWAYS ; MOV R3,R0 '0' R1 R2 NO_SHIFT MOVE_ALWAYS ; MOV R2,R1 !0 R0 LOADI8_ALWAYS ; MOV R0,#0 !0 CMPI8 R2 IMM_ALWAYS ; CMP R2,#0 !1 R0 SUB R0 ARITH_LT ; SUBLT R0,R0,#1 !0 CMPI8 R3 IMM_ALWAYS ; CMP R3,#0 !0 R3 RSUB R3 ARITH_LT ; RSBLT R3,R3,#0 '0' R0 R0 MVN_LT ; MVNLT R0,R0 '0' R0 R4 NO_SHIFT MOVE_ALWAYS ; MOV R4,R0 !32 R0 LOADI8_ALWAYS ; MOV R0,#32. !0 R1 LOADI8_ALWAYS ; MOV R1,#0 :divide_loop '0' R2 R2 ADDS R2 ARITH2_ALWAYS ; ADDS R2,R2,R2 '0' R1 R1 ADCS R1 ARITH2_ALWAYS ; ADCS R1,R1,R1 '0' R3 CMP R1 AUX_ALWAYS ; CMP R1,R3 '0' R3 R1 SUB R1 ARITH2_GE ; SUBGE R1,R1,R3 !1 R2 ADD R2 ARITH_GE ; ADDGE R2,R2,#1 !1 R0 SUB R0 ARITH_ALWAYS ; SUB R0,R0,#1 !0 CMPI8 R0 IMM_ALWAYS ; CMP R0,#0 ^~divide_loop JUMP_NE ; BNE loop '0' R2 R0 NO_SHIFT MOVE_ALWAYS ; MOV R0,R2 {R2} POP_ALWAYS ; Restore R2 {R3} POP_ALWAYS ; Restore R3 {R4} POP_ALWAYS ; Restore R4 '1' LR RETURN # Signed Divide :divides {R4} PUSH_ALWAYS ; Protect R4 {R3} PUSH_ALWAYS ; Protect R3 {R2} PUSH_ALWAYS ; Protect R2 '0' R0 R3 NO_SHIFT MOVE_ALWAYS ; MOV R3,R0 '0' R1 R2 NO_SHIFT MOVE_ALWAYS ; MOV R2,R1 !0 R0 LOADI8_ALWAYS ; MOV R0,#0 !0 CMPI8 R2 IMM_ALWAYS ; CMP R2,#0 !0 R2 RSUB R2 ARITH_LT ; RSBLT R2,R2,#0 !1 R0 SUB R0 ARITH_LT ; SUBLT R0,R0,#1 !0 CMPI8 R3 IMM_ALWAYS ; CMP R3,#0 !0 R3 RSUB R3 ARITH_LT ; RSBLT R3,R3,#0 '0' R0 R0 MVN_LT ; MVNLT R0,R0 '0' R0 R4 NO_SHIFT MOVE_ALWAYS ; MOV R4,R0 !32 R0 LOADI8_ALWAYS ; MOV R0,#32. !0 R1 LOADI8_ALWAYS ; MOV R1,#0 :divides_loop '0' R2 R2 ADDS R2 ARITH2_ALWAYS ; ADDS R2,R2,R2 '0' R1 R1 ADCS R1 ARITH2_ALWAYS ; ADCS R1,R1,R1 '0' R3 CMP R1 AUX_ALWAYS ; CMP R1,R3 '0' R3 R1 SUB R1 ARITH2_GE ; SUBGE R1,R1,R3 !1 R2 ADD R2 ARITH_GE ; ADDGE R2,R2,#1 !1 R0 SUB R0 ARITH_ALWAYS ; SUB R0,R0,#1 !0 CMPI8 R0 IMM_ALWAYS ; CMP R0,#0 ^~divides_loop JUMP_NE ; BNE loop !0 CMPI8 R4 IMM_ALWAYS ; CMP R4,#0 !0 R2 RSUB R2 ARITH_NE ; RSBNE R2,R2,#0 '0' R2 R0 NO_SHIFT MOVE_ALWAYS ; MOV R0,R2 {R2} POP_ALWAYS ; Restore R2 {R3} POP_ALWAYS ; Restore R3 {R4} POP_ALWAYS ; Restore R4 '1' LR RETURN # Unsigned Modulus :modulus {LR} PUSH_ALWAYS ; Prepare to leverage divide ^~divide CALL_ALWAYS ; Use divide '0' R1 R0 NO_SHIFT MOVE_ALWAYS ; MOV R0,R1 {LR} POP_ALWAYS ; Prepare for return '1' LR RETURN # Signed Modulus :moduluss {LR} PUSH_ALWAYS ; Prepare to leverage divide ^~divides CALL_ALWAYS ; Use divides '0' R1 R0 NO_SHIFT MOVE_ALWAYS ; MOV R0,R1 {LR} POP_ALWAYS ; Prepare for return '1' LR RETURN :GLOBAL__envp NULL :mystring "this is my string\n" ================================================ FILE: tests/data/m4.m4 ================================================ dnl 7 lines 3 code 1 blanks 3 comments The builtin `dnl' stands for “Discard to Next Line”: dnl this line is not emitted Other text is emitted You can also make comments with `#' # this is a comment # This is a comment, too ================================================ FILE: tests/data/menhir.mly ================================================ // 47 lines 31 code 7 comments 9 blanks (* Example from the menhir development with instrumented comments. * (* Note: nested C style comments are not allowed. *) * https://gitlab.inria.fr/fpottier/menhir/-/tree/master/demos/calc-alias *) %token INT "42" %token PLUS "+" %token MINUS "-" %token TIMES "*" %token DIV "/" %token LPAREN "(" %token RPAREN ")" %token EOL (* Token aliases can be used throughout the rest of the grammar. E.g., they can be used in precedence declarations: *) %left "+" "-" /* lowest " precedence */ %left "*" "/" /* medium precedence */ %nonassoc UMINUS // highest "precedence" %start main %% main: | e = expr EOL { e } (* Token aliases can also be used inside rules: *) expr: | i = "42" { i } | "(" e = expr ")" { e } | e1 = expr "+" e2 = expr { e1 + e2 } | e1 = expr "-" e2 = expr { e1 - e2 } | e1 = expr "*" e2 = expr { e1 * e2 } | e1 = expr "/" e2 = expr { e1 / e2 } | "-" e = expr %prec UMINUS { - e } ================================================ FILE: tests/data/meson.build ================================================ # 12 lines 6 code 2 comments 4 blanks project('xyz', 'c', meson_version : '>=0.30.0') # not counted xyz_gen = ''' # comment inside print("This is generated source.") ''' # this is a comment ================================================ FILE: tests/data/metal.metal ================================================ /* 32 lines 21 code 5 comments 6 blanks */ #include // comment struct Uniforms { float2 extent; }; struct VertexIn { float2 position [[attribute(0)]]; }; struct VertexOut { float2 position [[position]]; }; /* multi-line comment */ vertex VertexOut vs_main( VertexIn in [[stage_in]] ) { VertexOut out; return out; } fragment float4 fs_main( VertexOut in [[stage_in]] ) { return float4(0.0); } ================================================ FILE: tests/data/mlatu.mlt ================================================ // 22 lines 14 code 3 comments 5 blanks define divisible (Int, Int -> Bool +Fail) { (%) 0 (=) } // Here's a random comment that is definitely useful define fizzbuzz (Int -> String) { -> n; do (with (+Fail)) { n 5 divisible n 3 divisible } if { if { "FizzBuzz" } else { "Fizz" } } else { if { "Buzz" } else { n show } } } define fizzbuzzes (Int, Int -> +IO) { -> c, m; c fizzbuzz println (c < m) if { (c + 1) m fizzbuzzes } else {} // We don't need anything here } 1 100 fizzbuzzes // Comment at end ================================================ FILE: tests/data/moduledef.def ================================================ ; 17 lines 9 code 6 comments 2 blanks ; ; Definition file of KERNEL32.dll ; Automatic generated by gendef ; written by Kai Tietz 2008 ; LIBRARY "KERNEL32.dll" EXPORTS "BaseThreadInitThunk;@4" InterlockedPushListSList@8 AcquireSRWLockExclusive@4 AcquireSRWLockShared@4 ActivateActCtx@8 AddAtomA@4 AddAtomW@4 ================================================ FILE: tests/data/mojo.mojo ================================================ # 21 lines 15 code 3 comments 3 blanks ''' This is a docstring. # It has multiple lines. This is the end of the docstring. ''' def main(): # Hello Mojo! string = "Hello Mojo!" # The following line prints the string "Hello Mojo!" print(string) """ This piece of code prints the numbers "9", "6", and "3". 'Here is a quote.' """ for x in range(9, 0, -3): print(x) ================================================ FILE: tests/data/monkeyc.mc ================================================ // 69 lines 41 code 18 comments 10 blanks // Slightly modified template from the "Garmin.monkey-c" VS Code extension. import Toybox.Application; import Toybox.Graphics; import Toybox.Lang; import Toybox.System; import Toybox.WatchUi; class WatchFaceView extends WatchUi.WatchFace { function initialize() { WatchFace.initialize(); } // Load your resources here function onLayout(dc as Dc) as Void { setLayout(Rez.Layouts.WatchFace(dc)); } /* Called when this View is brought to the foreground. Restore the state of this View and prepare it to be shown. This includes loading resources into memory. */ function onShow() as Void { } // Update the view function onUpdate(dc as Dc) as Void { // Get the current time and format it correctly var timeFormat = "$1$:$2$"; var clockTime = System.getClockTime(); var hours = clockTime.hour; if (!System.getDeviceSettings().is24Hour) { if (hours > 12) { hours = hours - 12; } } else { if (getApp().getProperty("UseMilitaryFormat")) { timeFormat = "$1$$2$"; hours = hours.format("%02d"); } } var timeString = Lang.format(timeFormat, [hours, clockTime.min.format("%02d")]); // Update the view var view = View.findDrawableById("TimeLabel") as Text; view.setColor(getApp().getProperty("ForegroundColor") as Number); view.setText(timeString); View.onUpdate(dc); // Call the parent onUpdate function to redraw the layout } /* Called when this View is removed from the screen. Save the state of this View here. This includes freeing resources from memory. */ function onHide() as Void { } // The user has just looked at their watch. Timers and animations may be started here. function onExitSleep() as Void { } // Terminate any active timers and prepare for slow updates. function onEnterSleep() as Void { } } ================================================ FILE: tests/data/nextflow.nf ================================================ /* 18 lines 10 code 5 comments 3 blanks */ /* Nextflow - hello */ // comment cheers = Channel.from 'Bonjour', 'Ciao', 'Hello', 'Hola' process sayHello { echo true input: val x from cheers script: """ echo '$x world!' """ } ================================================ FILE: tests/data/nqp.nqp ================================================ # 24 lines 14 code 8 comments 2 blanks =begin Regex methods and functions =end =begin item match Match C<$text> against C<$regex>. If the C<$global> flag is given, then return an array of all non-overlapping matches. =end item sub match ($text, $regex, :$global?) { my $match := $text ~~ $regex; if $global { my @matches; while $match { nqp::push(@matches, $match); $match := $match.parse($text, :rule($regex), :c($match.to)); } @matches; } else { $match; } } ================================================ FILE: tests/data/odin.odin ================================================ // 29 lines 17 code 7 comments 5 blanks import "core:fmt" /* * Calculates the next number in the Collatz sequence * * If `x` is divisible by two, the result is `x` divided by two * If `x` is not divisible by two, the result is `x` multiplied by three plus one */ collatz :: inline proc(x: int) -> int { if x & 1 == 0 do return x >> 1; else do return x * 3 + 1; } steps :: proc(x: int) -> int { count := 0; y := x; for y != 1 { y = collatz(y); count += 1; } return count; } main :: proc() { fmt.println(steps(42)); // 8 } ================================================ FILE: tests/data/open_policy_agent.rego ================================================ # 13 lines 8 code 3 comments 2 blanks package application.authz # Only owner can update the pet's information # Ownership information is provided as part of OPA's input default allow = false allow { input.method == "PUT" some petid input.path = ["pets", petid] input.user == input.owner } ================================================ FILE: tests/data/openscad.scad ================================================ //! 34 lines 15 code 16 comments 3 blanks // https://en.wikibooks.org/wiki/OpenSCAD_User_Manual/Commented_Example_Projects // The idea is to twist a translated circle: // - /* linear_extrude(height = 10, twist = 360, scale = 0) translate([1,0]) circle(r = 1); */ module horn(height = 10, radius = 6, twist = 720, $fn = 50) { // A centered circle translated by 1xR and // twisted by 360° degrees, covers a 2x(2xR) space. // - radius = radius/4; // De-translate. // - translate([-radius,0]) // The actual code. // - linear_extrude(height = height, twist = twist, scale=0, $fn = $fn) translate([radius,0]) circle(r=radius); } translate([3,0]) mirror() horn(); translate([-3,0]) horn(); ================================================ FILE: tests/data/opentype.fea ================================================ # 54 lines 24 code 24 comments 6 blanks languagesystem DFLT dflt; languagesystem latn dflt; languagesystem latn DEU; languagesystem latn TRK; languagesystem cyrl dflt; feature smcp { sub [a-z] by [A.sc-Z.sc]; # Since all the rules in this feature are of the same type, they will be grouped in a single lookup. # Since no script or language keyword has been specified yet, # the lookup will be registered for this feature under all the language systems. } smcp; feature liga { sub f f by f_f; sub f i by f_i; sub f l by f_l; # Since all the rules in this feature are of the same type, they will be # grouped in a single lookup. # Since no script or language keyword has been specified yet, # the lookup will be registered for this feature under all the language systems. script latn; language dflt; # lookupflag 0; (implicit) sub c t by c_t; sub c s by c_s; # The rules above will be placed in a lookup that is registered for all # the specified languages for the script latn, but not any other scripts. language DEU; # script latn; (stays the same) # lookupflag 0; (stays the same) sub c h by c_h; sub c k by c_k; # The rules above will be placed in a lookup that is registered only # under the script 'latn', 'language DEU'. language TRK; # This will inherit both the top level default rules - the rules defined # before the first 'script' statement, and the script-level default # rules for 'latn: all the lookups of this feature defined after the # 'script latn' statement, and before the 'language DEU' statement. # If 'TRK' were not named here, it would not inherit the default rules # for the script 'latn'. } liga; feature kern { pos a y -150; # [more pos statements] # All the rules in this feature will be grouped in a single lookup # that is registered under all the languagesystems. } kern; ================================================ FILE: tests/data/org_mode.org ================================================ # 13 lines 7 code 2 comments 4 blanks #+TITLE: This is the title, not a comment # This is comment Some text * Heading 1 :PROPERTIES: :CUSTOM_ID: heading-1 :END: Text under heading 1 ================================================ FILE: tests/data/pan.pan ================================================ # 21 lines 11 code 4 comments 6 blanks # Pan example code, see https://quattor-pan.readthedocs.io/en/stable/pan-book/index.html prefix "/system/aii/osinstall/ks"; "clearpart" = append("vdb"); "ignoredisk" = list(); # no disks to ignore prefix "/system/blockdevices"; "physical_devs/vdb/label" = "msdos"; "partitions/vdb1" = dict( "holding_dev", "vdb", ); "files/{/srv/elasticsearch}" = dict('size', 0); # To facilitate adding other partitions at a later stage, a # logical volume will be created "volume_groups/vg1/device_list" = append("partitions/vdb1"); "logical_volumes" = lvs_add('vg1', dict("elasticsearch", -1)); ================================================ FILE: tests/data/pcss.pcss ================================================ /* 14 lines 6 code 5 comments 3 blanks */ .foo { color: #f00; &.bar { background: url("foobar.jpg"); } } // inline comments are allowed by some PostCSS syntaxes /* * block comments are standard */ ================================================ FILE: tests/data/pest.pest ================================================ // 9 lines 4 code 3 comments 2 blanks alpha = { 'a'..'z' | 'A'..'Z' } digit = { '0'..'9' } ident = { (alpha | digit)+ } ident_list = _{ !digit ~ ident ~ (" " ~ ident)+ } // ^ // ident_list rule is silent which means it produces no tokens ================================================ FILE: tests/data/phix.e ================================================ /* 40 lines 25 code 8 comments 7 blanks */ -- copied from cpp, not necessarily idiomatic Euphoria code include std/sequence.e -- bubble_sort_function public function bubble_sort(sequence a) integer t = 0 integer j = length(a) integer s = 1 while s > 0 do s = 0 integer i = 2 while i <= j do if a[i] < a[i - 1] then t = a[i] a[i] = a[i - 1] a[i - 1] = t s = 1 end if i += 1 end while j -= 1 end while return a end function sequence a = {4, 65, 2, -31, 0, 99, 2, 83, 782, 1} -- Single line comment ? {"Before:", a} a = bubble_sort(a) /* multi * line * comment */ ? {"After:", a, equal(a, {-31,0,1,2,2,4,65,83,99,782})} ================================================ FILE: tests/data/plantuml.puml ================================================ ' 35 lines 10 code 13 comments 12 blanks ' plantuml line comments must start at the beginning of a line. ' plantuml block comments must either start on a newline or start and end on the same line as they start. ' strings cannot span multiple lines. ' single quotes are a valid string wrapper '', but not when they are the first non-whitespace characters on a line ' comment at start uml @startuml !include '' this is also a comment rectangle "this is a string" as r /' this is a multi-line comment '/ /' this is also a multi-line comment '/ Container(C, "This is some more text", "text") node n /' this is a multi-line comment at the end of a line '/ /' this is /' a valid '/ block comment '/ component "'this is not a comment" /' this is a multi-line comment at the start of a line '/ interface i boundary "/' this is not a multi-line comment '/" System(s, "this is /' not part of a comment", "'/ this is also not part of a comment", "/' '/ /' neither is this") ' comment after start uml @enduml ================================================ FILE: tests/data/pofile.po ================================================ # 14 lines 6 code 5 comments 3 blanks #: lib/error.c:116 msgid "Unknown system error" msgstr "Error desconegut del sistema" #: disk-utils/addpart.c:15 #, c-format msgid " %s \n" msgstr " %s \n" #: disk-utils/addpart.c:19 msgid "Tell the kernel about the existence of a specified partition.\n" msgstr "Informer le noyau de l’existence d’une partition indiquée.\n" ================================================ FILE: tests/data/pofile_pot.pot ================================================ # 17 lines 8 code 5 comments 4 blanks #: disk-utils/addpart.c:60 disk-utils/delpart.c:61 disk-utils/resizepart.c:101 msgid "invalid partition number argument" msgstr "" #: disk-utils/addpart.c:61 msgid "invalid start argument" msgstr "" #: disk-utils/addpart.c:62 disk-utils/resizepart.c:111 msgid "invalid length argument" msgstr "" #: disk-utils/addpart.c:63 msgid "failed to add partition" msgstr "" ================================================ FILE: tests/data/poke.pk ================================================ /* 4 lines 2 code 1 comments 1 blanks */ var N = 3; fun getoff = offset,B>: { return 2#B; } ================================================ FILE: tests/data/pony.pony ================================================ // 12 lines 7 code 3 comments 2 blanks /* com- -ment */ actor Main """ Some Docs """ new create(env: Env) => env.out.print("Hello, world.") ================================================ FILE: tests/data/postcss.sss ================================================ // 27 lines 18 code 4 comments 5 blanks /** multi-line */ div { width: calc(99.9% * 1/3 - (30px - 30px * 1/3)); } div:nth-child(1n) { float: left; margin-right: 30px; clear: none; } div:last-child { margin-right: 0; } div:nth-child(3n) { margin-right: 0; float: right; } div:nth-child(3n + 1) { clear: both; } ================================================ FILE: tests/data/powershell.ps1 ================================================ # 17 lines 9 code 4 comments 4 blanks <# Test #> 'a' + "b" Write-Host @" Name: $name Address: $address "@ $template = @' Name: {0} Address: {0} '@ ================================================ FILE: tests/data/pug.pug ================================================ //- 13 lines, 8 code, 3 comments, 2 blanks doctype html // this comment will be translated to an HTML comment //- this comment will be excluded from the generated HTML html head title Hello, World! body p | Hello, | World! ================================================ FILE: tests/data/puppet.pp ================================================ # 18 lines 14 code 3 comments 1 blanks class example::class( $param1, $param2=2, $param3=undef, # pass this one ) { # comments are really simple some::resource { 'bar': param1 => param2, # comments here too param3 => param4; } some::other::resource { 'baz': } } ================================================ FILE: tests/data/pyret.arr ================================================ # 22 lines 9 code 8 comments 5 blanks fun single-quote(): doc: "this is a documentation string" 'foo' end #| Hello, this is a multiline message |# # This is a line message fun double-quotes(): "bar" end nested = #| doesn't start yet or yet |# "nested" ================================================ FILE: tests/data/python.py ================================================ # 15 lines, 10 code, 2 comments, 3 blanks def add(x, y): """ Hello World # Real Second line Second line """ string = "Hello World #\ " y += len(string) # Add the two numbers. x + y ================================================ FILE: tests/data/q.q ================================================ // 14 lines 5 code 5 comments 4 blanks /calc nav for sets of portfolios,ETFs,indices,.. /one day of ([]time;sym;price) sorted by time n:10000000;S:-10000?`4 t:([]time:09:30:00.0+til n;sym:n?S;price:n?1.0) /calc price deltas once \t update deltas price by sym from`t /for each portfolio a:([sym:-100?S]weight:100?1.0) \t r:select time,sums price*weight from t ij a ================================================ FILE: tests/data/qml.qml ================================================ // 20 lines 11 code 5 comments 4 blanks import QtQuick 2.7 import QtQuick.Controls 2.0 ApplicationWindow { visible: true /* * Multiline comment */ Text { text: "string type 1" } // comment function testfunc() { console.log('string type 2'); } } ================================================ FILE: tests/data/racket.rkt ================================================ ;;; 40 lines 15 code 14 comments 11 blanks #lang racket ; defines the language we are using ;;; Comments ;; Single line comments start with a semicolon #| Block comments can span multiple lines and... #| they can be nested! |# |# ;; S-expression comments discard the following expression ;; since this is syntax-aware, tokei counts this as code #; (this expression is discarded) ;; Constant (define %pi 3.14159265358979323846) #| This is a block comment |# (define (degrees->radians deg) (* deg (/ %pi 180))) ;; Function (define (sq x) (* x x)) (define (sum xs) "Sum list of elements." (foldl + 0 xs)) ; comment (define (sum-upto n) (/ (* n (+ 1 n)) 2)) (define (test-sums n) (= (sum-upto n) (sum (range (+ 1 n))))) (test-sums 100) ================================================ FILE: tests/data/raku.raku ================================================ # 49 lines 37 code 6 comments 6 blanks =begin pod =begin DESCRIPTION =head1 Test file for Tokei =end DESCRIPTION =begin code :lang say 'Hello World'; =end code =end pod #| Fibonacci with multiple dispatch multi sub fib (0 --> 0) {} multi sub fib (1 --> 1) {} multi sub fib (\n where * > 1) { fib(n - 1) + fib(n - 2) } #|{ Role shape for printing area of different shapes } role Shape { method area { ... } method print_area { say "Area of {self.^name} is {self.area}."; } } class Rectangle does Shape { has $.width is required; #= Width of rectangle has $.height is required; #= Height of rectangle method area { #`( area of rectangle: width times height ) $!width × $!height } } ================================================ FILE: tests/data/razor.cshtml ================================================ @* 55 lines 35 code 15 comments 5 blanks *@ @page "/" @using Microsoft.AspNetCore.Components.Web @namespace temp.Pages @addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers @{ // foo string foo = "bar"; /* * bar */ string bar = "foo"; } @* multi-line comment *@
An error has occurred. This application may no longer respond until reloaded. An unhandled exception has occurred. See browser dev tools for details. Reload 🗙
================================================ FILE: tests/data/razorcomponent.razor ================================================ @* 45 lines 16 code 21 comments 8 blanks *@ @page "/counter" @{ // foo string foo = "bar"; /* * bar */ string bar = "foo"; } Counter @* multi-line comment *@

Counter

Current count: @currentCount

@code { /* C# style multi-line comment */ private int currentCount = 0; private void IncrementCount() { // increment the count currentCount++; } } ================================================ FILE: tests/data/redscript.reds ================================================ // 75 lines 47 code 20 comments 8 blanks // redscript allows line comments /* as well as block comments */ // it supports global functions func add2(x: Int32, y: Int32) -> Int32 { return x + y; } // functions without a type annotation default to Void return type func tutorial() { let x: Int32 = 10; // compiler can infer types for local variables, y will be Int32 let y = 20; // it supports arithmetic let sum = x + y + 13; // as well as mutation let mutable = 0; mutable += 10; // numbers with decimal points default to type Float let num = 10.0; // you can cast between some types let uint: Uint8 = Cast(10); // array literals let arr = [1, 2, 3]; // array iteration for item in arr { // logging and string operations Log("at " + ToString(item)); } } // you can define your own classes public class IntTuple { let fst: Int32; let snd: Int32; // you can define static member functions public static func Create(fst: Int32, snd: Int32) -> ref { let tuple = new IntTuple(); tuple.fst = fst; tuple.snd = snd; return tuple; } public func Swap() { let tmp = this.fst; this.fst = this.snd; this.snd = tmp; } } // you can replace existing in-game methods by specifying the class they belong to @replaceMethod(CraftingSystem) private final func ProcessCraftSkill(xpAmount: Int32, craftedItem: StatsObjectID) { // instantiate a class using the new operator let xpEvent = new ExperiencePointsEvent(); xpEvent.amount = xpAmount * 100; xpEvent.type = gamedataProficiencyType.Crafting; GetPlayer(this.GetGameInstance()).QueueEvent(xpEvent); } // you can add new methods to existing classes as well // they are visible to other code using the class @addMethod(BackpackMainGameController) private final func DisassembleAllJunkItems() -> Void { let items = this.m_InventoryManager.GetPlayerItemsByType(gamedataItemType.Gen_Junk); let i = 0; for item in items { ItemActionsHelper.DisassembleItem(this.m_player, InventoryItemData.GetID(item)); }; // some methods require CName literals, they need to be prefixed with the n letter this.PlaySound(n"Item", n"OnBuy"); } ================================================ FILE: tests/data/renpy.rpy ================================================ # 32 lines 8 code 9 comments 15 blanks # Declare characters used by this game. The color argument colorizes the # name of the character. define e = Character("Eileen") # The game starts here. label start: # Show a background. This uses a placeholder by default, but you can # add a file (named either "bg room.png" or "bg room.jpg") to the # images directory to show it. scene bg room show eileen happy # These display lines of dialogue. e "You've created a new Ren'Py game." e 'Once you add a story, pictures, and music, you can release it to the world!' e `Testing, testing` # This ends the game. return ================================================ FILE: tests/data/roc.roc ================================================ # 36 lines 18 code 10 comments 8 blanks module [square] # this is a comment # this is another comment a1 = 1 a2 = 3.14159 # pi expect # simple check a1 == 1 expect a2 |> Num.toStr == "3.14159" ## Compute the square square = \x -> s = x * x # the line above is blank s expect square 3 == 9 ## """ ## this is not a multiline string, ## it's a doc comment ## """ multilineString = """ # this line is not a comment, it's actually code The line above is not blank, it's actually code """ expect multilineString |> Str.toUtf8 |> List.first == Ok '#' ================================================ FILE: tests/data/ron.ron ================================================ // 157 lines 137 code 7 comments 13 blanks #![enable(implicit_some)] Container( transform: ( id: "background", anchor: Middle, stretch: XY( x_margin: 0., y_margin: 0., keep_aspect_ratio: false), width: 20., height: 20., ), background: SolidColor(0.03, 0.03, 0.03, 1.0), children: [ Container( transform: ( id: "container_start", y: 180, width: 755., height: 170., anchor: Middle, ), background: SolidColor(1.0, 0.65, 0.0, 1.0), children: [ // Complex Button Button( transform: ( id: "start", width: 750., height: 165., tab_order: 1, anchor: Middle, mouse_reactive: true, ), button: ( text: "START GAME", font: File("font/square.ttf", ("TTF", ())), font_size: 75., normal_text_color: (1.0, 0.65, 0., 1.0), // ffa500 // hover_text_color: (1.0, 0.65, 0., 1.0), // press_text_color: (1.0, 0.65, 0., 1.0), normal_image: SolidColor(0., 0., 0., 1.), hover_image: SolidColor(0.1, 0.1, 0.1, 1.), press_image: SolidColor(0.15, 0.15, 0.15, 1.), ) ), ] ), Container( transform: ( id: "container_load", y: 0, width: 755., height: 170., anchor: Middle, ), background: SolidColor(1.0, 0.65, 0.0, 1.0), children: [ // Complex Button Button( transform: ( id: "load", width: 750., height: 165., tab_order: 3, anchor: Middle, mouse_reactive: true, ), button: ( text: "LOAD GAME", font: File("font/square.ttf", ("TTF", ())), font_size: 75., normal_text_color: (1.0, 0.65, 0., 1.0), // ffa500 normal_image: SolidColor(0., 0., 0., 1.), hover_image: SolidColor(0.1, 0.1, 0.1, 1.), press_image: SolidColor(0.15, 0.15, 0.15, 1.), ) ), ] ), Container( transform: ( id: "container_options", y: -180, width: 755., height: 170., anchor: Middle, ), background: SolidColor(1.0, 0.65, 0.0, 1.0), children: [ // Complex Button Button( transform: ( id: "options", width: 750., height: 165., tab_order: 3, anchor: Middle, mouse_reactive: true, ), button: ( text: "OPTIONS", font: File("font/square.ttf", ("TTF", ())), font_size: 75., normal_text_color: (1.0, 0.65, 0., 1.0), // ffa500 normal_image: SolidColor(0., 0., 0., 1.), hover_image: SolidColor(0.1, 0.1, 0.1, 1.), press_image: SolidColor(0.15, 0.15, 0.15, 1.), ) ), ] ), Container( transform: ( id: "container_credits", y: -360, width: 755., height: 170., anchor: Middle, ), background: SolidColor(1.0, 0.65, 0.0, 1.0), children: [ // Complex Button Button( transform: ( id: "credits", width: 750., height: 165., tab_order: 3, anchor: Middle, mouse_reactive: true, ), button: ( text: "CREDITS", font: File("font/square.ttf", ("TTF", ())), font_size: 75., normal_text_color: (1.0, 0.65, 0., 1.0), // ffa500 normal_image: SolidColor(0., 0., 0., 1.), hover_image: SolidColor(0.1, 0.1, 0.1, 1.), press_image: SolidColor(0.15, 0.15, 0.15, 1.), ) ), ] ), ], ) ================================================ FILE: tests/data/rpmspec.spec ================================================ # 42 lines 22 code 4 comments 16 blanks Name: example Version: 0.0.1 Release: 1%{?dist} Summary: an example specfile Group: URL: Source0: # test comments for requirements BuildRequires: Requires: %description %prep %setup -q # build the project %build %configure make build # install the files here %install make install %clean %files %defattr(-,root,root,-) %doc %changelog ================================================ FILE: tests/data/ruby.rb ================================================ # 20 lines 9 code 8 comments 3 blanks x = 3 if x < 2 p = "Smaller" else p = "Bigger" end =begin Comments Comments Comments Comments =end # testing. while x > 2 and x < 10 x += 1 end ================================================ FILE: tests/data/ruby_env ================================================ #!/usr/bin/env ruby # 11 lines 3 code 6 comments 2 blanks =begin Comment that only counts if detected as ruby Comments =end while x > 2 and x < 10 x += 1 end ================================================ FILE: tests/data/ruby_html.erb ================================================ <%= title %>

<%= header %>

<%= text %>

<%= template "footer" %> ================================================ FILE: tests/data/rust.rs ================================================ //! 48 lines 36 code 6 comments 6 blanks //! ```rust //! fn main () { //! // Comment //! //! println!("Hello World!"); //! } //! ``` /* /**/ */ fn main() { let start = r##"/*##\" \"##; // comment loop { if x.len() >= 2 && x[0] == '*' && x[1] == '/' { // found the */ break; } } } fn foo<'a, 'b>(name: &'b str) { let this_ends = "a \"test/*."; call1(); call2(); let this_does_not = /* a /* nested */ comment " */ "*/another /*test call3(); */"; } fn foobar() { let does_not_start = // " "until here, test/* test"; // a quote: " let also_doesnt_start = /* " */ "until here, test,*/ test"; // another quote: " } fn foo() { let a = 4; // /* let b = 5; let c = 6; // */ } ================================================ FILE: tests/data/scheme.scm ================================================ ;;; 26 lines 14 code 4 comments 8 blanks (import (srfi srfi-1)) ; for reduce ;; Constant (define %pi 3.14159265358979323846) #| This is a block comment |# (define (degrees->radians deg) (* deg (/ %pi 180))) ;; Function (define (sq x) (* x x)) (define (sum xs) "Sum list of elements." (reduce + 0 xs)) ; comment (define (sum-upto n) (/ (* n (1+ n)) 2)) (define (test-sums n) (= (sum-upto n) (sum (iota (1+ n))))) (test-sums 100) ================================================ FILE: tests/data/shaderlab.shader ================================================ // 43 lines 31 code 8 comments 4 blanks Shader "Custom/Sample shader" { Properties { _MainTex ("Texture", 2D) = "white" {} } SubShader { Tags { "Queue"="Transparent" "RenderType"="Transparent" } // blending Blend SrcAlpha OneMinusSrcAlpha /* multi-line comment */ Pass { CGPROGRAM #pragma vertex vert struct appdata { float4 vertex : POSITION; float2 uv : TEXCOORD0; }; sampler2D _MainTex; // vertex v2f vert (appdata v) { v2f o; o.vertex = UnityObjectToClipPos(v.vertex); o.uv = TRANSFORM_TEX(v.uv, _MainTex); return o; } ENDCG } } } ================================================ FILE: tests/data/slang.slang ================================================ // 15 lines 8 code 5 comments 2 blanks Texture2D in_tex; RWTexture2D out_tex; // Blit compute shader [shader("compute")] [numthreads(8, 8, 1)] void main(uint2 id: SV_DispatchThreadID) { /* Perform the blit */ out_tex[id] = in_tex[id]; return; } ================================================ FILE: tests/data/slint.slint ================================================ // 26 lines 21 code 2 comments 3 blanks component MyButton inherits Text { color: black; // ... } export component MyApp inherits Window { preferred-width: 200px; preferred-height: 100px; Rectangle { width: 200px; height: 100px; background: green; }/* */ MyButton { x:0;y:0; // hello text: "hello"; } MyButton { // world y:0; x: 50px; text: "world"; } } ================================================ FILE: tests/data/solidity.sol ================================================ // 14 lines 6 code 7 comments 1 blanks pragma solidity >=0.4.22 <0.6.0; // Comment line contract Foo { /* Comment line Comment line Comment line */ function foo(address bar) public { require(bar != 0); } } ================================================ FILE: tests/data/sql.sql ================================================ -- 12 lines 4 code 5 comments 3 blanks SELECT * FROM Users WHERE FirstName is not null; -- select rows where the user has a first name /* this is the beginning of a block comment insert a new user into the Users table -- line comment in a block comment */ INSERT INTO Users (FirstName, LastName) VALUES ("John", "Does"); ================================================ FILE: tests/data/srecode.srt ================================================ ;; 37 lines 23 code 2 comments 12 blanks set escape_start "$" set escape_end "$" set mode "srecode-template-mode" set priority "70" set comment_start ";;" set comment_end "" set comment_prefix ";;" set SEPARATOR "----" set DOLLAR "$" context file prompt MAJORMODE "Major Mode for templates: " read srecode-read-major-mode-name prompt START "Escape Start Characters: " default "{{" prompt END "Escape End Characters: " default "}}" template empty :file :user :time :srt "Insert a skeleton for a template file." ---- $>:filecomment$ set mode "$?MAJORMODE$" set escape_start "$?START$" set escape_end "$?END$" context file $^$ ;; end ---- ================================================ FILE: tests/data/stan.stan ================================================ // 142 lines 123 code 17 comments 2 blanks // Example code from https://github.com/TheEconomist/us-potus-model/blob/85be55ae7b0bc68cb155a9ca975e155837eb4851/scripts/model/poll_model_2020.stan data{ int N_national_polls; // Number of polls int N_state_polls; // Number of polls int T; // Number of days int S; // Number of states (for which at least 1 poll is available) + 1 int P; // Number of pollsters int M; // Number of poll modes int Pop; // Number of poll populations int state[N_state_polls]; // State index int day_state[N_state_polls]; // Day index int day_national[N_national_polls]; // Day index int poll_state[N_state_polls]; // Pollster index int poll_national[N_national_polls]; // Pollster index int poll_mode_state[N_state_polls]; // Poll mode index int poll_mode_national[N_national_polls]; // Poll mode index int poll_pop_state[N_state_polls]; // Poll mode index int poll_pop_national[N_national_polls]; // Poll mode index int n_democrat_national[N_national_polls]; int n_two_share_national[N_national_polls]; int n_democrat_state[N_state_polls]; int n_two_share_state[N_state_polls]; vector[N_national_polls] unadjusted_national; vector[N_state_polls] unadjusted_state; // cov_matrix[S] ss_cov_mu_b_walk; // cov_matrix[S] ss_cov_mu_b_T; // cov_matrix[S] ss_cov_poll_bias; //*** prior input vector[S] mu_b_prior; vector[S] state_weights; real sigma_c; real sigma_m; real sigma_pop; real sigma_measure_noise_national; real sigma_measure_noise_state; real sigma_e_bias; // covariance matrix and scales cov_matrix[S] state_covariance_0; real random_walk_scale; real mu_b_T_scale; real polling_bias_scale; } transformed data { real national_cov_matrix_error_sd = sqrt(transpose(state_weights) * state_covariance_0 * state_weights); cholesky_factor_cov[S] cholesky_ss_cov_poll_bias; cholesky_factor_cov[S] cholesky_ss_cov_mu_b_T; cholesky_factor_cov[S] cholesky_ss_cov_mu_b_walk; // scale covariance matrix[S, S] ss_cov_poll_bias = state_covariance_0 * square(polling_bias_scale/national_cov_matrix_error_sd); matrix[S, S] ss_cov_mu_b_T = state_covariance_0 * square(mu_b_T_scale/national_cov_matrix_error_sd); matrix[S, S] ss_cov_mu_b_walk = state_covariance_0 * square(random_walk_scale/national_cov_matrix_error_sd); // transformation cholesky_ss_cov_poll_bias = cholesky_decompose(ss_cov_poll_bias); cholesky_ss_cov_mu_b_T = cholesky_decompose(ss_cov_mu_b_T); cholesky_ss_cov_mu_b_walk = cholesky_decompose(ss_cov_mu_b_walk); } parameters { vector[S] raw_mu_b_T; matrix[S, T] raw_mu_b; vector[P] raw_mu_c; vector[M] raw_mu_m; vector[Pop] raw_mu_pop; real mu_e_bias; real rho_e_bias; vector[T] raw_e_bias; vector[N_national_polls] raw_measure_noise_national; vector[N_state_polls] raw_measure_noise_state; vector[S] raw_polling_bias; real mu_b_T_model_estimation_error; } transformed parameters { //*** parameters matrix[S, T] mu_b; vector[P] mu_c; vector[M] mu_m; vector[Pop] mu_pop; vector[T] e_bias; vector[S] polling_bias = cholesky_ss_cov_poll_bias * raw_polling_bias; vector[T] national_mu_b_average; real national_polling_bias_average = transpose(polling_bias) * state_weights; real sigma_rho; //*** containers vector[N_state_polls] logit_pi_democrat_state; vector[N_national_polls] logit_pi_democrat_national; //*** construct parameters mu_b[:,T] = cholesky_ss_cov_mu_b_T * raw_mu_b_T + mu_b_prior; // * mu_b_T_model_estimation_error for (i in 1:(T-1)) mu_b[:, T - i] = cholesky_ss_cov_mu_b_walk * raw_mu_b[:, T - i] + mu_b[:, T + 1 - i]; national_mu_b_average = transpose(mu_b) * state_weights; mu_c = raw_mu_c * sigma_c; mu_m = raw_mu_m * sigma_m; mu_pop = raw_mu_pop * sigma_pop; e_bias[1] = raw_e_bias[1] * sigma_e_bias; sigma_rho = sqrt(1-square(rho_e_bias)) * sigma_e_bias; for (t in 2:T) e_bias[t] = mu_e_bias + rho_e_bias * (e_bias[t - 1] - mu_e_bias) + raw_e_bias[t] * sigma_rho; //*** fill pi_democrat for (i in 1:N_state_polls){ logit_pi_democrat_state[i] = mu_b[state[i], day_state[i]] + mu_c[poll_state[i]] + mu_m[poll_mode_state[i]] + mu_pop[poll_pop_state[i]] + unadjusted_state[i] * e_bias[day_state[i]] + raw_measure_noise_state[i] * sigma_measure_noise_state + polling_bias[state[i]]; } logit_pi_democrat_national = national_mu_b_average[day_national] + mu_c[poll_national] + mu_m[poll_mode_national] + mu_pop[poll_pop_national] + unadjusted_national .* e_bias[day_national] + raw_measure_noise_national * sigma_measure_noise_national + national_polling_bias_average; } model { //*** priors raw_mu_b_T ~ std_normal(); //mu_b_T_model_estimation_error ~ scaled_inv_chi_square(7, 1); to_vector(raw_mu_b) ~ std_normal(); raw_mu_c ~ std_normal(); raw_mu_m ~ std_normal(); raw_mu_pop ~ std_normal(); mu_e_bias ~ normal(0, 0.02); rho_e_bias ~ normal(0.7, 0.1); raw_e_bias ~ std_normal(); raw_measure_noise_national ~ std_normal(); raw_measure_noise_state ~ std_normal(); raw_polling_bias ~ std_normal(); //*** likelihood n_democrat_state ~ binomial_logit(n_two_share_state, logit_pi_democrat_state); n_democrat_national ~ binomial_logit(n_two_share_national, logit_pi_democrat_national); } generated quantities { matrix[T, S] predicted_score; for (s in 1:S){ //predicted_score[1:T, s] = inv_logit(mu_a[1:T] + to_vector(mu_b[s, 1:T])); predicted_score[1:T, s] = inv_logit(to_vector(mu_b[s, 1:T])); } } ================================================ FILE: tests/data/stata.do ================================================ * 16 lines 6 code 7 comments 3 blanks * This is a comment **** Any number of * symbol use "foo.dta", replace gen x = 1*2 gen x2 = 1/2 /* Here's a comment block */ if c(username) == "foobar" { global FOO 1 } // Finally another symbol for comment ================================================ FILE: tests/data/stratego.str ================================================ // 24 lines 12 code 6 comments 6 blanks module stratego strategies /** // */ main = !"/* " ; id // */ foo = ?'a' rules foobar: "//" -> "\\" // " ' /* " ' */ foo: a -> a // /* where b := 'b' // */ // ; c := $[quotes with anti quotes [b], which are not supported by tokei atm so this is a commented line of code] // ; c := ${quotes with anti quotes {b}, which are not supported by tokei atm so this is a commented line of code} // ; c := $, which are not supported by tokei atm so this is a commented line of code> ================================================ FILE: tests/data/stylus.styl ================================================ // 20 lines, 10 code, 5 comments, 5 blanks /* * Multi-line comment */ // Single-line comment #app position: absolute left: 0 top: 0 .item::before content: "Stylus" color: orange background-color: white .item::after content: 'Lang' ================================================ FILE: tests/data/svelte.svelte ================================================ ================================================ FILE: tests/data/swift.swift ================================================ // 24 lines 6 code 14 comments 4 blanks // Single-line comment /* multi-line comment */ /* /* nested */ */ /* multi line comment */ /* nested /* */ /* nested */ */ import UIKit class ViewController: UIViewController { override func viewDidLoad() { super.viewDidLoad() // Do any additional setup after loading the view. } } ================================================ FILE: tests/data/swig.i ================================================ /* 16 lines 8 code 5 comments 3 blanks */ %module mymodule /* * Wrapper-includes */ %{ #include "myheader.h" //dummy header %} // Now list ANSI C/C++ declarations int foo; int bar(int x); %rename(my_print) print; extern void print(const char *); ================================================ FILE: tests/data/tact.tact ================================================ // 20 lines 12 code 4 comments 4 blanks import "@stdlib/deploy"; // comment /* comment */ fun global() { let str: String = "\n \r \t \u1234 \xFF"; // comment while (true) { // comment if /* comment */ (true) { /* comment */ } } } // "quoted" struct St { /* " */ field1: Int; // /* field2: Int as uint128; field3: Int; // */ } ================================================ FILE: tests/data/templ.templ ================================================ // 24 lines, 13 code, 8 comments, 3 blanks package test templ Foo() {
} /* some css class. */ css button() { padding: 7px; border-radius: 5px; } // doSomething does something script doSomething() { alert("something") } ================================================ FILE: tests/data/thrift.thrift ================================================ // 38 lines 29 code 2 comments 7 blanks namespace java test namespace py test /* /* */ service Twitter extends core.BaseService { void ping(), bool postTweet(1: Tweet tweet) throws (1: TwitterUnavailable unavailable), TweetSearchResult searchTweets(1: string query), } enum TweetType { TWEET, # 1 /* RETWEET = 2, // 2 DM = 0xa, // 3 */ REPLY } struct Tweet { 1: required i32 userId, 2: required string userName = "/*", 3: required string text = '...', 4: optional Location loc, 5: optional TweetType tweetType = TweetType.TWEET, 16: optional string language = "en\"glish", // */ } const string TEST1 = // " "starts here, test/* test" // a quote: " const string TEST2 = /* " */ 'starts here, test,*/ test' # another quote: " ================================================ FILE: tests/data/tsx.tsx ================================================ // 9 lines, 5 code, 3 comments, 1 blanks /** string two numbers together */ const stringNums = (x: number, y: number) => { // the line below makes a string const firstNum = x + ""; const secondNum = firstNum + y; return secondNum; }; ================================================ FILE: tests/data/ttcn.ttcn3 ================================================ // 16 lines 7 code 6 comments 3 blanks /** * @description A TTCN-3 demo module * @author John Doe */ module demo { import from definitions all; control { log("Hello world!"); // write something to log // execute(altstepWithTimeout()); execute(test(), 5.0); /* terminate after 5s */ } } ================================================ FILE: tests/data/twig.twig ================================================ {# 16 lines 14 code 1 comments 1 blanks #}
    {% for product in products %}

  • {{- product.title -}}

    {{ product.description | capitalize }}

    {{ product.price_min | default('0') }}{% if product.price_varies %} - {{ product.price_max | number_format(2, '.', ',') }}{% endif %}

    • {% endfor %}
================================================ FILE: tests/data/typescript.ts ================================================ // 33 lines, 20 code, 10 comments, 3 blanks /* Multi-line comment with blanks * */ // Comment class Person { #age: number; #name: string; // end of line comment #height: number; constructor(age: number, name: string, height: number) { this.#age = age; this.#name = name; this.#height = height; } } let main = () => { // Comment with quote " let person = new Person( 5, `Phill the giant`, 7 ); }; main(); ================================================ FILE: tests/data/typst.typ ================================================ // 16 lines 9 code 3 comments 4 blanks // Some example settings #set document(title: "a title", author: "an author") #set page(numbering: "1 / 1", number-align: center) #set par(justify: true) #set text(size: 13pt, lang: "fr") // with a trailing comment #set heading(numbering: "1.1") /* with another trailing comment */ #let foo = "multiline string" #let bar = "singleline string" /* comment */ /* nested /* comment */ */ #lorem(50) ================================================ FILE: tests/data/uiua.ua ================================================ # 9 lines 5 code 3 comments 1 blanks # Calculate factorial # Result ? Number Factorial ← |1 ( ×. # Line comment ) FactorialThree ← Factorial 3 # Another line comment FactorialFour ← Factorial 4 ================================================ FILE: tests/data/unison.u ================================================ -- 16 lines 6 code 8 comments 2 blanks x = 3 if x < 2 then "-- {- -} -- Smaller. Test escaped quote \"!" else "Bigger -- {- -} --" {- Comments Comments -- nested -- Comments Comments -} -- testing quote in comment "hello there!" List.map (a -> a + 1) [1,2,3] ================================================ FILE: tests/data/urweb.ur ================================================ (* 14 lines 8 code 4 comments 2 blanks *) fun main () = return Hello world! (* multi line comment *) (* uncounted comment *)

Hello world!

 ================================================ FILE: tests/data/urweb_urp.urp ================================================ # 3 lines 1 code 1 comments 1 blanks urweb # uncounted comment ================================================ FILE: tests/data/urweb_urs.urs ================================================ (* 3 lines 1 code 1 comments 1 blanks *) val main : unit -> transaction page (* uncounted comment *) ================================================ FILE: tests/data/vb6_bas.bas ================================================ ' 12 lines 6 code 3 comments 3 blanks Attribute VB_Name = "Module1" Public Function SayHello(Name As String) ' Create a response string Dim Response As String Response = "Hello " & Name 'Return response string SayHello = Response End Function ================================================ FILE: tests/data/vb6_cls.cls ================================================ ' 22 lines 17 code 3 comments 2 blanks VERSION 1.0 CLASS BEGIN MultiUse = -1 'True Persistable = 0 'NotPersistable DataBindingBehavior = 0 'vbNone DataSourceBehavior = 0 'vbNone MTSTransactionMode = 0 'NotAnMTSObject END Attribute VB_Name = "Class1" Attribute VB_GlobalNameSpace = False Attribute VB_Creatable = True Attribute VB_PredeclaredId = False Attribute VB_Exposed = False 'This is a comment Private Sub Class_Initialize() 'This is another comment Dim test As String test = "TESTING" End Sub ================================================ FILE: tests/data/vb6_frm.frm ================================================ ' 34 lines 29 code 3 comments 2 blanks VERSION 5.00 Begin VB.Form Form1 Caption = "Form1" ClientHeight = 3015 ClientLeft = 120 ClientTop = 465 ClientWidth = 4560 LinkTopic = "Form1" ScaleHeight = 3015 ScaleWidth = 4560 StartUpPosition = 3 'Windows Default Begin VB.CommandButton btnTest Caption = "Test" Height = 495 Left = 720 TabIndex = 0 Top = 840 Width = 1095 End End Attribute VB_Name = "Form1" Attribute VB_GlobalNameSpace = False Attribute VB_Creatable = False Attribute VB_PredeclaredId = True Attribute VB_Exposed = False 'This is a comment Private Sub btnTest_Click() 'This is another comment Dim test As String test = "TESTING" End Sub ================================================ FILE: tests/data/vbscript.vbs ================================================ ' 8 lines 3 code 3 comments 2 blanks Dim MyStr1, MyStr2 MyStr1 = "Hello" ' This is also a comment MyStr2 = "Goodbye" REM Comment on a line ================================================ FILE: tests/data/velocity.vm ================================================ ## 11 lines 4 code 5 comments 2 blanks This text is visible. #* This text, as part of a multi-line comment, is not visible. This text is not visible; it is also part of the multi-line comment. This text still not visible. *# This text is outside the comment, so it is visible. ## This text is not visible. #macro( d ) #end ================================================ FILE: tests/data/vhdl.vhd ================================================ -- 34 lines 20 code 7 comments 7 blanks /* Since VHDL 2008 C-Style delimited comment are allowed. */ library IEEE; use IEEE.STD_LOGIC_1164.ALL; use IEEE.NUMERIC_STD.ALL; entity tb is Port ( clk : in STD_LOGIC; -- clock rst : in STD_LOGIC; -- reset -- removed: in STD_LOGIC_VECTOR(7 downto 0) ); end tb; -- architecture architecture behavioural of tb is signal toggle : STD_LOGIC := '0'; begin -- Toggles signal process(clk, rst) begin if (rst='1') then toggle <= '0'; else toggle <= not toggle; end if; end process; end ================================================ FILE: tests/data/visualbasic.vb ================================================ ' 7 lines 4 code 2 comments 1 blanks Public Class C Public Sub M() ' This is a comment End Sub End Class ================================================ FILE: tests/data/vqe.qasm ================================================ // 89 lines 58 code 21 comments 10 blanks /* * Variational eigensolver example * * Goal is to estimate the energy for a fixed set of parameters. * The parameters are updated outside of this program and a new * OpenQASM circuit is generated for the next iteration. */ include "stdgates.inc"; const int[32] n = 10; // number of qubits const int[32] layers = 3; // number of entangler layers const int[32] prec = 16; // precision of all types const int[32] shots = 1000; // number of shots per Pauli observable // Parameters could be written to local variables for this // iteration, but we will request them using extern functions extern get_parameter(uint[prec], uint[prec]) -> angle[prec]; extern get_npaulis() -> uint[prec]; extern get_pauli(int[prec]) -> bit[2 * n]; // The energy calculation uses floating point division, // so we do that calculation in an extern function extern update_energy(int[prec], uint[prec], float[prec]) -> float[prec]; gate entangler q { for uint i in [0:n-2] { cx q[i], q[i+1]; } } def xmeasure(qubit q) -> bit { h q; return measure q; } def ymeasure(qubit q) -> bit { s q; h q; return measure q; } /* Pauli measurement circuit. * The first n-bits of spec are the X component. * The second n-bits of spec are the Z component. */ def pauli_measurement(bit[2*n] spec, qubit[n] q) -> bit { bit b = 0; for uint[prec] i in [0: n - 1] { bit temp; if(spec[i]==1 && spec[n+i]==0) { temp = xmeasure(q[i]); } if(spec[i]==0 && spec[n+i]==1) { temp = measure q[i]; } if(spec[i]==1 && spec[n+i]==1) { temp = ymeasure(q[i]); } b ^= temp; } return b; } // Circuit to prepare trial wave function def trial_circuit(qubit[n] q) { for int[prec] l in [0: layers - 1] { for uint[prec] i in [0: n - 1] { angle[prec] theta; theta = get_parameter(l * layers + i); ry(theta) q[i]; } if(l != layers - 1) entangler q; } } /* Apply VQE ansatz circuit and measure a Pauli operator * given by spec. Return the number of 1 outcomes. */ def counts_for_term(bit[2*n] spec, qubit[n] q) -> uint[prec] { uint[prec] counts; for uint i in [1: shots] { bit b; reset q; trial_circuit q; b = pauli_measurement(spec, q); counts += int[1](b); } return counts; } // Estimate the expected energy def estimate_energy(qubit[n] q) -> float[prec] { float[prec] energy; uint[prec] npaulis = get_npaulis(); for int[prec] t in [0:npaulis-1] { bit[2*n] spec = get_pauli(t); uint[prec] counts; counts = counts_for_term(spec, q); energy = update_energy(t, counts, energy); } return energy; } qubit[n] q; float[prec] energy; energy = estimate_energy(q); ================================================ FILE: tests/data/vue.vue ================================================ ================================================ FILE: tests/data/webassembly.wat ================================================ ;; 10 lines 8 code 1 comments 1 blanks (module (import "console" "log" (func $log (param i32 i32))) (import "js" "mem" (memory 1)) (data (i32.const 0) "Hi") (func (export "writeHi") i32.const 0 ;; pass offset 0 to log i32.const 2 ;; pass length 2 to log call $log)) ================================================ FILE: tests/data/wenyan.wy ================================================ //! 141 lines 107 code 1 comments 33 blanks 吾嘗觀「「曆法」」之書。方悟「今何紀元時」「彼時何小時」「彼刻何刻」「彼分何分」「彼秒何秒」之義。 吾嘗觀「「畫譜」」之書。方悟「備紙」「擇筆」「蘸色」「落筆」「運筆」「提筆」「設色」「裱畫」之義。 吾嘗觀「「算經」」之書。方悟「倍圓周率」「正弦」「餘弦」之義。 施「今何紀元時」。名之曰「紀元時」。 施「彼時何小時」於「紀元時」。名之曰「小時」。 施「彼刻何刻」於「紀元時」。名之曰「刻」。 施「彼分何分」於「紀元時」。名之曰「分」。 施「彼秒何秒」於「紀元時」。名之曰「秒」。 有數四百。名之曰「紙縱」。 有數四百。名之曰「紙橫」。 除二於「紙縱」。名之曰「半縱」。 除二於「紙橫」。名之曰「半橫」。 吾有一數。名之曰「比例」。 批曰。「「文氣淋灕。字句切實」」。 若「半橫」小於「半縱」者。 昔之「比例」者。今「半橫」是矣。 若非。 昔之「比例」者。今「半縱」是矣。 云云。 吾有一術。名之曰「縱坐標」。欲行是術。必先得一數。曰「南」。是術曰。 乘「南」以「比例」。減其於「半縱」。乃得矣。 是謂「縱坐標」之術也。 吾有一術。名之曰「橫坐標」。欲行是術。必先得一數。曰「東」。是術曰。 乘「東」以「比例」。減其於「半橫」。乃得矣。 是謂「橫坐標」之術也。 吾有一術。名之曰「極坐標」。欲行是術。必先得二數。曰「距」。曰「角」。是術曰。 施「餘弦」於「角」。乘其以「距」。取一以施「縱坐標」。名之曰「縱」。 施「正弦」於「角」。乘其以「距」。取一以施「橫坐標」。名之曰「橫」。 吾有一物。名之曰「坐標」。其物如是。 物之「「橫」」者。數曰「橫」。 物之「「縱」」者。數曰「縱」。 是謂「坐標」之物也。乃得「坐標」。 是謂「極坐標」之術也。 吾有一術。名之曰「畫鐘面」。 欲行是術。必先得一物。曰「紙」。一數。曰「半徑」。 是術曰。 有數一千零二十四。名之曰「割圓」。 夫「半徑」。夫零。取二以施「極坐標」。名之曰「始坐標」。 夫「紙」。夫「始坐標」之「「橫」」。夫「始坐標」之「「縱」」。取三以施「落筆」。 有數一。名之曰「甲」。 為是「割圓」遍。 除「甲」以「割圓」。乘其以「倍圓周率」。名之曰「乙」。 夫「半徑」。夫「乙」。取二以施「極坐標」。名之曰「坐標」。 夫「紙」。夫「坐標」之「「橫」」。夫「坐標」之「「縱」」。取三以施「運筆」。 加「甲」以一。昔之「甲」者。今其是矣。 云云。 施「蘸色」於「紙」於「「鈦白」」。 施「設色」於「紙」。 施「蘸色」於「紙」於「「黑」」。 施「提筆」於「紙」。 有數零。名之曰「丙」。 為是六十遍。 除「丙」以六十。乘其以「倍圓周率」。名之曰「丁」。 夫「半徑」。夫「丁」。取二以施「極坐標」。名之曰「正刻外坐標」。 夫「紙」。夫「正刻外坐標」之「「橫」」。夫「正刻外坐標」之「「縱」」。取三以施「落筆」。 乘九分五於「半徑」。夫「丁」。取二以施「極坐標」。名之曰「正刻內坐標」。 夫「紙」。夫「正刻內坐標」之「「橫」」。夫「正刻內坐標」之「「縱」」。取三以施「運筆」。 施「提筆」於「紙」。 加「丙」以一。昔之「丙」者。今其是矣。 云云。 有數零。名之曰「丙」。 為是十二遍。 除「丙」以十二。乘其以「倍圓周率」。名之曰「戊」。 夫「半徑」。夫「戊」。取二以施「極坐標」。名之曰「初刻外坐標」。 夫「紙」。夫「初刻外坐標」之「「橫」」。夫「初刻外坐標」之「「縱」」。取三以施「落筆」。 乘八分五於「半徑」。夫「戊」。取二以施「極坐標」。名之曰「初刻內坐標」。 夫「紙」。夫「初刻內坐標」之「「橫」」。夫「初刻內坐標」之「「縱」」。取三以施「運筆」。 施「提筆」於「紙」。 加「丙」以一。昔之「丙」者。今其是矣。 云云。 是謂「畫鐘面」之術也。 吾有一術。名之曰「畫指針」。 欲行是術。必先得一物。曰「紙」。五數。曰「角」。曰「針長」。曰「尾長」。曰「針角」。曰「尾角」。 是術曰。 夫「針長」。加「針角」於「角」。取二以施「極坐標」。名之曰「甲」。 乘負一於「尾長」。減「尾角」於「角」。取二以施「極坐標」。名之曰「乙」。 乘負一於「尾長」。加「尾角」於「角」。取二以施「極坐標」。名之曰「丙」。 夫「針長」。減「針角」於「角」。取二以施「極坐標」。名之曰「丁」。 夫「紙」。夫「甲」之「「橫」」。夫「甲」之「「縱」」。取三以施「落筆」。 夫「紙」。夫「乙」之「「橫」」。夫「乙」之「「縱」」。取三以施「運筆」。 夫「紙」。夫「丙」之「「橫」」。夫「丙」之「「縱」」。取三以施「運筆」。 夫「紙」。夫「丁」之「「橫」」。夫「丁」之「「縱」」。取三以施「運筆」。 夫「紙」。夫「甲」之「「橫」」。夫「甲」之「「縱」」。取三以施「運筆」。 施「蘸色」於「紙」於「「花青」」。 施「設色」於「紙」。 是謂「畫指針」之術也。 吾有一術。名之曰「执笔」。是術曰。 施「(()=>document.getElementById("out").innerHTML="")」。 施「今何紀元時」。名之曰「紀元時」。 施「彼時何小時」於「紀元時」。名之曰「時」。 施「彼分何分」於「紀元時」。名之曰「分」。 施「彼刻何刻」於「紀元時」。名之曰「刻」。 施「彼秒何秒」於「紀元時」。名之曰「秒」。 乘「刻」以十五。加其於「分」。昔之「分」者。今其是矣。 除「秒」以六十。加其於「分」。昔之「分」者。今其是矣。 除「分」以六十。加其於「時」。昔之「時」者。今其是矣。 除「分」以六十。乘其以「倍圓周率」。乘其以負一。名之曰「分角」。 除「時」以十二。乘其以「倍圓周率」。乘其以負一。名之曰「時角」。 除「秒」以六十。乘其以「倍圓周率」。乘其以負一。名之曰「秒角」。 施「備紙」於「紙橫」。於「紙縱」。名之曰「紙」。 施「畫鐘面」於「紙」。於九分。 施「畫指針」於「紙」。於「秒角」。於八分。於一分。於三毫。於一分。 施「畫指針」於「紙」。於「分角」。於七分五釐。於一分。於三毫。於三分。 施「畫指針」於「紙」。於「時角」。於五分五釐。於八釐。於五毫。於五分。 施「裱畫」於「紙」於「「out」」。 是謂「执笔」之術也。 施「(x=>setInterval(x, 500))」於「执笔」。 ================================================ FILE: tests/data/wgsl.wgsl ================================================ // 13 lines 10 code 2 comments 1 blanks // comment [[stage(vertex)]] fn vs_main([[builtin(vertex_index)]] in_vertex_index: u32) -> [[builtin(position)]] vec4 { const x = f32(i32(in_vertex_index) - 1); const y = f32(i32(in_vertex_index & 1u) * 2 - 1); return vec4(x, y, 0.0, 1.0); } [[stage(fragment)]] fn fs_main() -> [[location(0)]] vec4 { return vec4(1.0, 0.0, 0.0, 1.0); } ================================================ FILE: tests/data/xsl.xsl ================================================ ================================================ FILE: tests/data/xtend.xtend ================================================ // 23 lines 13 code 4 comments 6 blanks class Test { static def void main(String[] args) { /* * Multiline comment */ val f = new Foo() f.bar() // Not counted } } class Foo { def bar() { println('string type 1') println("string type 2") println('''string type 3''') } } ================================================ FILE: tests/data/yaml.yaml ================================================ # 34 lines 29 code 3 comments 2 blanks # Manifest file from Kubernetes documentation: # https://kubernetes.io/docs/tutorials/stateless-application/guestbook/ apiVersion: apps/v1 kind: Deployment metadata: name: redis-master labels: app: redis spec: selector: matchLabels: app: redis role: master tier: backend replicas: 1 template: metadata: labels: app: redis role: master tier: backend spec: containers: - name: master image: k8s.gcr.io/redis:e2e resources: requests: cpu: 100m memory: 100Mi ports: - containerPort: 6379 ================================================ FILE: tests/data/zencode.zs ================================================ // 21 lines 9 code 7 comments 5 blanks // This is a single line comment. /* This is a multiline comment on a single line. */ /* This is a multiline comment. */ var str = "/*"; var arr = [str, @"wysiwyg", '\"']; for item in arr { print(item); // Found the */ } // Comment with quote " var badStr = // Comment before value "\""; badStr = // Another comment before value @'zen'; ================================================ FILE: tests/data/zig.zig ================================================ // 11 lines 5 code 3 comments 3 blanks /// Documentation comment pub fn main() void { const a = 5; // not-counted // Leading-comment const b = c"line comment embedded //"; const c = \\line comment embedded // // } ================================================ FILE: tests/data/zokrates.zok ================================================ // 11 lines 3 code 6 comments 2 blanks /* This is a multi-line comment written in more than just one line. */ def main() -> field { // an inline comment return 42; // on a line. } ================================================ FILE: tests/embedding/file_triggeringprincipal_frame_1.html ================================================ < 0!DOCTYPE HTML> Frame 1
================================================ FILE: tokei.example.toml ================================================ # The width of the terminal output in columns. columns = 80 # Sort languages based on the specified column. sort = "lines" # If set, tokei will only show the languages in `types`. types = ["Python"] # Any doc strings (e.g. `"""hello"""` in python) will be counted as comments. treat_doc_strings_as_comments = true