Showing preview only (290K chars total). Download the full file or copy to clipboard to get everything.
Repository: guybedford/chomp
Branch: main
Commit: 0164c9f72473
Files: 33
Total size: 278.1 KB
Directory structure:
gitextract__re7x5ob/
├── .github/
│ └── workflows/
│ ├── release.yml
│ └── test.yml
├── .gitignore
├── .gitmodules
├── Cargo.toml
├── LICENSE
├── README.md
├── chompfile.toml
├── docs/
│ ├── chompfile.md
│ ├── cli.md
│ ├── extensions.md
│ └── task.md
├── node-chomp/
│ ├── README.md
│ ├── index.js
│ └── package.json
├── src/
│ ├── ansi_windows.rs
│ ├── chompfile.rs
│ ├── engines/
│ │ ├── cmd.rs
│ │ ├── deno.rs
│ │ ├── mod.rs
│ │ └── node.rs
│ ├── extensions.rs
│ ├── http_client.rs
│ ├── main.rs
│ ├── server.rs
│ └── task.rs
└── test/
├── chompfile.toml
├── fixtures/
│ ├── app.js
│ ├── many/
│ │ ├── one/
│ │ │ └── config.yml
│ │ └── two/
│ │ └── config.yml
│ └── src/
│ ├── app.ts
│ └── dep.ts
└── unit/
└── ok-node.mjs
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/workflows/release.yml
================================================
on:
push:
tags: '*'
name: Create Release
jobs:
publish-crate:
name: Publish to crates.io
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
- run: cargo login ${CRATES_IO_TOKEN}
env:
CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
- run: cargo build --release --locked
- name: publish chomp
run: cargo publish
create-github-release:
name: Create GitHub Release
needs: publish-crate
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Create Release Notes
uses: actions/github-script@v4.0.2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.request(`POST /repos/${{ github.repository }}/releases`, {
tag_name: "${{ github.ref }}",
generate_release_notes: true
});
build:
name: Build assets for ${{ matrix.os }}
needs: create-github-release
runs-on: ${{ matrix.os }}
strategy:
matrix:
name: [
linux,
windows,
macos
]
include:
- name: linux
os: ubuntu-latest
artifact_name: chomp
asset_name: chomp-linux
asset_extension: .tar.gz
- name: windows
os: windows-latest
artifact_name: chomp.exe
asset_name: chomp-windows
asset_extension: .zip
- name: macos
os: macos-latest
artifact_name: chomp
asset_name: chomp-macos
asset_extension: .tar.gz
steps:
- uses: actions/checkout@v1
- name: Set env
run: |
RELEASE_VERSION=$(echo ${GITHUB_REF:10})
echo "asset_name=${{ matrix.asset_name }}-${RELEASE_VERSION}${{ matrix.asset_extension }}" >> $GITHUB_ENV
shell: bash
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
- name: Build
run: cargo build --release --locked
- name: Archive release
shell: bash
run: |
cp "target/release/${{ matrix.artifact_name }}" "${{ matrix.artifact_name }}"
if [ "${{ matrix.os }}" = "windows-latest" ]; then
7z a "${asset_name}" "${{ matrix.artifact_name }}"
else
tar czf "${asset_name}" "${{ matrix.artifact_name }}"
fi
- name: Upload binaries to release
uses: svenstaro/upload-release-action@v1-release
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: chomp*${{ matrix.asset_extension }}
file_glob: true
tag: ${{ github.ref }}
================================================
FILE: .github/workflows/test.yml
================================================
name: Test
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
jobs:
build-ubuntu:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build
run: cargo build --verbose
- name: Set PATH
run: echo "$(pwd)/target/debug:$PATH" >> $GITHUB_PATH
- uses: actions/checkout@v3
with:
repository: 'guybedford/chomp-extensions'
path: 'chomp-extensions'
- name: Run Core Tests
run: chomp -c test/chompfile.toml test
env:
CHOMP_CORE: ../chomp-extensions
- name: Run Template Tests
run: chomp -c chomp-extensions/chompfile.toml test
build-windows:
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- name: Install latest stable
uses: actions-rs/toolchain@v1
with:
toolchain: stable
override: true
components: cargo
- name: Build
run: cargo build --verbose
- name: Set PATH
run: echo echo "$(pwd)/target/debug" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- uses: actions/checkout@v3
with:
repository: 'guybedford/chomp-extensions'
path: 'chomp-extensions'
- name: Run Core Tests
run: chomp -c test/chompfile.toml test
env:
CHOMP_CORE: ../chomp-extensions
- name: Run Template Tests
run: chomp -c chomp-extensions/chompfile.toml test
================================================
FILE: .gitignore
================================================
node_modules
target
test/package.json
package-lock.json
sandbox
test/output
vendor
================================================
FILE: .gitmodules
================================================
================================================
FILE: Cargo.toml
================================================
[package]
name = "chompbuild"
version = "0.3.0"
authors = ["Guy Bedford <guybedford@gmail.com>"]
edition = "2021"
license = "Apache-2.0"
repository = "https://github.com/guybedford/chomp/"
homepage = "https://chompbuild.com/"
keywords = ["make", "task", "runner", "javascript", "web"]
categories = ["command-line-utilities", "development-tools", "web-programming"]
readme = "README.md"
description = "Make-like parallel task runner with a JS extension system"
[[bin]]
name = "chomp"
path = "src/main.rs"
[target.'cfg(target_os="windows")'.dependencies.winapi]
version = "0.3"
features = ["consoleapi", "errhandlingapi", "fileapi", "handleapi"]
[dependencies]
anyhow = "1"
async-recursion = "1"
capturing-glob = "0"
base64 = "0.22"
clap = "4"
convert_case = "0.11"
derivative = "2"
dirs = "6"
futures = "0"
hyper = { version = "1", features = ["client", "http1", "http2"] }
hyper-tls = "0.6"
hyper-util = { version = "0.1", features = ["client-legacy", "tokio", "http1", "http2"] }
http-body-util = "0.1"
bytes = "1"
lazy_static = "1"
mime_guess = "2"
notify = "8"
notify-debouncer-mini = "0.7"
num_cpus = "1"
percent-encoding = "2"
regex = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_v8 = "0.181.0"
sha2 = "0.11"
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
tokio-util = "0.7"
toml = "0.8"
uuid = { version = "1", features = ["v4"] }
v8 = "0.89"
warp = { version = "0.4", features = ["server", "websocket"] }
directories = "6"
pathdiff = "0"
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
# CHOMP
[](https://crates.io/crates/chompbuild)
[](https://discord.gg/5E9zrhguTy)
Chomp is a frontend task runner with advanced features focused on _ease-of-use_ and _not getting in the way_!
1. An advanced task runner with a single command!
1. Easily adapt existing projects / task systems - no need for a rewrite.
1. You enable and manage advanced task runner features with single-line updates.
Chomp is a great option for frontend projects where the goal is getting advanced task runner features (like smart caching) without complexity and overhead.
## One-line migration from npm scripts
Chomp can import a project's established `package.json` scripts without breaking them, as it supports the same features:
```bash
chomp --init --import-scripts
```
Now you can run your npm scripts using Chomp!
> i.e `npm run <task>` becomes `chomp <task>` and behaves the same, and you can opt in to further features as needed.
The only difference is, with Chomp — it's faster. And, with a few more tweaks, you can enable smart caching, parallelism, and more!
## What features does Chomp provide?
Chomp is an advanced task runner. It provides features similar to [turbo](https://turbo.build/repo) and [nx](https://nx.dev/) but focuses on ease of use, *not monorepos. It's based on the same principles as traditional make files.
### Parallelism
Chomp [runs tasks in parallel](./docs/task.md#serial-dependencies), based on an extecuted task's dependencies!
### Watch/Serve
Chomp [watches any task](./docs/task.md#watched-rebuilds) by including a `--watch` or `--serve` option! Read more about the power of [`--watch`](./docs/task.md#watched-rebuilds) and [`--serve`](./docs/task.md#static-server).
### A JS extension system
Chomp has a [JS extension system](./docs/extensions.md) that allows you to extend Chomp with your own custom tasks
### Smart caching
Chomp [caches tasks](./docs/task.md#task-caching) based on task dependencies like other tasks or updated files. You don't have to worry about it!
> \*Chomp works for monrepos but it's architected for ease of use and not getting in the way first.
## Install
If you use [Cargo](https://rustup.rs/), run:
```
cargo install chompbuild
```
If you don't use Cargo, run:
```
npm install -g chomp
```
> Note: npm scripts add over 100ms to the script run time.
Common platform binaries are also available for [all releases](https://github.com/guybedford/chomp/releases).
To quickly set up Chomp in a GitHub Actions CI workflow, see the [Chomp GitHub Action](https://github.com/guybedford/chomp-action).
## Documentation
* [CLI Usage](https://github.com/guybedford/chomp/blob/main/docs/cli.md)
* [Chompfile Definitions](https://github.com/guybedford/chomp/blob/main/docs/chompfile.md)
* [Task Definitions](https://github.com/guybedford/chomp/blob/main/docs/task.md)
* [Extensions](https://github.com/guybedford/chomp/blob/main/docs/extensions.md)
## Getting Started
### Migrating from npm Scripts
To convert an existing project using npm `"scripts"` to Chomp, run:
```sh
$ chomp --init --import-scripts
√ chompfile.toml created with 2 package.json script tasks imported.
```
or the shorter version:
```sh
$ chomp -Ii
√ chompfile.toml created with 2 package.json script tasks imported.
```
Then use `chomp <name>` instead of `npm run <name>`, and enjoy the new features of task dependence, incremental builds, and parallelism!
### Hello World
`chomp` works against a [`chompfile.toml`](https://github.com/guybedford/chomp/blob/main/docs/chompfile.md) [TOML configuration](https://toml.io/) in the same directory as the `chomp` command is run.
Chomp builds up tasks as trees of files which depend on other files, then runs those tasks with [maximum parallelism](https://github.com/guybedford/chomp/blob/main/docs/task.md#task-dependence).
For example, here's a task called `hello` which builds `hello.txt` based on the contents of `name.txt`, which itself is built by another command:
chompfile.toml
```toml
version = 0.1
[[task]]
target = 'name.txt'
run = '''
echo "No name.txt, writing one."
echo "World" > name.txt
'''
[[task]]
name = 'hello'
target = 'hello.txt'
dep = 'name.txt'
run = '''
echo "Hello $(cat name.txt)" > hello.txt
'''
```
with this file saved, the hello command will run all dependency commands before executing its own command:
```sh
$ chomp hello
🞂 name.txt
No name.txt, writing one.
√ name.txt [4.4739ms]
🞂 hello.txt
√ hello.txt [5.8352ms]
$ cat hello.txt
Hello World
```
Finally it populates the `hello.txt` file with the combined output.
Subsequent runs use the mtime of the target files to determine what needs to be rerun.
Rerunning the `hello` command will see that the `hello.txt` target is defined, and that the `name.txt` dependency didn't change, so it will skip running the command again:
```sh
chomp hello
● name.txt [cached]
● hello.txt [cached]
```
Changing the contents of `name.txt` will then invalidate the `hello.txt` target only, not rerunning the `name.txt` command:
```sh
$ echo "Chomp" > name.txt
$ chomp hello
● name.txt [cached]
hello.txt invalidated by name.txt
🞂 hello.txt
√ hello.txt [5.7243ms]
$ cat hello.txt
Hello Chomp
```
The [`deps`](https://github.com/guybedford/chomp/blob/main/docs/task.md#task-dependence) array can be defined for targets, whose targets will then be run first with [invalidation based on target / deps mtime comparisons](https://github.com/guybedford/chomp/blob/main/docs/task.md#task-caching) per the standard Makefile approach.
Powershell is used on Windows, while Bash is used on POSIX systems. Since both `echo` and `>` are defined on both systems, the examples above work cross-platform (Powershell is automatically put into UTF-8 mode for `>` to work similarly).
Note that `&&` and `||` are not supported in Powershell, so multiline scripts and `;` are preferred instead.
#### JS Tasks
Alternatively we can use `engine = 'node'` or `engine = 'deno'` to write JavaScript in the `run` function instead:
chompfile.toml
```toml
version = 0.1
[[task]]
target = 'name.txt'
engine = 'node'
run = '''
import { writeFile } from 'fs/promises';
console.log("No name.txt, writing one.");
await writeFile(process.env.TARGET, 'World');
'''
[[task]]
name = 'hello'
target = 'hello.txt'
deps = ['name.txt']
engine = 'node'
run = '''
import { readFile, writeFile } from 'fs/promises';
const name = (await readFile(process.env.DEP, 'utf8')).trim();
await writeFile(process.env.TARGET, `Hello ${name}`);
'''
```
Tasks are run with maximum parallelism as permitted by the task graph, which can be controlled via the [`-j` flag](https://github.com/guybedford/chomp/blob/main/docs/cli.md#jobs) to limit the number of simultaneous executions.
Using the [`--watch` flag](https://github.com/guybedford/chomp/blob/main/docs/cli.md#watch) watches all dependencies and applies incremental rebuilds over invalidations only.
Or, using `chomp hello --serve` runs a [static file server](https://github.com/guybedford/chomp/blob/main/docs/task.md#static-server) with watched rebuilds.
See the [task documentation](https://github.com/guybedford/chomp/blob/main/docs/task.md) for further details.
#### Monorepos
There is no first-class monorepo support in chomp, but some simple techniques can achieve the same result.
For example, consider a monorepo where `packages/[pkgname]/chompfile.toml` defines per-package tasks.
A base-level `chompfile.toml` could run the `test` task of all the sub-packages with the following `chompfile.toml`:
```toml
[[task]]
name = 'test'
dep = 'packages/#/chompfile.toml'
run = 'chomp -c $DEP test'
```
`chomp test` will then use [task interpolation](https://github.com/guybedford/chomp/blob/main/docs/task.md#task-interpolation) to run the multiple sub-package test tasks in parallel. A similar approach can also be used for a [basic unit testing](https://github.com/guybedford/chomp/blob/main/docs/task.md#testing).
By adding [`serial = 'true'`](https://github.com/guybedford/chomp/blob/main/docs/task.md#serial-dependencies), the interpolation can be made to run in series rather than in parallel.
Cross-project dependencies are [not currently supported](https://github.com/guybedford/chomp/issues/119). Instead, if `packages/a/chompfile.toml`'s build task depends on `packages/b/chompfile.toml`'s build task to run first, then `packages/a/chompfile.toml` might look like:
```toml
[[task]]
name = 'build'
run = 'cargo build'
dep = 'build:deps'
[[task]]
name = 'build:deps'
run = 'chomp -c ../a build'
```
This would still be fast, so long as `packages/a/chompfile.toml`'s `build` task has its targets and dependencies properly configured to do zero work if the all target mtimes are greater than their dependencies.
### Extensions
Extensions are able to register task templates for use in Chompfiles.
Extensions are loaded using the `extensions` list, which can be any local or remote JS file:
```toml
version = 0.1
extensions = [
"./local.js",
"https://remote.com/extension.js"
]
```
A core extensions library is provided with useful templates for the JS ecosystem, with
the short protocol `chomp:ext`, a shorthand for the `@chompbuild/extensions` package contents.
A simple example is included below.
_See the [@chompbuild/extensions package](https://github.com/guybedford/chomp-extensions) for extension descriptions and examples._
#### Example: TypeScript with SWC
To compile TypeScript with the SWC template:
```toml
version = 0.1
extensions = ['chomp@0.1:swc']
[[task]]
name = 'build:typescript'
template = 'swc'
target = 'lib/##.js'
deps = ['src/##.ts']
```
In the above, all `src/**/*.ts` files will be globbed, have SWC run on them, and output into `lib/[file].js` along with their source maps.
The `##` and `#` interpolation syntax is special because, unlike glob dependencies (which are also supported), there must be a 1:1 relationship between a dependency and its target.
Only non-existent files, or files whose `src` mtimes are invalidated will be rebuilt. If SWC itself is updated, all files that depend on it will be re-built.
Specific files or patterns can be built directly by name as well, skipping all other build work:
```sh
chomp lib/main.js lib/dep.js
🞂 lib/dep.js
🞂 lib/app.js
√ lib/dep.js [317.2838ms]
√ lib/app.js [310.0831ms]
```
Patterns are also supported for building tasks by name or filename (the below two commands are equivalent):
```sh
$ chomp lib/*.js
$ chomp :build:*
```
To remove the template magic, run `chomp --eject` to convert the `chompfile.toml` into its untemplated form:
```sh
$ chomp --eject
√ chompfile.toml template tasks ejected
```
Resulting in the updated _chompfile.toml_:
```toml
version = 0.1
[[task]]
name = 'build:typescript'
target = 'lib/##.js'
dep = 'src/##.ts'
stdio = 'stderr-only'
run = 'node ./node_modules/@swc/cli/bin/swc.js $DEP -o $TARGET --no-swcrc --source-maps -C jsc.parser.syntax=typescript -C jsc.parser.importAssertions=true -C jsc.parser.topLevelAwait=true -C jsc.parser.importMeta=true -C jsc.parser.privateMethod=true -C jsc.parser.dynamicImport=true -C jsc.target=es2016 -C jsc.experimental.keepImportAssertions=true'
```
# License
Apache-2.0
================================================
FILE: chompfile.toml
================================================
version = 0.1
default-task = 'build'
[[task]]
name = 'build'
deps = ['src/**/*.rs']
run = 'cargo build'
[[task]]
name = 'build:release'
deps = ['src/**/*.rs']
run = 'cargo build --release --locked'
[[task]]
name = 'test'
dep = 'build'
cwd = 'test'
run = '../target/debug/chomp test'
[[task]]
name = 'install'
serial = true
dep = 'build'
run = 'cp ./target/[dD]ebug/chomp* ~/bin/'
[[task]]
name = 'inline-version'
deps = ['Cargo.toml']
targets = ['src/main.rs', 'node-chomp/package.json']
engine = 'node'
run = '''
import { readFileSync, writeFileSync } from 'fs';
const toml = readFileSync('Cargo.toml', 'utf8');
const [, version] = toml.match(/version\s*=\s*\"(\d+\.\d+\.\d+)\"/);
const main = readFileSync('src/main.rs', 'utf8');
writeFileSync('src/main.rs', main.replace(/let version = "\d+.\d+.\d+/g, `let version = "${version}`));
const pjson = JSON.parse(readFileSync('node-chomp/package.json', 'utf8'));
pjson.version = version;
writeFileSync('node-chomp/package.json', JSON.stringify(pjson, null, 2));
'''
================================================
FILE: docs/chompfile.md
================================================
# Chompfile
Chomp projects are defined by a `chompfile.toml`, with Chompfiles defined using the [TOML configuration format](https://toml.io/).
The default Chompfile is `chompfile.toml`, located in the same directory as the `chomp` binary is being run from.
Custom configuration can be used via `chomp -c custom.toml` or `chomp -c ./nested/chompfile.toml`.
All paths within a Chompfile are relative to the Chompfile itself regardless of the invocation CWD.
## Example
To create a new Chomp project, create a new file called `chompfile.toml` and add the following lines:
chompfile.toml
```toml
version = 0.1
[[task]]
name = 'build'
run = 'echo "Chomp Chomp"'
```
In the command line, type `chomp build` or just `chomp` (_"build"_ is the default task when none is given):
```sh
$ chomp
🞂 :build
Chomp Chomp
√ :build [6.3661ms]
```
to get the runner output.
Every Chompfile must start with the `version = 0.1` version number, at least until the project stabilizes.
See the [task documentation](tasks.md) for defining tasks.
## Chompfile Definitions
The Chompfile supports the following definitions:
chompfile.toml
```toml
# Every Chompfile must start with the Chompfile version, currently 0.1
version = 0.1
# The default task name to run when `chomp` is run without any CLI arguments
default_task = "test"
# List of Chomp Extensions to load
extensions = ["extension-path"]
# Environment variables for all runs
[env]
ENV_VAR = "value"
# Default environment variables to only set if not already for all runs
[env-default]
DEFAULT_VAR = "value"
# Static server options for `chomp --serve`
[server]
# Static server root path, relative to the Chomp file
root = "public"
# Static server port
port = 1010
# Default template options by registered template name
# When multiple tasks use the same template, this avoids duplicated `[template-options]` at the task level
[template-options.<template name>]
key = value
# Task definitions
# Tasks are a TOML list of Task objects, which define the task graph
[[task]]
name = "TASK"
run = "shell command"
```
See the [task documentation](task.md) for defining tasks, and the [extension documentation](extensions.md) for defining Chompfile extensions.
================================================
FILE: docs/cli.md
================================================
# CLI Flags
Usage:
```
chomp [FLAGS/OPTIONS] <TARGET>...
```
Chomp takes the following arguments and flags:
* [`<TARGET>...`](#target): List of targets to build
* [`-C, --clear-cache`](#clear-cache): Clear URL extension cache
* [`-c, --config`](#config): Custom chompfile project or path [default: chompfile.toml]
* [`--eject`](#eject): Ejects templates into tasks saving the rewritten chompfile.toml
* [`-f, --force`](#force): Force rebuild targets
* [`-F, --format`](#format): Format and save the chompfile.toml
* [`-h, --help`](#help): Prints help information
* [`-I, --import-scripts`](#import-scripts): Import npm package.json "scripts" into the chompfile.toml
* [`-i, --init`](#init): Initialize the chompfile.toml if it does not exist
* [`-j, --jobs`](#jobs): Maximum number of jobs to run in parallel
* [`-l, --list`](#list): List the available chompfile tasks
* [`-p, --port`](#port): Custom port to serve
* [`-r, --rerun`](#rerun): Rerun the listed targets without caching
* [`-s, --serve`](#serve): Run a local dev server
* [`-R, --server-root`](#server-root): Server root path
* [`-V, --version`](#version): Prints version information
* [`-w, --watch`](#watch): Watch the input files for changes
## Target
The main arguments of the `chomp` command are a list of targets to build.
Build targets can be task names, file paths relative to the `chompfile.toml`, or glob patterns of task names or file paths to build.
To disambiguate task names from file paths, task names can always be referenced with a `:` prefix - `chomp :test` instead of `chomp test`.
Only the necessary work to produce the provided targets will be performed, taking into account [task dependence](task.md#task-dependence).
When no target is provided, the `default-task` defined in the Chompfile is run, if set.
## Clear Cache
When loading Chomp extensions from external URLs via the [`extensions` configuration](task.md#loading-extensions),
remote extensions are cached in the user-local `[cachedir]/.chomp/` folder.
Extensions are cached permanently regardless of cache headers to optimize for task run execution time.
Run `chomp --clear-cache` to clear these caches.
Where possible, use unique versioned URLs for remote extensions.
## Config
Usually Chomp will look for `chompfile.toml` within the current working directory.
Running `chomp -c ./path/to/chompfile.toml` allows running Chomp on a folder that is not the current working directory,
or running Chomp against a Chompfile with another name than `chompfile.toml`.
## Force
When running a task, the default [invalidation rules](task.md#task-invalidation-rules) of that [task dependence graph](task.md#task-dependence) will apply.
To treat all tasks in the target graph as invalidated, the `chomp -f task` flag can be useful to ensure everything is fresh.
## Format
`chomp --format` will apply the default serialization formatting to the `chompfile.toml` file.
Note this command will overwrite the existing `chompfile.toml` with the new formatting.
This command is compatible with the [`--config`](#config) flag to choosing the Chompfile to operate on.
Due to limitations with the Rust TOML implementation, comments are currently stripped by this operation.
## Help
CLI help is available via `chomp -h`.
## Jobs
Sets the maximum number of task runs to spawn in parallel. Defaults to the logical CPU count.
By default tasks in Chomp are run with [maximum parallelization](task.md#task-parallelization).
## List
`chomp --list` will output a listing of the named tasks of the current `chompfile.toml` or Chompfile specified by [`--config`](#config).
## Port
When using [`chomp --serve`](#serve) to run a local static server, customizes the static server port. Defaults to `8080`.
## Rerun
Useful to rerun specific tasks without caching without invalidating the whole tree.
`chomp -r x` will rerun task `x` even if it is cached, but without rerunning its cached dependencies.
To invalidate the full task graph use [`chomp --force`](#force).
## Serve
Enables the file watcher, and runs a static server with the optionally [`--port`](#port) and [`--server-root`](#server-root), which are also customizable in the [Chompfile](chompfile.md).
When serving, a list of [task targets](#target) is still taken to watch.
## Server Root
When using [`chomp --serve`](#serve) to run a local static server, customizes the site root to serve. Defaults to the same folder as the Chompfile.
## Version
The current Chomp version is available via `chomp --version`
## Watch
The `--watch` flag instructs Chomp to continue running after completing the tasks, and listen to any changes to all files that were touched by the [task dependency graph](task.md#task-dependence).
A [list of targets](#target) is supplied like any other Chomp run, which then informs which files are watched.
================================================
FILE: docs/extensions.md
================================================
# Extensions
## Overview
Executions are loaded through the embedded V8 environment in Chomp, which includes a very simple `console.log` implementation and basic error handling, an `ENV` global detailed below and a `Chomp` global detailed below.
Extensions must be declared in the active `chompfile.toml` in use via the `extensions` list, and can be loaded from any path or URL.
URL extensions are fetched from the network and cached indefinitely in the global Chomp cache folder.
All executions are immediately invoked during the initialization phase, and all registrations must be made during this phase. Any
hook registrations via the `Chomp` global made after this initialization phase will throw an error.
Registrations made by Chomp extensions can then hook into phases of the Chomp task running lifecycle including the process of task list population, template expansion and batching of tasks.
## Publishing Extensions
When developing extensions, it is recommended to load them by relative file paths:
chompfile.toml
```toml
version = 0.1
extensions = ['./local-extension.js']
```
When sharing the extension between projects, hosting it on any remote URL is supported by Chomp.
Note that remote URLs are cached indefinitely regardless of cache headers for performance so it is recommended to include the version in the URL. `chomp --cache-clear` can be used to clear this remote cache.
If publishing to npm, templates will be available on any npm CDN like `unpkg.com` or `ga.jspm.io`.
If publishing to JSPM, set the `package.json` property `"type": "script"` to inform JSPM the `.js` files are scripts and not modules to avoid incorrect processing.
## API
JavaScript extensions register hooks via the `Chomp` global scripting interface.
TypeScript typing is not currently available for the `Chomp` global. PRs to provide this typing integration would be welcome.
### ENV
The `ENV` JS global is available in extensions, and contains all environment variables as a dictionary.
The following Chomp-specific environment variables are also defined:
* `ENV.CHOMP_EJECT`: When `--eject` is passed for template injection this is set to `"1"`.
* `ENV.CHOMP_POOL_SIZE`: Set to the maximum number of jobs for Chomp, which is usually the CPU count, or the value passed to the `-j` flag when running Chomp.
### Core Templates
Some [experimental Chomp extensions](https://github.com/guybedford/chomp-extensions) are provided for the JS ecosystem, and PRs to this repo are very welcome.
These templates can be loaded via the `chomp:[name]` extension names.
By default these templates are loaded from the JSPM CDN at `https://ga.jspm.io/npm:@chompbuild/extensions@x.y.z/[name].js`.
This path can be overridden to an alternative remote URL or local path by setting the `CHOMP_CORE` environment variable.
### Chomp.addExtension(extension: string)
Extensions may load other extensions from any path or URL. Relative URLs are supported for loading extensions relative to the current extension location. Examples:
```js
Chomp.addExtension('https://site.com/extension.js');
Chomp.addExtension('./local.js');
```
Extensions are resolved to absolute URLs internally so that a given extension can only be loaded once even if `addExtension` is
called repeatedly on the same extension script.
### Chomp.registerTask(task: ChompTask)
Arbitrary tasks may be added as if they were defined in the users Chompfile. This is useful for common tasks
that are independent of the exact project, such as initialization, workflow and bootstrapping tasks.
`ChompTask` is the same interface as in the TOML definition, except that base-level kebab-case properties are
instead provided as camelCase.
Note that extension-registered tasks are not output when running template ejection via `chomp --eject`.
#### Example: Configuration Initialization Task
An example of an initialization task is to create a configuration file if it does not exist:
```js
Chomp.registerTask({
name: 'config:init',
engine: 'node',
target: 'my.config.json',
run: `
import { writeFileSync } from 'fs';
const defaultConfig = {
some: 'config'
};
// (this task only never runs when my.config.json does not exist)
writeFileSync(process.env.TARGET, JSON.stringify(defaultConfig, null, 2));
console.log(\`\${process.env.TARGET} initialized successfully.\`);
`
});
```
### Chomp.registerTemplate(name: string, template: (task: ChompTask) => ChompTask[])
Registers a template function to a template name. In the case of multiple registration, the last registered template function for a given template name will apply, permitting overrides.
Template task expansion happens early during initialization, and is independent of user options. All template tasks are expanded into
untemplated tasks internally until the final flat non-templated task list is found, which is used as the task list for the runner.
Tasks with a `template` field will call the associated registered template function with the task as the first argument to the template function. The template function can then return an array of tasks to register for the current run (whether they execute is still as defined by the task graph). Templates may return tasks that in turn use templates, which are then expanded recursively.
When `--eject` is used, this same expanded template list is saved back to the `chompfile.toml` itself to switch to an untemplated
configuration form. The `ENV.CHOMP_EJECT` global variable can be used to branch behaviour during ejection to provide a more user-suitable output where appropriate.
For template usage options validation, normal JS errors thrown will be reported with their message to the user. Template options
should be validated this way.
#### Example: Runner Template
An example of a simple run template to abstract the execution details of a task:
```toml
version = 0.1
[[task]]
name = 'template-example'
template = 'echo'
[task.template-options]
message = 'chomp chomp'
```
With execution `chomp template-example` writing `chomp chomp` to the console.
```js
Chomp.registerTemplate('echo', function (task) {
if (typeof task.templateOptions.message !== 'string')
throw new Error('Echo template expects a string message in template-options.');
if (task.run || task.engine)
throw new Error('Echo template does not expect a run or engine field.');
return [{
// task "name" and "deps" need to be passed through manually
// and similarly for tasks that define targets
name: task.name,
deps: task.deps,
run: `echo ${JSON.stringify(task.templateOptions.message)}`
}];
});
```
Templates get a lot more useful when they use the Deno or Node engines, as they can then
fully encapsulate custom wrapper code for arbitrary computations from ecosystem libraries
that do not have CLIs.
### Chomp.registerBatcher(name: string, batcher: (batch: CmdOp[], running: BatchCmd[]) => BatcherResult | undefined)
#### Overview
Batchers act as reducers of task executions into system execution calls. They allow for custom queueing and coalescing of task runs.
For example, consider a task that performs an `npm install` - only one `npm install` operation must ever run at a time, if a previous install is running the task should wait for it to finish executing first (queuing). Furthermore, if two npm installs (`npm install a` and `npm install b` say) are queued at the same time they can be combined together into a single npm install call: `npm install a b` (coalescing).
This is the primary use case for batchers - combining together task executions into singular executions where that will save time.
_Batching is a complex topic, and is more about the exact use case solutions at hand. In most cases extensions needn't worry about batching until they really need to carefully optimize and control execution invocations for performance._
#### Lifecycle
Task executions are collected as a `CmdOp` list, with batching run periodically against this list. Batchers then combine and queue executions as necessary.
Under the batching model, the lifecycle of an execution includes the following steps:
1. Task is batched as a `CmdOp` representing the execution of the task (the `run` and `engine` pair). This forms the batch command queue, `batch`, which is a fixed list for a given batching operation.
2. Every 5 milliseconds, if there are batched commands, the batcher phase is initiated on the `batch` list, where all registered batchers are each passed the `batch` queue to process it in order. They are also passed the list of running executions `running` as the second argument.
3. As each `CmdOp` in the `batch` is processed by a batcher, by being assigned by the `BatcherResult` of the batcher, it is removed from the `batch` list so the next batcher will not see it. The final _default batcher_ will just naively run the execution with simple CPU-based pooling.
#### CmdOp
The queued commands of `CmdOp` are defined as:
```typescript
interface CmdOp {
id: number,
run: string,
engine: 'deno' | 'node' | 'cmd',
name?: string,
cwd?: string,
env: Record<string, string>,
}
```
The `id` of the task operation is importantly used to key the batching process. Task executions are primarily defined by their `run` and `engine` pair.
#### BatchCmd
Batched execution commands have an almost identical interface to `CmdOp`, except with a list of `ids` of the `CmdOp` ids whose completion is fulfilled by this execution.
```typescript
interface BatchCmd {
ids: number[],
run: string,
engine: 'deno' | 'node' | 'cmd',
cwd?: string,
env: BTreeMap<string, string>,
}
```
Each `BatchCmd` real spawned execution thus corresponds to one or more `CmdOp` execution, as the reduction output of batching.
#### BatcherResult
`BatcherResult`, the return value of the batching function, forms the execution combining and queuing operation of the batcher. It has three optional return properties - `queue`, `exec` and `completionMap`. `queue` is a list of tasks to defer for the next batch queue allowing the ability to delay their execution. `exec` is the list of `BatchCmd` executions to immediately invoke. And `completionMap`, less commonly used, allows associating the completion of one of the operations in the batch to the completion of a currently running task in the already-`running` list.
```typescript
interface BatcherResult {
queue?: number[],
exec?: BatchCmd[],
completionMap?: Record<number, number>,
}
```
As soon as any batcher assigns the `id` of `CmdOp` via one of these properties, that task command is considered assigned, and removed from the batch list for the next batcher. At the end of calling all batchers, any remaining task commands in the batch list are just batched by the default batcher.
#### Example: Default Batcher
The code for the default batcher is a good example of how simple batching can work. It does not combine executions so creates one batch execution for each task execution, while respecting the job limit reflected via `ENV.CHOMP_POOL_SIZE` which ensures the CPUs is utilized efficiently:
```js
// Chomp's default batcher, without any batcher extensions:
Chomp.registerBatcher('defaultBatcher', function (batch, running) {
// If we are already running the maximum number of jobs, defer the whole batch
if (running.length >= ENV.CHOMP_POOL_SIZE)
return { defer: batch.map(({ id }) => id) };
return {
// Create a single execution for every item in the batch, up to the pool size
exec: batch.slice(0, POOL_SIZE - running.length).map(({ id, run, engine, name, cwd, env }) => ({
ids: [id],
run,
engine,
cwd,
env
})),
// Once we hit the pool size limit, defer the remaining batched executions
defer: batch.slice(POOL_SIZE - running.length).map(({ id }) => id)
};
});
```
Because batchers run one after another, having this exact above default batcher run last means it will take care of pooling so most batchers don't need to worry so much about it. Instead most batchers just focus on the specific executions they are interested in to run, queue and combine those executions they care about specifically, while within the pool limit.
Thus, most batchers are of the form:
```js
Chomp.registerBatcher('my-batcher', function (batch, running) {
if (running.length >= ENV.CHOMP_POOL_SIZE) return;
const exec = [];
for (const item of batch) {
// ignore anything not intresting to this batcher, or if we have hit the pool limit
if (!is_interesting(item) || exec.length + running.length >= POOL_SIZE) continue;
// push the batched execution we're interested in,
// usually matching it up with another to combine their executions
exec.push({ ...item, ids: [item.id] });
}
return { exec };
});
```
#### Running List and Completion Map
All commands that have already been spawned and have not returned a final status code and terminated their running OS process
are provided in the `running` list as the second argument to the batcher.
In the `running` list, each `BatchCmd` will also have an associated `id` corresponding to the batch id, which is distinct from the `CmdOp` id.
The completion map is a map from `CmdOp` id to `BatchCmd` id, which allows associating the current batch execution fulfillment with the completion of a currently executing task. This is useful for eg a generic `npm install` operation, where if an `npm install` is already running we should simply attach to that instead of queueing another `npm install`. Effectively a mapping forming a late adding to the `ids` list of that previously batched command. Because multiple `CmdOp`s can map to the same `BatchCmd` in this map structure, it supports the same type of many-to-one completion attachment as the `ids` list, just for already-running tasks as opposed to as part of the baching reduction to begin with - the difference being already running tasks cannot be altered or stopped as they have already started.
#### Combining Tasks
The coalescing of batch tasks implies reparsing the `run` or `env` vars of `CmdOp` task executions to collate them into a single `run` and `env` on a `BatchCmd` return of the `exec` property of the `BatcherResult`. It was a choice between this model, or modelling the data structure of the command calls more abstractly first, which seemed unnecessary overhead when execution parsing can suffice and with the flexibility of the JS language implementation the edge cases can generally be well handled.
See the core [Chomp extensions](https://github.com/guybedford/chomp-extensions) repo for further understanding through the direct examples of these hooks in use. The `npm.js` batcher demonstrates `npm install` batching and the `swc.js` batcher demonstrates compilation batching.
================================================
FILE: docs/task.md
================================================
# Chomp Tasks
The [`chompfile.toml`](chompfile.md) defines Chomp tasks as a list of Task objects of the form:
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'echo'
run = 'echo "Chomp"'
```
_<div style="text-align: center">An example Chompfile.</div>_
Running `chomp echo` will output the echo command.
## Task API
Tasks support the following optional properties:
* **name**: `String`, the unique task name string.
* **target**: `String`, the file path created or modified by this task. Singular sugar for a single `targets: [String]`.
* **targets**: `String[]`, the list of file paths created or modified by this task, identical to `target` when there is a single target.
* **dep**: `String`, the task names or file paths this task [depends on](#task-dependence). Singular sugar for a single `deps: [String]`.
* **deps**: `String[]`, the task names of file paths this task [depends on](#task-dependence), identical to `dep` when there is a single dependency.
* **serial**: `Boolean`, whether [task dependencies](#task-dependence) should be processed in serial order. Defaults to false for parallel task processing.
* **invalidation**: `"always" | "mtime" (default) | "not-found"`, the [task caching invalidation rules](#task-invalidation). By default a task is cached based on its target path having an mtime greater than its dependencies per "make" semantics. `"always"` never caches, and `"not-found"` will never rerun the task if the target exists.
* **display**: `"none" | "init-status" | "init-only" | "status-only" | "dot"`, defaults to `"init-status"`. Useful to reduce noise in the output log. Init is the note that the task has begun, while status is the note of task success or caching. Task errors will always be reported even with `display: 'none'`. `"dot"` outputs a dot for each run only, for a test-like output when used alongside `stdio = 'stderr-only'`.
* **echo**: `Boolean`, defaults to false - whether to echo the executed command of the task.
* **stdio**: `"none" | "no-stdin" | "stdout-only" | "stderr-only" | "all"`, defaults to `"all"` where stderr and stdout are piped to the main process output and stdin is also accepted. Set to `"no-stdin"` to disable the stdin for tasks. `"stdout-only"` and `"stderr-only"` will output only those streams.
* **engine**: `"node" | "deno" | "cmd" (default)`, the [execution engine](#task-execution) to use for the `run` string. For `node` or `deno` it is a Node.js or Deno program source string as if executed in the current directory.
* **run**: `String`, the source code string to run in the `engine`.
* **cwd**: `String`, the working directory to use for the `engine` execution.
* **env**: `{ [key: String]: String }`, custom environment variables to set for the `engine` execution.
* **env-default**: `{ [key: String]: String }`, custom default environment variables to set for the `engine` execution, only if not already present in the system environment.
* **env-replace**: `Boolean`, defaults to `true`. Whether to support `${{VAR}}` style static environment variable replacements in the `env` and `env-default` environment variable declarations and the `run` script of Shell engine tasks.
* **template**: `String`, a registered template name to use for task generation as a [template task](#extensions).
* **template-options**: `{ [option: String]: any }`, the dictionary of options to apply to the `template` [template generation](#extensions), as defined by the template itself.
* **validation**: `"none" | "ok-only" | "targets-only" | "ok-targets (default)`, Validation check to determine task success condition. The default is to check the defined targets all exist and the task exited with a success status code. `"ok-only"` just verifies the status code, `"targets-only"` just verifies the targets, and `"none"` always treats the task as successful.
## Task Execution
Chomp tasks are primarily characterized by their `"run"` and `"engine"` pair, `"run"` representing the source code of a task execution in the `"engine"` execution environment. Currently supported engines include the shell execution (the default), Node.js (`engine = 'node'`) or Deno (`engine = 'deno'`).
There are two ways to execute in Chomp:
* Execute a task by _name_ - `chomp [name]` or `chomp :[name]` where `[name]` is the `name` field of the task being run.
* Execute a task by _target_ file path - `chomp [target]` where `[target]` is the local file path to generate relative to the Chompfile being run.
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'my-task'
target = 'output.txt'
run = 'cat "Chomp Chomp" > output.txt'
```
_<div style="text-align: center">This task writes the text `Chomp Chomp` into the file at `output.txt`, defining this file as a target output path of the task so that the task is cached.</div>_
This task writes the text `Chomp Chomp` into the file at `output.txt`, defining this as a target file output of the task.
```sh
$ chomp my-task
$ chomp :my-task
$ chomp output.txt
🞂 output.txt
√ output.txt [3.8352ms]
```
_<div style="text-align: center">The same task can be called by task name (with or without `:` prefix) or by target path.</div>_
The leading `:` can be useful to disambiguate task names from file names when necessary. Setting a `name` on a task is completely optional.
Once the task has been called, with the target file already existing it will treat it as cached and skip subsequent executions:
```sh
$ chomp my-task
● output.txt [cached]
```
### Task Completion
A task is considered to have succeeded if it completes with a zero exit code, and the target or targets
expected of the task all exist.
If the spawned process returns a non-zero exit code the task and all its parents will be marked as failed.
If after completion, any of the targets defined for the task still do not exist, then the task is also marked as failed.
### Shell Tasks
The default `engine` is the shell environment - PowerShell on Windows or Bash on posix machines.
Common commands like `echo`, `pwd`, `cat`, `rm`, `cp`, `cd`, as well as operators like `$(cmd)`, `>`, `>>`, `|` form a subset of shared behaviours that can work when scripting between all platforms. With some care and testing, it is possible to write cross-platform shell task scripts. For PowerShell 5, Chomp will execute PowerShell in UTF-8 mode (applying to `>`, `>>` and `|`), although a BOM will still be output when writing a new file with `>`. Since `&&` and `||` are not supported in Powershell, multiline scripts and `;` are preferred instead.
For example, here is an SWC task (assuming Babel is installed via `npm install @swc/core @swc/cli -D`):
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'build:swc'
target = 'lib/app.js'
dep = 'src/app.ts'
run = 'swc $DEP -o $TARGET --source-maps'
```
_<div style="text-align: center">SWC task compiling the TypeScript module `src/app.ts` into a JS module `lib/app.js`, and supporting configuration in an `.swcrc` file.</div>_
The above works without having to reference the full `node_modules/.bin/swc` command prefix since `node_modules/.bin` is automatically included in the Chomp spawned `PATH`, relative to the Chompfile itself.
### Environment Variables
In addition to the `run` property, two other useful task properties are `env` and `cwd` which allow customizing the exact execution environment.
In PowerShell, defined environment variables in the task `env` are in addition made available as local variables supporting output via `$NAME` instead of `$Env:Name` for better cross-compatibility with posix shells. This process is explicit only - system-level environment variables are not given this treatment though.
In addition, static environment variable replacements are available via `${{VAR}}`, with optional spacing. Replacements that cannot be resolved to a known environment variable will be replaced with an empty string. Static replacements are available for environment variables and the shell engine run command. Set `env-replace = false` to disable static environment variable replacement for a given task.
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'env-vars'
run = '''
${{ECHO}} $PARAM1 $PARAM2
'''
[task.env]
PARAM1 = 'Chomp'
[task.default-env]
ECHO = 'echo'
PARAM2 = '${{ PARAM1 }}'
```
_<div style="text-align: center">Custom environment variables are also exposed as local variables in PowerShell, while `${{VAR}}` provides static replacements.</div>_
On both Posix and Windows, `chomp env-vars` will output: `Chomp Chomp`, unless the system has overrides of the `CMD` or `PARAM2` environment variables to alternative values.
`default-env` permits the definition of default environment variables which are only set to the default values if these environment variables are not already set in the system environment or via the global Chompfile environment variables. Just like `env`, all variables in `default-env` are also defined as PowerShell local variables, even when they are already set in the environment and the default does not apply.
The following task-level environment variables are always defined:
* `TARGET`: The path to the primary target (the interpolation target or first target).
* `TARGETS`: The `:`-separated list of target paths for multiple targets.
* `DEP`: The path to the primary dependency (the interpolation dependency or first dependency file).
* `DEPS`: The `:`-separated list of expanded dependency paths.
* `MATCH` When using [task interpolation](#task-interpolation) this provides the matched interpolation replacement value (although the `TARGET` will always be the fully substituted interpolation target for interpolation tasks).
The `PATH` environment variable is automatically extended to include `.bin` in the current folder as well as `node_modules/.bin` in the Chompfile folder.
### Node.js Engine
The `"node"` engine allows writing a Node.js program in the `run` field of a task. This is a useful way to encapsulate cross-platform build scripts which aren't possible with cross-platform shell scripting.
For example, the same SWC task in Node.js can be written:
_chompfile.toml_ls
```toml
version = 0.1
[[task]]
name = 'build:swc'
target = 'lib/app.js'
dep = 'src/app.ts'
engine = 'node'
run = '''
import swc from '@swc/core';
import { readFileSync, writeFileSync } from 'fs';
import { basename } from 'path';
const input = readFileSync(process.env.DEP, 'utf8');
const { code, map } = await swc.transform(input, {
filename: process.env.DEP,
sourceMaps: true,
jsc: {
parser: {
syntax: "typescript",
},
transform: {},
},
});
writeFileSync(process.env.TARGET, code + '\n//# sourceMappingURL=' + basename(process.env.TARGET) + '.map');
writeFileSync(process.env.TARGET + '.map', JSON.stringify(map));
'''
```
It is usually preferable to write tasks using shell scripts since they can be much faster than bootstrapping Node.js or Deno, and can more easily support [batching](extensions.md#chompregisterbatchername-string-batcher-batch-cmdop-running-batchcmd--batcherresult--undefined).
> It is usually easier to use the existing [`chomp:swc` experimental template extension](#extensions) instead of writing your own custom task for SWC.
### Deno Engine
Just like the `"node"` engine, the `"deno"` engine permits using JS to create build scripts.
The primary benefits being URL import support (no need for package management for tasks) and TypeScript type support (although unfortunately no editor plugins for Chompfiles means it doesn't translate to author time currently). Using a CDN like [JSPM.dev](https://jspm.org/docs/cdn#jspmdev) (importing eg `https://jspm.dev/[pkg]` etc) can be useful for these scripts to load npm packages.
By default the Deno engine will run with full permissions since that is generally the nature of build scripts.
## Task Interpolation
Chomp works best when each task builds a single file target, instead of having a large monolithic build.
To extend the previous example to build all of `src` into `lib`, we use **task interpolation** with a `#` or `##`, which means the same thing as a `*` or `**/*` glob respectively, but it retains the important property of being a reversible mapping which is necessary for tracing task invalidations.
Replacing `app` with `##` in the previous [SWC Shell example](#shell-tasks), we can achieve the full folder build:
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'build:swc'
target = 'lib/##.js'
dep = 'src/##.ts'
run = 'swc $DEP -o $TARGET --source-maps'
```
_<div style="text-align: center">Chomp task compiling all `.ts` files in `src` into JS modules in `lib`.</div>_
By treating each file as a separate build, we get natural build parallelization and caching where only files changed in `src` cause rebuilds.
Just like any other target, interpolation targets can be built directly (or even with globbing):
```sh
$ chomp lib/app.js
```
_<div style="text-align: center">When building an exact interpolation target, only the minimum work is done to build `lib/app.js` - no other files in `src` need to be checked other than `src/app.js`.</div>_
Only a single interpolation `dep` and `target` can be defined (with the `#` interpolation character), although additional dependencies or targets may be defined in addition by using the `deps` array instead, for example to make each compilation depend on the npm install:
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'npm:install'
run = 'npm install'
[[task]]
name = 'build:swc'
target = 'lib/##.js'
deps = ['src/##.js', 'npm:install']
run = 'swc $DEP -o $TARGET --source-maps'
```
_<div style="text-align: center">`$DEP` and `$TARGET` will always be the primary dependency and target (the interpolation item or the first in the list). Additional dependencies and targets can always be defined.</div>_
### Testing
While Chomp is not designed to be a test runner, it can easily provide many the features of one.
Tests can be run with interpolation. Since interpolation expands a glob of dependencies to operate on, this same technique can be used to create targetless tests:
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'test:unit:#'
display = 'status'
stdio = 'stderr-only'
dep = ['test/unit/#.js', 'dist/build.js']
run = 'node $DEP'
```
_<div style="text-align: center">Task interpolation without a target runs the task over all dependencies, and is always invalidated, exactly what is needed for a test runner.</div>_
In the above, all files `test/unit/**/*.js` will be expanded by the `test:unit` test resulting in a separate task run for each file. Since no `targets` are defined, the task is always invalidated and re-run.
Using the `display` and `stdio` options it is also possible to hide any test output and the command init logs in the reporter.
By using `#` in the `name` of the task, individual test or test patterns can be run by name or using glob patterns:
```sh
$ chomp --watch test:unit:some-test test:unit:some-suite-*
```
The above would run the tests `test/unit/some-test.js`, and all `test/unit/some-suite-*.js`, watching the full build graph and every unit test file for changes and rerunning the tests on change.
Alternatively all unit tests can be run by passing the empty string replacement:
```sh
$ chomp test:unit:
$ chomp test:unit:**/*
```
_<div style="text-align: center">Both lines above are equivalent given the task name `test:unit:#`, running all the unit tests.</div>_
## Task Dependence
Using the `deps` and `targets` properties (which are interchangeable with their singular forms `dep` and `target` for a single list item), task dependence graphs are built.
When processing a task, the task graph is constructed and processed in graph order where a task will not begin until its dependencies have completed processing.
Dependencies of tasks are always treated as being parallel - to ensure one task always happens before another the best way is usually to treat it as a dependency. For example by having a test task depend on the build target.
Task parallelization can be controlled by the [`-j` flag](cli.md#jobs) to set the maximum number of parallel child processes to spawn.
For example, here is a build that compiles with SWC, then builds into a single file with RollupJS:
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'npm:install'
run = 'npm install'
[[task]]
name = 'build:swc'
target = 'lib/##.js'
deps = ['src/##.js', 'npm:install']
run = 'swc $DEP -o $TARGET --source-maps'
[[task]]
name = 'build:rollup'
dep = 'lib/**/*.js'
target = 'dist/app.js'
run = 'rollup lib/app.js -d dist -m'
```
_<div style="text-align: center">Practical example of a Chomp build graph using task dependence from the npm install to per-file SWC compilation to Rollup into a single file or set of files.</div>_
Following the task graph from the top, since `build:rollup` depends on all deps in `lib`, this will make it depend on all the separate file interpolation jobs of `build:swc` and in turn their dependence. With each of the `build:swc` tasks depending on `npm:install`, this task is always run first. Then only once the `npm install` is completed successfully, the compilation of all `src/**/*.ts` into `lib/#.js` will happen with full parallelization in the task graph.
Task dependency inputs can themselves be the result of targets of other tasks. Build order is fully determined by the graph in this way.
## Watched Rebuilds
Taking the [previous example](#task-dependence) and running:
```sh
$ chomp build:rollup --watch
```
_<div style="text-align: center">Fine-grained watched rebuilds are a first-class feature in Chomp.</div>_
will build the `dist/app.js` file and then continue watching all of the input files in `src/**/*.ts` as well as the `package.json`. A change to any of these files will then trigger a granular live rebuild of only the changed TypeScript file or files.
## Static Server
As a convenience a simple local static file server is also provided:
```sh
$ chomp build-rollup --serve
```
_<div style="text-align: center">Running the Chomp static server.</div>_
This behaves identically to the watched rebuilds above, but will also serve the folder on localhost for browser and URL tests. This may seem outside of the expected features for a task runner, but it is actually closely associated with the watched rebuild events - a websocket protocol for in-browser hot-reloading is a [planned future addition](https://github.com/guybedford/chomp/issues/61).
Server configuration can be controlled via the [`serve`](cli.md#serve) options in the Chompfile or the [`--server-root`](cli.md#server-root) and [`--port`](cli.md#port) flags.
By separating monolithic builds into sub-compilations on the file system this enables caching, parallelization, finer-grained generic build control and comprehensive incremental builds with watcher support. Replacing monolithic JS build systems with make-style file caching all the commonly expected features of JS dev workflows can still be maintained.
## Task Caching
Tasks are cached when the _modified time_ of their `targets` is more recent than the modified time of their `deps` per standard Make-style semantics.
For example, if we change the npm task definition from the previous example to define the `dep` as the `package.json` and the `target` as the `package-lock.json`:
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'npm:install'
run = 'npm install'
target = 'package-lock.json'
dep = 'package.json'
```
The `npm install` operation will now be treated as cached and skipped, unless the `package.json` has been more recently modified than the `package-lock.json`.
The invalidation rule is a binary rule indicating whether or not a given task should rerun or be treated as cached.
The explicit rules of invalidation for this `mtime` invalidation are:
* If no targets are defined for a task, it is always invalidated.
* Otherwise, if no deps are defined for a task, it is invalidated only if the targets do not exist.
* Otherwise, if the mtime of any dep is greater than the mtime of any target, the task is invalidated.
Task invalidation can be customized with the `invalidation` property on a task:
* `invalidation = 'mtime'` _(default)_: This is the default invalidation, as per the rules described above.
* `invalidation = 'always'`: The task is always invalidated and rerun, without exception.
* `invalidation = 'not-found'`: The task is only invalidated when not all targets are defined.
## Serial Dependencies
In some cases, it can be preferred to write a serial pipeline of steps that should be followed.
This can be achieved by setting `serial = true` on the task:
_chompfile.toml_
```toml
version = 0.1
[[task]]
name = 'test'
serial = true
deps = ['test:a', 'test:b', 'test:c']
[[tas]]
name = 'test:a'
run = 'echo a'
[[task]]
name = 'test:b'
run = 'echo b'
[[task]]
name = 'test:c'
run = 'echo c'
```
_<div style="text-align: center">Example of a serial `test` task executing `test:a` then `test:b` then `testc` in sequence.</div>_
Running `chomp test` with the above, will run each of `test:a`, `test:b` and `test:c` one after the other to completion instead of running their dependence graphs in parallel by default, logging `a b c` every time.
## Extensions
Extensions are loaded via the `extensions` list in the Chompfile, and can define custom task templates, which can encapsulate the details of a task execution into a simpler definition.
For convenience Chomp provides an experimental [core extensions library](https://github.com/guybedford/chomp-extensions).
For example, to replace the npm, SWC and RollupJS compilations from the previous examples with their extension templates:
_chompfile.toml_
```toml
version = 0.1
extensions = ['chomp:npm', 'chomp:swc', 'chomp:rollup']
[[task]]
name = 'npm:install'
template = 'npm'
[[task]]
name = 'build:swc'
template = 'swc'
target = 'lib/##.js'
deps = ['src/##.js', 'npm:install']
[task.template-options]
source-maps = true
[[task]]
name = 'build:rollup'
template = 'rollup'
deps = 'lib/**/*.js'
[task.template-options]
outdir = 'dist'
entries = ['lib/app.js']
```
_<div style="text-align: center">Using the `chomp:npm`, `chomp:swc` and `chomp:rollup` [experimental core extensions](https://github.com/guybedford/chomp-extensions) allows writing these tasks encapsulated from their implementations.</div>_
Templates can be loaded from any file path or URL.
### Remote Extensions
Extensions support any `https://` URLs or local file paths.
Remote extensions are loaded once and cached locally by Chomp, regardless of cache headers, to ensure the fastest run time.
The remote extension cache can be cleared by running `chomp --clear-cache`.
### Ejection
`chomp --eject` transforms the Chompfile into the expanded untemplated form without extensions, allowing an opt-out from extension template workflows if it ever feels too magical. In this way templates become a sort of task construction utility.
### Writing Templates
> Read more on writing extensions in the [extensions documentation](extensions.md)
Chomp extensions can be loaded from any URL or local file path. To write custom templates, create a local extension file `local-extension.js` referencing it in the extensions list of the Chompfile:
_chompfile.toml_
```toml
version = 0.1
extensions = ['./local-extension.js']
```
_local-extension.js_
```js
Chomp.registerTemplate('npm', function (task) {
return [{
name: task.name,
run: 'npm install',
target: 'package-lock.json',
deps: ['package.json']
}];
});
Chomp.registerTemplate('swc', function (task) {
const { sourceMaps } = task.templateOptions;
return [{
name: task.name,
targets: task.targets,
deps: task.deps,
run: `swc $DEP -o $TARGET${sourceMaps ? ' --source-maps' : ''}`
}];
});
Chomp.registerTemplate('rollup', function (task) {
if (task.targets.length > 0)
throw new Error('Targets is not supported by the Rollup template, use the "outdir" and "entries" template options instead.');
const { outdir, entries } = task.templateOptions;
const targets = entries.map(entry => outdir + '/' + entry.split('/').pop());
return [{
name: task.name,
deps: task.deps,
targets,
run: `rollup ${entries.join(' ')} -d ${outdir} -m`
}];
});
```
_<div style="text-align: center">Chomp extension template registration example loaded via a local extension at `local-extension.js` for the `npm`, `swc` and `rollup` templates.</div>_
Templates are functions on tasks returning a new list of tasks. All TOML properties apply but with _camelCase_ instead of _kebab-case_.
PRs to the default Chomp extensions library are welcome, or host your own on your own domain or via an npm CDN. For support on the JSPM CDN, add `"type": "script"` to the `package.json` of the package to avoid incorrect processing since template extensions are currently scripts and not modules.
Because remote extensions are cached, it is recommended to always use unique URLs with versions when hosting extensions remotely.
See the extensions documentation for the full [extensions API](extensions.md#api).
================================================
FILE: node-chomp/README.md
================================================
# Chomp Node
Node.js wrapper for [Chomp](https://chompbuild.com)
Usable via your favourite JS package manager:
```
npm install -g chomp
```
Chomp should then be usable as a CLI through npm:
```
chomp --version
```
Note: Installing Chomp via `cargo install chompbuild` is the recommended Chomp installation workflow for performance, or via [downloading the binaries into your path](https://github.com/guybedford/chomp/releases).
================================================
FILE: node-chomp/index.js
================================================
#!/usr/bin/env node
import BinWrapper from 'bin-wrapper';
import { readFileSync } from 'fs';
import { spawn } from 'child_process';
import { fileURLToPath } from 'url';
let { version } = JSON.parse(readFileSync(new URL('package.json', import.meta.url), 'utf8'));
if (version.match(/-rebuild(\.\d)?$/))
version = version.split('-rebuild')[0];
const base = `https://github.com/guybedford/chomp/releases/download/${version}`
const bin = new BinWrapper({ skipCheck: true })
.src(`${base}/chomp-macos-${version}.tar.gz`, 'darwin')
.src(`${base}/chomp-linux-${version}.tar.gz`, 'linux', 'x64')
.src(`${base}/chomp-windows-${version}.zip`, 'win32', 'x64')
.dest(fileURLToPath(new URL('./vendor', import.meta.url)))
.use(process.platform === 'win32' ? 'chomp.exe' : 'chomp')
.version(version);
await bin.run();
spawn(bin.path(), process.argv.slice(2), { stdio: 'inherit' });
================================================
FILE: node-chomp/package.json
================================================
{
"name": "chomp",
"version": "0.3.0",
"description": "'JS Make' - parallel task runner CLI for the frontend ecosystem with a JS extension system",
"bin": {
"chomp": "index.js"
},
"type": "module",
"keywords": [
"task",
"runner",
"build",
"development",
"make"
],
"author": "Guy Bedford",
"license": "Apache-2.0",
"dependencies": {
"bin-wrapper": "^4.1.0"
},
"main": "index.js",
"repository": {
"type": "git",
"url": "git+https://github.com/guybedford/chomp.git"
},
"bugs": {
"url": "https://github.com/guybedford/chomp/issues"
},
"files": [
"index.js"
],
"homepage": "https://github.com/guybedford/chomp#readme"
}
================================================
FILE: src/ansi_windows.rs
================================================
/// Enables ANSI code support on Windows 10.
///
/// This uses Windows API calls to alter the properties of the console that
/// the program is running in.
///
/// https://msdn.microsoft.com/en-us/library/windows/desktop/mt638032(v=vs.85).aspx
///
/// Returns a `Result` with the Windows error code if unsuccessful.
#[cfg(windows)]
pub fn enable_ansi_support() -> Result<(), u32> {
// ref: https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#EXAMPLE_OF_ENABLING_VIRTUAL_TERMINAL_PROCESSING @@ https://archive.is/L7wRJ#76%
use std::ffi::OsStr;
use std::iter::once;
use std::os::windows::ffi::OsStrExt;
use std::ptr::null_mut;
use winapi::um::consoleapi::{GetConsoleMode, SetConsoleMode};
use winapi::um::errhandlingapi::GetLastError;
use winapi::um::fileapi::{CreateFileW, OPEN_EXISTING};
use winapi::um::handleapi::INVALID_HANDLE_VALUE;
use winapi::um::winnt::{FILE_SHARE_WRITE, GENERIC_READ, GENERIC_WRITE};
const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004;
unsafe {
// ref: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
// Using `CreateFileW("CONOUT$", ...)` to retrieve the console handle works correctly even if STDOUT and/or STDERR are redirected
let console_out_name: Vec<u16> =
OsStr::new("CONOUT$").encode_wide().chain(once(0)).collect();
let console_handle = CreateFileW(
console_out_name.as_ptr(),
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_WRITE,
null_mut(),
OPEN_EXISTING,
0,
null_mut(),
);
if console_handle == INVALID_HANDLE_VALUE {
return Err(GetLastError());
}
// ref: https://docs.microsoft.com/en-us/windows/console/getconsolemode
let mut console_mode: u32 = 0;
if 0 == GetConsoleMode(console_handle, &mut console_mode) {
return Err(GetLastError());
}
// VT processing not already enabled?
if console_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0 {
// https://docs.microsoft.com/en-us/windows/console/setconsolemode
if 0 == SetConsoleMode(
console_handle,
console_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING,
) {
return Err(GetLastError());
}
}
}
Ok(())
}
================================================
FILE: src/chompfile.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::Result;
use directories::UserDirs;
use regex::{Captures, Regex};
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
path::{Component, Path, PathBuf},
};
#[derive(Copy, Clone, Debug, Serialize, Deserialize, Hash, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
#[derive(Default)]
pub enum ChompEngine {
#[default]
Shell,
Node,
Deno,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, Hash, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
#[derive(Default)]
pub enum TaskDisplay {
None,
Dot,
#[default]
InitStatus,
StatusOnly,
InitOnly,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, Hash, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
#[derive(Default)]
pub enum TaskStdio {
#[default]
All,
NoStdin,
StdoutOnly,
StderrOnly,
None,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields, rename_all = "kebab-case")]
pub struct Chompfile {
pub version: f32,
#[serde(default, skip_serializing_if = "is_default")]
pub echo: bool,
pub default_task: Option<String>,
#[serde(default, skip_serializing_if = "is_default")]
pub extensions: Vec<String>,
#[serde(default, skip_serializing_if = "is_default")]
pub env: HashMap<String, String>,
#[serde(default, skip_serializing_if = "is_default")]
pub env_default: HashMap<String, String>,
#[serde(default, skip_serializing_if = "is_default")]
pub server: ServerOptions,
#[serde(default, skip_serializing_if = "is_default")]
pub task: Vec<ChompTaskMaybeTemplated>,
#[serde(default, skip_serializing_if = "is_default")]
pub template_options: HashMap<String, HashMap<String, toml::value::Value>>,
}
#[derive(Debug, Serialize, PartialEq, Deserialize, Clone)]
pub struct ServerOptions {
#[serde(default = "default_root", skip_serializing_if = "is_default")]
pub root: String,
#[serde(default = "default_port", skip_serializing_if = "is_default")]
pub port: u16,
}
fn default_root() -> String {
".".to_string()
}
fn default_port() -> u16 {
5776
}
impl Default for ServerOptions {
fn default() -> Self {
ServerOptions {
root: ".".to_string(),
port: default_port(),
}
}
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
#[serde(rename_all = "kebab-case")]
#[derive(Default)]
pub enum InvalidationCheck {
NotFound,
#[default]
Mtime,
Always,
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
#[serde(rename_all = "kebab-case")]
#[derive(Default)]
pub enum ValidationCheck {
#[default]
OkTargets,
TargetsOnly,
OkOnly,
NotOk,
None,
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
#[serde(rename_all = "kebab-case")]
#[derive(Default)]
pub enum WatchInvalidation {
#[default]
RestartRunning,
SkipRunning,
}
#[derive(Debug, Serialize, PartialEq, Deserialize, Clone)]
#[serde(deny_unknown_fields, rename_all = "kebab-case")]
pub struct ChompTaskMaybeTemplated {
pub name: Option<String>,
pub target: Option<String>,
pub targets: Option<Vec<String>>,
pub dep: Option<String>,
pub deps: Option<Vec<String>>,
pub args: Option<Vec<String>>,
pub serial: Option<bool>,
pub watch_invalidation: Option<WatchInvalidation>,
pub invalidation: Option<InvalidationCheck>,
pub validation: Option<ValidationCheck>,
pub display: Option<TaskDisplay>,
pub stdio: Option<TaskStdio>,
pub engine: Option<ChompEngine>,
pub run: Option<String>,
pub cwd: Option<String>,
pub env_replace: Option<bool>,
pub template: Option<String>,
pub echo: Option<bool>,
pub template_options: Option<HashMap<String, toml::value::Value>>,
pub env: Option<HashMap<String, String>>,
pub env_default: Option<HashMap<String, String>>,
}
impl ChompTaskMaybeTemplated {
pub fn new() -> Self {
ChompTaskMaybeTemplated {
name: None,
run: None,
args: None,
cwd: None,
deps: None,
dep: None,
targets: None,
target: None,
display: None,
engine: None,
env_replace: None,
env: None,
env_default: None,
echo: None,
invalidation: None,
validation: None,
serial: None,
stdio: None,
template: None,
template_options: None,
watch_invalidation: None,
}
}
pub fn targets_vec(&self, cwd: &str) -> Result<Vec<String>> {
if let Some(ref target) = self.target {
let target_str = resolve_path(target, cwd);
Ok(vec![target_str])
} else if let Some(ref targets) = self.targets {
let targets = targets.iter().map(|t| resolve_path(t, cwd)).collect();
Ok(targets)
} else {
Ok(vec![])
}
}
pub fn deps_vec(&self, chompfile: &Chompfile, cwd: &str) -> Result<Vec<String>> {
let names = chompfile
.task
.iter()
.filter_map(|t| t.name.as_ref())
.collect::<Vec<_>>();
if let Some(ref dep) = self.dep {
let dep_str = if names.contains(&dep) || skip_special_chars(dep) {
dep.to_string()
} else {
resolve_path(dep, cwd)
};
Ok(vec![dep_str])
} else if let Some(ref deps) = self.deps {
let deps = deps
.iter()
.map(|dep| {
if names.contains(&dep) || skip_special_chars(dep) {
dep.to_owned()
} else {
resolve_path(dep, cwd)
}
})
.collect();
Ok(deps)
} else {
Ok(vec![])
}
}
}
fn skip_special_chars(s: &str) -> bool {
s.contains(':') || s.contains("&prev") || s.contains("&next")
}
fn is_default<T: Default + PartialEq>(t: &T) -> bool {
t == &T::default()
}
#[derive(Debug, Serialize, PartialEq, Deserialize, Clone)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct ChompTaskMaybeTemplatedJs {
pub name: Option<String>,
pub target: Option<String>,
pub targets: Option<Vec<String>>,
pub dep: Option<String>,
pub deps: Option<Vec<String>>,
pub args: Option<Vec<String>>,
pub serial: Option<bool>,
pub invalidation: Option<InvalidationCheck>,
pub validation: Option<ValidationCheck>,
pub watch_invalidation: Option<WatchInvalidation>,
pub display: Option<TaskDisplay>,
pub stdio: Option<TaskStdio>,
pub engine: Option<ChompEngine>,
pub run: Option<String>,
pub cwd: Option<String>,
pub echo: Option<bool>,
pub env_replace: Option<bool>,
pub template: Option<String>,
pub template_options: Option<HashMap<String, toml::value::Value>>,
pub env: Option<HashMap<String, String>>,
pub env_default: Option<HashMap<String, String>>,
}
impl From<ChompTaskMaybeTemplatedJs> for ChompTaskMaybeTemplated {
fn from(val: ChompTaskMaybeTemplatedJs) -> Self {
ChompTaskMaybeTemplated {
cwd: val.cwd,
name: val.name,
args: val.args,
target: val.target,
targets: val.targets,
display: val.display,
stdio: val.stdio,
invalidation: val.invalidation,
validation: val.validation,
dep: val.dep,
deps: val.deps,
echo: val.echo,
serial: val.serial,
env_replace: val.env_replace,
env: val.env,
env_default: val.env_default,
run: val.run,
engine: val.engine,
template: val.template,
template_options: val.template_options,
watch_invalidation: val.watch_invalidation,
}
}
}
pub fn resolve_path(target: &str, cwd: &str) -> String {
path_from(cwd, target).to_string_lossy().replace('\\', "/")
}
/// https://stackoverflow.com/questions/68231306/stdfscanonicalize-for-files-that-dont-exist
/// build a usable path from a user input which may be absolute
/// (if it starts with / or ~) or relative to the supplied base_dir.
/// (we might want to try detect windows drives in the future, too)
pub fn path_from<P: AsRef<Path>>(base_dir: P, input: &str) -> PathBuf {
let tilde = Regex::new(r"^~(/|$)").unwrap();
if input.starts_with('/') {
// if the input starts with a `/`, we use it as is
input.into()
} else if tilde.is_match(input) {
// if the input starts with `~` as first token, we replace
// this `~` with the user home directory
PathBuf::from(&*tilde.replace(input, |c: &Captures| {
if let Some(user_dirs) = UserDirs::new() {
format!("{}{}", user_dirs.home_dir().to_string_lossy(), &c[1],)
} else {
// warn!("no user dirs found, no expansion of ~");
c[0].to_string()
}
}))
} else {
// we put the input behind the source (the selected directory
// or its parent) and we normalize so that the user can type
// paths with `../`
normalize_path(base_dir.as_ref().join(input))
}
}
/// Improve the path to try remove and solve .. token.
///
/// This assumes that `a/b/../c` is `a/c` which might be different from
/// what the OS would have chosen when b is a link. This is OK
/// for broot verb arguments but can't be generally used elsewhere
///
/// This function ensures a given path ending with '/' still
/// ends with '/' after normalization.
pub fn normalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
let ends_with_slash = path.as_ref().to_str().is_some_and(|s| s.ends_with('/'));
let mut normalized = PathBuf::new();
for component in path.as_ref().components() {
match &component {
Component::ParentDir => {
if !normalized.pop() {
normalized.push(component);
}
}
_ => {
normalized.push(component);
}
}
}
if ends_with_slash {
normalized.push("");
}
normalized
}
================================================
FILE: src/engines/cmd.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::chompfile::TaskStdio;
use crate::engines::BatchCmd;
use regex::Regex;
use std::collections::BTreeMap;
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use tokio::process::{Child, Command};
fn replace_env_vars(arg: &str, env: &BTreeMap<String, String>) -> String {
let mut out_arg = arg.to_string();
if out_arg.find('$').is_none() {
return out_arg;
}
for (name, value) in env {
if !out_arg.contains(name) {
continue;
}
let mut env_str = String::from("$");
env_str.push_str(name);
if out_arg.contains(&env_str) {
out_arg = out_arg.replace(&env_str, value);
if out_arg.find('$').is_none() {
return out_arg;
}
}
let mut env_str_curly = String::from("${");
env_str_curly.push_str(name);
env_str_curly.push('}');
if out_arg.contains(&env_str_curly) {
out_arg = out_arg.replace(&env_str_curly, value);
if out_arg.find('$').is_none() {
return out_arg;
}
}
}
for (name, value) in env::vars() {
let name = name.to_uppercase();
if !out_arg.contains(&name) {
continue;
}
let mut env_str = String::from("$");
env_str.push_str(&name);
if out_arg.contains(&env_str) {
out_arg = out_arg.replace(&env_str, &value);
if out_arg.find('$').is_none() {
return out_arg;
}
}
let mut env_str_curly = String::from("${");
env_str_curly.push_str(&name);
env_str_curly.push('}');
if out_arg.contains(&env_str_curly) {
out_arg = out_arg.replace(&env_str_curly, &value);
if out_arg.find('$').is_none() {
return out_arg;
}
}
}
out_arg
}
fn set_cmd_stdio(command: &mut Command, stdio: TaskStdio) {
match stdio {
TaskStdio::All => {}
TaskStdio::StderrOnly => {
command.stdin(Stdio::null());
command.stdout(Stdio::null());
}
TaskStdio::StdoutOnly => {
command.stdin(Stdio::null());
command.stderr(Stdio::null());
}
TaskStdio::NoStdin => {
command.stdin(Stdio::null());
}
TaskStdio::None => {
command.stdin(Stdio::null());
command.stdout(Stdio::null());
command.stderr(Stdio::null());
}
};
}
#[cfg(target_os = "windows")]
pub fn create_cmd(
cwd: &str,
path: &str,
batch_cmd: &BatchCmd,
fastpath_fallback: bool,
) -> Option<Child> {
let run = batch_cmd.run.trim();
lazy_static! {
static ref CMD: Regex = Regex::new(
"(?x)
^(?P<cmd>[^`~!\\#&*()\t\\{\\[|;'\"\\n<>?\\\\\\ ]+?)
(?P<args>(?:\\ (?:
[^`~!\\#&*()\t\\{\\[|;'\"\\n<>?\\\\\\ ]+ |
(?:\"[^\"\\n\\\\]*?\") |
(?:'[^'\"\\n\\\\]*?')
)*?)*?)$
"
)
.unwrap();
static ref ARGS: Regex = Regex::new(
"(?x)
\\ (?:[^`~!\\#&*()\t\\{\\[|;'\"\\n<>?\\\\\\ ]+ |
(?:\"[^\"\\n\\\\]*?\") |
(?:'[^'\"\\n\\\\]*?'))
"
)
.unwrap();
}
if batch_cmd.echo {
println!("{}", &run);
}
// fast path for direct commands to skip the shell entirely
if let Some(capture) = CMD.captures(run) {
let mut cmd = String::from(&capture["cmd"]);
let mut do_spawn = true;
// Path-like must be exact
if cmd.contains('/') || cmd.contains('\\') {
// canonicalize returns UNC...
let cmd_buf = PathBuf::from(&cmd);
let cmd_buf = if Path::is_absolute(&cmd_buf) {
cmd_buf
} else {
let mut buf = PathBuf::from(&cwd);
buf.push(cmd_buf);
buf
};
if let Ok(unc_path) = fs::canonicalize(cmd_buf) {
let unc_str = unc_path.to_str().unwrap();
if unc_str.starts_with(r"\\?\") {
cmd = String::from(&unc_path.to_str().unwrap()[4..]);
} else {
do_spawn = false;
}
} else {
do_spawn = false;
}
}
if do_spawn {
// Try ".cmd" extension first
// Note: this requires latest Rust version
let mut cmd_with_ext = cmd.to_owned();
cmd_with_ext.push_str(".cmd");
let mut command = Command::new(&cmd_with_ext);
command.env("PATH", path);
for (name, value) in &batch_cmd.env {
command.env(name, value);
}
command.current_dir(cwd);
for arg in ARGS.captures_iter(&capture["args"]) {
let arg = arg.get(0).unwrap().as_str();
let first_char = arg.as_bytes()[1];
let arg_str = if first_char == b'\'' || first_char == b'"' {
&arg[2..arg.len() - 1]
} else {
&arg[1..arg.len()]
};
if !batch_cmd.env.is_empty() {
command.arg(replace_env_vars(arg_str, &batch_cmd.env));
} else {
command.arg(arg_str);
}
}
set_cmd_stdio(&mut command, batch_cmd.stdio.unwrap_or_default());
match command.spawn() {
Ok(child) => return Some(child),
Err(_) => {
let mut command = Command::new(&cmd);
command.env("PATH", path);
for (name, value) in &batch_cmd.env {
command.env(name, value);
}
command.current_dir(cwd);
for arg in ARGS.captures_iter(&capture["args"]) {
let arg = arg.get(0).unwrap().as_str();
let first_char = arg.as_bytes()[1];
let arg_str = if first_char == b'\'' || first_char == b'"' {
&arg[2..arg.len() - 1]
} else {
&arg[1..arg.len()]
};
if !batch_cmd.env.is_empty() {
command.arg(replace_env_vars(arg_str, &batch_cmd.env));
} else {
command.arg(arg_str);
}
}
set_cmd_stdio(&mut command, batch_cmd.stdio.unwrap_or_default());
match command.spawn() {
Ok(child) => return Some(child),
Err(_) => {
if !fastpath_fallback {
return None;
}
} // fallback to shell
}
}
};
}
}
let shell = if env::var("PSModulePath").is_ok() {
"powershell"
} else {
panic!("Powershell is required on Windows for arbitrary scripts");
// "cmd"
};
let mut command = Command::new(shell);
if shell == "powershell" {
command.arg("-ExecutionPolicy");
command.arg("Unrestricted");
command.arg("-NonInteractive");
command.arg("-NoLogo");
// ensure file operations use UTF8
let mut run_str = String::from(
"$PSDefaultParameterValues['Out-File:Encoding']='utf8';$ErrorActionPreference='Stop';",
);
// we also set _custom_ variables as local variables for easy substitution
for (name, value) in &batch_cmd.env {
run_str.push_str(&format!("${}='{}';", name, value.replace("'", "''")));
}
run_str.push('\n');
run_str.push_str(run);
command.arg(run_str);
} else {
command.arg("/d");
// command.arg("/s");
command.arg("/c");
command.arg(run);
}
command.env("PATH", path);
for (name, value) in &batch_cmd.env {
command.env(name, value);
}
command.current_dir(cwd);
set_cmd_stdio(&mut command, batch_cmd.stdio.unwrap_or_default());
Some(command.spawn().unwrap())
}
#[cfg(not(target_os = "windows"))]
pub fn create_cmd(
cwd: &str,
path: &str,
batch_cmd: &BatchCmd,
fastpath_fallback: bool,
) -> Option<Child> {
let run = batch_cmd.run.trim();
lazy_static! {
static ref CMD: Regex = Regex::new(
"(?x)
^(?P<cmd>[^`~!\\#&*()\t\\{\\[|;'\"\\n<>?\\\\\\ ]+?)
(?P<args>(?:\\ (?:
[^`~!\\#&*()\t\\{\\[|;'\"\\n<>?\\\\\\ ]+ |
(?:\"[^\"\\n\\\\]*?\") |
(?:'[^'\"\\n\\\\]*?')
)*?)*?)$
"
)
.unwrap();
static ref ARGS: Regex = Regex::new(
"(?x)
\\ (?:[^`~!\\#&*()\t\\{\\[|;'\"\\n<>?\\\\\\ ]+ |
(?:\"[^\"\\n\\\\]*?\") |
(?:'[^'\"\\n\\\\]*?'))
"
)
.unwrap();
}
if batch_cmd.echo {
println!("{}", run);
}
// Spawn needs an exact path for Ubuntu?
// fast path for direct commands to skip the shell entirely
if let Some(capture) = CMD.captures(&run) {
let mut cmd = capture["cmd"].to_string();
let mut do_spawn = true;
// Path-like must be exact
if cmd.contains("/") {
let cmd_buf = PathBuf::from(&cmd);
let cmd_buf = if Path::is_absolute(&cmd_buf) {
cmd_buf
} else {
let mut buf = PathBuf::from(&cwd);
buf.push(cmd_buf);
buf
};
if let Ok(canonical) = fs::canonicalize(cmd_buf) {
cmd = String::from(&canonical.to_str().unwrap()[4..]);
} else {
do_spawn = false;
}
}
if do_spawn {
let mut command = Command::new(&cmd);
command.env("PATH", &path);
for (name, value) in &batch_cmd.env {
command.env(name, value);
}
command.current_dir(cwd);
for arg in ARGS.captures_iter(&capture["args"]) {
let arg = arg.get(0).unwrap().as_str();
let first_char = arg.as_bytes()[1];
let arg_str = if first_char == b'\'' || first_char == b'"' {
&arg[2..arg.len() - 1]
} else {
&arg[1..arg.len()]
};
if batch_cmd.env.len() > 0 {
command.arg(replace_env_vars(arg_str, &batch_cmd.env));
} else {
command.arg(arg_str);
}
}
set_cmd_stdio(&mut command, batch_cmd.stdio.unwrap_or_default());
match command.spawn() {
Ok(child) => return Some(child),
Err(_) => {
if !fastpath_fallback {
return None;
}
} // fallback to shell
}
}
}
let mut command = Command::new("bash");
command.env("PATH", path);
for (name, value) in &batch_cmd.env {
command.env(name, value);
}
command.current_dir(cwd);
command.arg("-e");
command.arg("-c");
command.arg(&run);
set_cmd_stdio(&mut command, batch_cmd.stdio.unwrap_or_default());
Some(command.spawn().unwrap())
}
================================================
FILE: src/engines/deno.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::engines::check_target_mtimes;
use crate::engines::create_cmd;
use crate::engines::CmdPool;
use crate::engines::Exec;
use crate::engines::{BatchCmd, ExecState};
use futures::future::FutureExt;
use std::env;
use std::time::Instant;
use tokio::fs;
use uuid::Uuid;
const DENO_CMD: &str = "deno run -A --unstable --no-check $CHOMP_MAIN";
pub fn deno_runner(cmd_pool: &mut CmdPool, mut cmd: BatchCmd, targets: Vec<String>) {
let start_time = Instant::now();
let uuid = Uuid::new_v4();
let mut tmp_file = env::temp_dir();
tmp_file.push(format!("{}.ts", uuid.as_simple()));
let tmp_file2 = tmp_file.clone();
cmd.env.insert(
"CHOMP_MAIN".to_string(),
tmp_file.to_str().unwrap().to_string(),
);
cmd.env.insert(
"CHOMP_PATH".to_string(),
std::env::args().next().unwrap().to_string(),
);
let targets = targets.clone();
let write_future = fs::write(tmp_file, cmd.run.to_string());
cmd.run = DENO_CMD.to_string();
let exec_num = cmd_pool.exec_num;
cmd_pool.exec_cnt += 1;
let pool = cmd_pool as *mut CmdPool;
let echo = cmd.echo;
cmd.echo = false;
let child = create_cmd(
cmd.cwd.as_ref().unwrap_or(&cmd_pool.cwd),
&cmd_pool.path,
&cmd,
false,
);
let future = async move {
let cmd_pool = unsafe { &mut *pool };
let exec = &mut cmd_pool.execs.get_mut(&exec_num).unwrap();
write_future.await.expect("unable to write temporary file");
exec.child.as_ref()?;
if echo {
println!("<Deno exec>");
}
exec.state = match exec.child.as_mut().unwrap().wait().await {
Ok(status) => {
if status.success() {
ExecState::Completed
} else {
ExecState::Failed
}
}
Err(e) => match exec.state {
ExecState::Terminating => ExecState::Terminated,
_ => panic!("Unexpected exec error {:?}", e),
},
};
cmd_pool.exec_cnt -= 1;
fs::remove_file(&tmp_file2)
.await
.expect("unable to cleanup tmp file");
let end_time = Instant::now();
// finally we verify that the targets exist
let mtime = check_target_mtimes(targets, true).await;
Some((exec.state, mtime, end_time - start_time))
}
.boxed_local()
.shared();
cmd_pool.execs.insert(
exec_num,
Exec {
cmd,
child,
future,
state: ExecState::Executing,
},
);
cmd_pool.exec_num += 1;
}
================================================
FILE: src/engines/mod.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
mod cmd;
mod deno;
mod node;
use crate::chompfile::ChompEngine;
use crate::chompfile::TaskStdio;
use crate::engines::deno::deno_runner;
use crate::engines::node::node_runner;
use crate::extensions::BatcherResult;
use crate::task::check_target_mtimes;
use crate::ExtensionEnvironment;
use anyhow::Result;
use anyhow::{anyhow, Error};
use cmd::create_cmd;
use futures::future::Shared;
use futures::future::{Future, FutureExt};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashSet;
use std::env;
use std::path::Path;
use std::pin::Pin;
use std::rc::Rc;
use std::time::Duration;
use std::time::Instant;
use tokio::fs;
use tokio::process::Child;
use tokio::time::sleep;
pub fn replace_env_vars_static(arg: &str, env: &BTreeMap<String, String>) -> String {
let mut out_arg = String::new();
let mut pos = 0;
while let Some(idx) = arg[pos..].find("${{") {
let close_idx = match arg[pos + idx + 3..].find("}}") {
Some(idx) => idx,
None => {
out_arg.push_str("${{");
pos = pos + idx + 3;
continue;
}
} + pos
+ idx
+ 3;
let var_str = arg[pos + idx + 3..close_idx].trim();
out_arg.push_str(&arg[pos..pos + idx]);
if let Some(replacement) = env.get(var_str) {
out_arg.push_str(replacement);
} else {
if let Ok(replacement) = std::env::var(var_str) {
out_arg.push_str(&replacement);
}
}
pos = close_idx + 2;
}
out_arg.push_str(&arg[pos..]);
out_arg
}
pub struct CmdPool<'a> {
cmd_num: usize,
pub extension_env: &'a mut ExtensionEnvironment,
cmds: BTreeMap<usize, CmdOp>,
exec_num: usize,
execs: BTreeMap<usize, Exec<'a>>,
exec_cnt: usize,
batching: BTreeSet<usize>,
cmd_execs: BTreeMap<usize, usize>,
cwd: String,
path: String,
pool_size: usize,
batch_future: Option<Shared<Pin<Box<dyn Future<Output = Result<(), Rc<Error>>> + 'a>>>>,
}
#[derive(Hash, Serialize, PartialEq, Eq, Debug)]
pub struct CmdOp {
pub name: Option<String>,
pub id: usize,
pub run: String,
pub env: BTreeMap<String, String>,
pub cwd: Option<String>,
pub engine: ChompEngine,
pub stdio: TaskStdio,
pub targets: Vec<String>,
pub echo: bool,
}
#[derive(Debug, Deserialize, Serialize, Hash, PartialEq, Eq)]
pub struct BatchCmd {
pub id: Option<usize>,
pub run: String,
#[serde(default)]
pub echo: bool,
pub env: BTreeMap<String, String>,
pub cwd: Option<String>,
pub engine: ChompEngine,
pub stdio: Option<TaskStdio>,
pub ids: Vec<usize>,
}
#[derive(Debug, Clone, Copy)]
pub enum ExecState {
Executing,
Completed,
Failed,
Terminating,
Terminated,
}
#[derive(Debug)]
pub struct Exec<'a> {
cmd: BatchCmd,
child: Option<Child>,
state: ExecState,
future:
Shared<Pin<Box<dyn Future<Output = Option<(ExecState, Option<Duration>, Duration)>> + 'a>>>,
}
impl<'a> CmdPool<'a> {
pub fn new(
pool_size: usize,
cwd: String,
extension_env: &'a mut ExtensionEnvironment,
) -> CmdPool<'a> {
#[cfg(not(target_os = "windows"))]
let path = {
let mut path = String::from(&cwd);
path += "/.bin:";
path.push_str(&cwd);
path += "/node_modules/.bin";
path += ":";
path.push_str(&env::var("PATH").unwrap_or_default());
path
};
#[cfg(target_os = "windows")]
let path = {
let mut path = cwd.replace('/', "\\");
path += "\\.bin;";
path.push_str(&cwd.replace('/', "\\"));
path += "\\node_modules\\.bin;";
path.push_str(&env::var("PATH").unwrap_or_default());
path
};
CmdPool {
cmd_num: 0,
cwd,
path,
cmds: BTreeMap::new(),
exec_num: 0,
exec_cnt: 0,
execs: BTreeMap::new(),
pool_size,
extension_env,
batching: BTreeSet::new(),
cmd_execs: BTreeMap::new(),
batch_future: None,
}
}
pub fn terminate(&mut self, cmd_num: usize, name: &str) {
// Note: On Windows, terminating a process does not terminate
// the child processes, which can leave zombie processes behind
println!("Terminating {}...", name);
let exec_num = self.cmd_execs.get(&cmd_num).unwrap();
let exec = &mut self.execs.get_mut(exec_num).unwrap();
if matches!(exec.state, ExecState::Executing) {
exec.state = ExecState::Terminating;
let child = exec.child.as_mut().unwrap();
child.start_kill().expect("Unable to terminate process");
}
}
pub fn get_exec_future(
&mut self,
cmd_num: usize,
) -> Pin<
Box<dyn Future<Output = Result<(ExecState, Option<Duration>, Duration), Rc<Error>>> + 'a>,
> {
let pool = self as *mut CmdPool;
async move {
let this = unsafe { &mut *pool };
loop {
if let Some(exec_num) = this.cmd_execs.get(&cmd_num) {
let exec = &this.execs[exec_num];
let result = exec.future.clone().await;
if result.is_none() {
return Err(Rc::new(match exec.cmd.engine {
ChompEngine::Shell => anyhow!("Unable to initialize shell command engine"),
ChompEngine::Node => anyhow!("Unable to initialize the Node.js Chomp engine.\n\x1b[33mMake sure Node.js is correctly installed and the \x1b[1mnode\x1b[0m\x1b[33m command bin is in the environment PATH.\x1b[0m\n\nSee \x1b[36;4mhttps://nodejs.org/en/download/\x1b[0m\n"),
ChompEngine::Deno => anyhow!("Unable to initialize the Deno Chomp engine.\n\x1b[33mMake sure Deno is correctly installed and the \x1b[1mdeno\x1b[0m\x1b[33m bin is in the environment PATH.\x1b[0m\n\nSee \x1b[36;4mhttps://deno.land/#installation\x1b[0m\n"),
}));
}
return Ok(result.unwrap());
}
if this.batch_future.is_none() {
this.create_batch_future();
}
this.batch_future.as_ref().unwrap().clone().await?;
}
}.boxed_local()
}
fn create_batch_future(&mut self) {
// This is bad Rust, but it's also totally fine given the static execution model
// (in Zig it might even be called idomatic)...
let pool = self as *mut CmdPool;
let cmds = &mut self.cmds as *mut BTreeMap<usize, CmdOp>;
self.batch_future = Some(
async move {
// batches with 5 millisecond execution groupings
sleep(Duration::from_millis(5)).await;
// pool itself is static. Rust doesn't know this.
let this = unsafe { &mut *pool };
// cmds are immutable, and retained as long as executions. Rust doesn't know this.
let cmds = unsafe { &mut *cmds };
let mut batch: HashSet<&CmdOp> =
this.batching.iter().map(|cmd_num| &cmds[cmd_num]).collect();
let running: HashSet<&BatchCmd> = this
.execs
.values()
.filter(|exec| matches!(&exec.state, ExecState::Executing))
.map(|exec| &exec.cmd)
.collect();
let mut global_completion_map: Vec<(usize, usize)> = Vec::new();
let mut batched: Vec<BatchCmd> = Vec::new();
let mut batcher = 0;
if this.extension_env.has_batchers() {
'outer: loop {
let (
BatcherResult {
defer: mut queue,
mut exec,
mut completion_map,
},
next,
) = this.extension_env.run_batcher(batcher, &batch, &running)?;
if let Some(completion_map) = completion_map.take() {
for (cmd_num, exec_num) in completion_map {
batch.remove(&cmds[&cmd_num]);
this.batching.remove(&cmd_num);
global_completion_map.push((cmd_num, exec_num));
}
}
if let Some(queue) = queue.take() {
for cmd_num in queue {
batch.remove(&cmds[&cmd_num]);
}
}
if let Some(mut exec) = exec.take() {
for cmd in exec.drain(..) {
for cmd_num in cmd.ids.iter() {
this.batching.remove(cmd_num);
batch.remove(&cmds[cmd_num]);
}
batched.push(cmd);
}
}
match next {
Some(num) => batcher = num,
None => break 'outer,
};
}
}
for (cmd_num, exec_num) in global_completion_map {
this.execs.get_mut(&exec_num).unwrap().cmd.ids.push(cmd_num);
}
for cmd in batched.drain(..) {
this.new_exec(cmd).await;
}
// any leftover unbatched just get batched
for cmd in batch {
if this.exec_cnt == this.pool_size {
break;
}
this.batching.remove(&cmd.id);
this.new_exec(BatchCmd {
id: None,
echo: cmd.echo,
run: cmd.run.to_string(),
cwd: cmd.cwd.clone(),
engine: cmd.engine,
env: cmd.env.clone(),
stdio: Some(cmd.stdio),
ids: vec![cmd.id],
})
.await;
}
this.batch_future = None;
Ok(())
}
.boxed_local()
.shared(),
);
}
async fn new_exec(&mut self, mut cmd: BatchCmd) {
let exec_num = self.exec_num;
cmd.id = Some(exec_num);
let mut targets = Vec::new();
for id in &cmd.ids {
let cmd = &self.cmds[id];
if let Some(name) = &cmd.name {
println!("\x1b[1m▶ {}\x1b[0m", name);
}
for target in &cmd.targets {
let target_path = Path::new(target);
if let Some(parent) = target_path.parent() {
fs::create_dir_all(parent).await.unwrap();
}
targets.push(target.to_string());
}
}
// cmd_execs and execs must be populated together without an await between them:
// get_exec_future reads cmd_execs first and then indexes execs, so any yield in between
// lets another future observe cmd_execs populated while execs is still missing.
for id in &cmd.ids {
self.cmd_execs.insert(*id, exec_num);
}
let pool = self as *mut CmdPool;
match cmd.engine {
ChompEngine::Shell => {
let start_time = Instant::now();
self.exec_cnt += 1;
let child = create_cmd(
cmd.cwd.as_ref().unwrap_or(&self.cwd),
&self.path,
&cmd,
true,
);
let future = async move {
let this = unsafe { &mut *pool };
let exec = &mut this.execs.get_mut(&exec_num).unwrap();
exec.state = match exec.child.as_mut().unwrap().wait().await {
Ok(status) => {
if status.success() {
ExecState::Completed
} else {
ExecState::Failed
}
}
Err(e) => match exec.state {
ExecState::Terminating => ExecState::Terminated,
_ => panic!("Unexpected exec error {:?}", e),
},
};
let end_time = Instant::now();
this.exec_cnt -= 1;
// finally we verify that the targets exist
let mtime = check_target_mtimes(targets, true).await;
Some((exec.state, mtime, end_time - start_time))
}
.boxed_local()
.shared();
self.execs.insert(
exec_num,
Exec {
cmd,
child,
future,
state: ExecState::Executing,
},
);
self.exec_num += 1;
}
ChompEngine::Node => node_runner(self, cmd, targets),
ChompEngine::Deno => deno_runner(self, cmd, targets),
};
}
pub fn batch(
&mut self,
name: Option<String>,
run: &String,
targets: Vec<String>,
env: BTreeMap<String, String>,
replacements: bool,
cwd: Option<String>,
engine: ChompEngine,
stdio: TaskStdio,
echo: bool,
) -> usize {
let id = self.cmd_num;
let run = if matches!(engine, ChompEngine::Shell) && replacements {
replace_env_vars_static(run, &env)
} else {
run.to_string()
};
self.cmds.insert(
id,
CmdOp {
id,
cwd,
name,
run,
env,
echo,
engine,
stdio,
targets,
},
);
self.cmd_num = id + 1;
self.batching.insert(id);
if self.batch_future.is_none() {
self.create_batch_future();
}
id
}
}
================================================
FILE: src/engines/node.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::engines::check_target_mtimes;
use crate::engines::create_cmd;
use crate::engines::CmdPool;
use crate::engines::Exec;
use crate::engines::{BatchCmd, ExecState};
use base64::{engine::general_purpose, Engine as _};
use futures::future::FutureExt;
use percent_encoding::percent_encode;
use percent_encoding::NON_ALPHANUMERIC;
use std::time::Instant;
// Custom node loader to mimic current working directory despite loading from a tmp file
// Note: We dont have to percent encode as we're not using `,! characters
// If this becomes a problem, switch to base64 encoding rather
const NODE_LOADER: &str = "let s;export function resolve(u,c,d){if(c.parentURL===undefined){const i=u.indexOf('data:text/javascript;base64,');s=Buffer.from(u.slice(i+28),'base64');return{url:u.slice(0,i)+(u[i-1]==='/'?'':'/')+'[cm]',format:'module',shortCircuit:true}}return d(u,c)}export function load(u,c,d){if(u.endsWith('[cm]'))return{source:s,format:'module',shortCircuit:true};return d(u,c)}export{load as getFormat,load as getSource}";
pub fn node_runner(cmd_pool: &mut CmdPool, mut cmd: BatchCmd, targets: Vec<String>) {
let start_time = Instant::now();
cmd.env.insert(
"CHOMP_PATH".to_string(),
std::env::args().next().unwrap().to_string(),
);
let targets = targets.clone();
// On posix, command starts executing before we wait on it!
cmd.run = format!(
"node --no-warnings --loader \"data:text/javascript,{}\" \"data:text/javascript;base64,{}\"",
percent_encode(NODE_LOADER.to_string().as_bytes(), NON_ALPHANUMERIC),
general_purpose::STANDARD.encode(cmd.run.as_bytes())
);
let echo = cmd.echo;
cmd.echo = false;
let run_clone = if echo { Some(cmd.run.clone()) } else { None };
let exec_num = cmd_pool.exec_num;
cmd_pool.exec_cnt += 1;
let pool = cmd_pool as *mut CmdPool;
let child = create_cmd(
cmd.cwd.as_ref().unwrap_or(&cmd_pool.cwd),
&cmd_pool.path,
&cmd,
false,
);
let future = async move {
let cmd_pool = unsafe { &mut *pool };
let exec = &mut cmd_pool.execs.get_mut(&exec_num).unwrap();
exec.child.as_ref()?;
if echo {
println!("{}", run_clone.as_ref().unwrap());
}
exec.state = match exec.child.as_mut().unwrap().wait().await {
Ok(status) => {
if status.success() {
ExecState::Completed
} else {
ExecState::Failed
}
}
Err(e) => match exec.state {
ExecState::Terminating => ExecState::Terminated,
_ => panic!("Unexpected exec error {:?}", e),
},
};
cmd_pool.exec_cnt -= 1;
let end_time = Instant::now();
// finally we verify that the targets exist
let mtime = check_target_mtimes(targets, true).await;
Some((exec.state, mtime, end_time - start_time))
}
.boxed_local()
.shared();
cmd_pool.execs.insert(
exec_num,
Exec {
cmd,
child,
future,
state: ExecState::Executing,
},
);
cmd_pool.exec_num += 1;
}
================================================
FILE: src/extensions.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::chompfile::ChompTaskMaybeTemplatedJs;
use crate::engines::BatchCmd;
use crate::engines::CmdOp;
use crate::ChompTaskMaybeTemplated;
use crate::Chompfile;
use anyhow::{anyhow, Error, Result};
use convert_case::{Case, Casing};
use serde::Deserialize;
use serde_v8::from_v8;
use serde_v8::to_v8;
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::rc::Rc;
pub struct ExtensionEnvironment {
isolate: v8::OwnedIsolate,
has_extensions: bool,
global_context: v8::Global<v8::Context>,
}
#[derive(Debug, PartialEq, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BatcherResult {
pub defer: Option<Vec<usize>>,
pub exec: Option<Vec<BatchCmd>>,
pub completion_map: Option<HashMap<usize, usize>>,
}
struct Extensions {
pub tasks: Vec<ChompTaskMaybeTemplatedJs>,
can_register: bool,
includes: Vec<String>,
templates: HashMap<String, v8::Global<v8::Function>>,
batchers: Vec<(String, v8::Global<v8::Function>)>,
}
impl Extensions {
fn new() -> Self {
Extensions {
can_register: true,
tasks: Vec::new(),
includes: Vec::new(),
templates: HashMap::new(),
batchers: Vec::new(),
}
}
}
fn create_template_options(
template: &str,
task_options: &Option<HashMap<String, toml::value::Value>>,
default_options: &HashMap<String, HashMap<String, toml::value::Value>>,
convert_case: bool,
) -> HashMap<String, toml::value::Value> {
let mut options = HashMap::new();
if let Some(task_options) = task_options {
for (key, value) in task_options {
let converted_key = if convert_case {
key.from_case(Case::Kebab).to_case(Case::Camel)
} else {
key.to_string()
};
options.insert(converted_key, value.clone());
}
};
if let Some(default_options) = default_options.get(template) {
for (key, value) in default_options {
let converted_key = key.from_case(Case::Kebab).to_case(Case::Camel);
if options.contains_key(&converted_key) {
continue;
}
options.insert(converted_key, value.clone());
}
}
options
}
pub fn expand_template_tasks(
chompfile: &Chompfile,
extension_env: &mut ExtensionEnvironment,
cwd: &str,
) -> Result<(bool, Vec<ChompTaskMaybeTemplated>)> {
let mut out_tasks = Vec::new();
let mut has_templates = false;
// expand tasks into initial job list
let mut task_queue: VecDeque<ChompTaskMaybeTemplated> = VecDeque::new();
for (idx, task) in chompfile.task.iter().enumerate() {
if task.deps.is_some() && task.dep.is_some() {
return Err(anyhow!("Invalid task: Both 'dep' and 'deps' fields are used by task {}, either a single dep or list of deps must be provided.", idx));
}
if task.targets.is_some() && task.target.is_some() {
return Err(anyhow!("Invalid task: Both 'target' and 'targets' fields are used by task {}, either a single target or list of targets must be provided.", idx));
}
let mut cloned = task.clone();
if let Some(ref template) = task.template {
cloned.template_options = Some(create_template_options(
template,
&task.template_options,
&chompfile.template_options,
true,
))
};
task_queue.push_back(cloned);
}
while !task_queue.is_empty() {
let mut task = task_queue.pop_front().unwrap();
if task.template.is_none() {
out_tasks.push(task);
continue;
}
has_templates = true;
let template = task.template.as_ref().unwrap();
if task.deps.is_none() {
task.deps = Some(Default::default());
}
let js_task = ChompTaskMaybeTemplatedJs {
cwd: task.cwd.clone(),
name: task.name.clone(),
target: None,
targets: Some(task.targets_vec(cwd)?),
invalidation: Some(task.invalidation.unwrap_or_default()),
validation: Some(task.validation.unwrap_or_default()),
dep: None,
deps: Some(task.deps_vec(chompfile, cwd)?),
args: task.args.clone(),
echo: task.echo,
display: task.display,
stdio: Some(task.stdio.unwrap_or_default()),
serial: task.serial,
env_replace: task.env_replace,
env: task.env,
env_default: task.env_default,
run: task.run,
engine: task.engine,
template: None,
template_options: task.template_options,
watch_invalidation: task.watch_invalidation,
};
let mut template_tasks: Vec<ChompTaskMaybeTemplatedJs> =
extension_env.run_template(template, &js_task)?;
// template functions output a list of tasks
for mut template_task in template_tasks.drain(..).rev() {
template_task.template_options = if let Some(ref template) = template_task.template {
Some(create_template_options(
template,
&template_task.template_options,
&chompfile.template_options,
false,
))
} else {
None
};
task_queue.push_front(template_task.into());
}
}
Ok((has_templates, out_tasks))
}
pub fn init_js_platform() {
let platform = v8::new_default_platform(0, false).make_shared();
v8::V8::initialize_platform(platform);
v8::V8::initialize();
}
fn chomp_log(
scope: &mut v8::HandleScope,
args: v8::FunctionCallbackArguments,
mut _rv: v8::ReturnValue,
) {
let mut msg = String::new();
let len = args.length();
let mut i = 0;
while i < len {
// TODO: better object logging - currently throws on objects
let arg: v8::Local<v8::Value> = args.get(i);
if i > 0 {
msg.push_str(", ");
}
msg.push_str(&arg.to_rust_string_lossy(scope));
i += 1;
}
println!("{}", &msg);
}
fn chomp_include(
scope: &mut v8::HandleScope,
args: v8::FunctionCallbackArguments,
mut _rv: v8::ReturnValue,
) {
let include: String = {
let tc_scope = &mut v8::TryCatch::new(scope);
from_v8(tc_scope, args.get(0)).expect("Unable to register include")
};
let mut extension_env = scope
.get_slot::<Rc<RefCell<Extensions>>>()
.unwrap()
.borrow_mut();
if !extension_env.can_register {
panic!("Chomp does not yet support dynamic includes.");
}
extension_env.includes.push(include);
}
fn chomp_register_task(
scope: &mut v8::HandleScope,
args: v8::FunctionCallbackArguments,
mut _rv: v8::ReturnValue,
) {
let task: ChompTaskMaybeTemplatedJs = {
let tc_scope = &mut v8::TryCatch::new(scope);
from_v8(tc_scope, args.get(0)).expect("Unable to register task")
};
let mut extension_env = scope
.get_slot::<Rc<RefCell<Extensions>>>()
.unwrap()
.borrow_mut();
if !extension_env.can_register {
panic!("Chomp does not support dynamic task registration.");
}
extension_env.tasks.push(task);
}
fn chomp_register_template(
scope: &mut v8::HandleScope,
args: v8::FunctionCallbackArguments,
mut _rv: v8::ReturnValue,
) {
let name = args.get(0).to_string(scope).unwrap();
let name_str = name.to_rust_string_lossy(scope);
let tpl = v8::Local::<v8::Function>::try_from(args.get(1)).unwrap();
let tpl_global = v8::Global::new(scope, tpl);
let mut extension_env = scope
.get_slot::<Rc<RefCell<Extensions>>>()
.unwrap()
.borrow_mut();
if !extension_env.can_register {
panic!("Chomp does not support dynamic template registration.");
}
extension_env.templates.insert(name_str, tpl_global);
}
fn chomp_register_batcher(
scope: &mut v8::HandleScope,
args: v8::FunctionCallbackArguments,
mut _rv: v8::ReturnValue,
) {
let name = args.get(0).to_string(scope).unwrap();
let name_str = name.to_rust_string_lossy(scope);
let batch = v8::Local::<v8::Function>::try_from(args.get(1)).unwrap();
let batch_global = v8::Global::new(scope, batch);
let mut extension_env = scope
.get_slot::<Rc<RefCell<Extensions>>>()
.unwrap()
.borrow_mut();
if !extension_env.can_register {
panic!("Chomp does not support dynamic batcher registration.");
}
// remove any existing batcher by the same name
if let Some(prev_batcher) = extension_env
.batchers
.iter()
.position(|name| name.0 == name_str)
{
extension_env.batchers.remove(prev_batcher);
}
extension_env.batchers.push((name_str, batch_global));
}
impl ExtensionEnvironment {
pub fn new(global_env: &BTreeMap<String, String>) -> Self {
let mut isolate = v8::Isolate::new(Default::default());
let global_context = {
let mut handle_scope = v8::HandleScope::new(&mut isolate);
let context = v8::Context::new(&mut handle_scope);
let global = context.global(&mut handle_scope);
let scope = &mut v8::ContextScope::new(&mut handle_scope, context);
let chomp_key = v8::String::new(scope, "Chomp").unwrap();
let chomp_val = v8::Object::new(scope);
global.set(scope, chomp_key.into(), chomp_val.into());
let console_key = v8::String::new(scope, "console").unwrap();
let console_val = v8::Object::new(scope);
global.set(scope, console_key.into(), console_val.into());
let log_fn = v8::FunctionTemplate::new(scope, chomp_log)
.get_function(scope)
.unwrap();
let log_key = v8::String::new(scope, "log").unwrap();
console_val.set(scope, log_key.into(), log_fn.into());
let version_key = v8::String::new(scope, "version").unwrap();
let version_str = v8::String::new(scope, "0.1").unwrap();
chomp_val.set(scope, version_key.into(), version_str.into());
let task_fn = v8::FunctionTemplate::new(scope, chomp_register_task)
.get_function(scope)
.unwrap();
let task_key = v8::String::new(scope, "registerTask").unwrap();
chomp_val.set(scope, task_key.into(), task_fn.into());
let tpl_fn = v8::FunctionTemplate::new(scope, chomp_register_template)
.get_function(scope)
.unwrap();
let template_key = v8::String::new(scope, "registerTemplate").unwrap();
chomp_val.set(scope, template_key.into(), tpl_fn.into());
let batch_fn = v8::FunctionTemplate::new(scope, chomp_register_batcher)
.get_function(scope)
.unwrap();
let batcher_key = v8::String::new(scope, "registerBatcher").unwrap();
chomp_val.set(scope, batcher_key.into(), batch_fn.into());
let include_fn = v8::FunctionTemplate::new(scope, chomp_include)
.get_function(scope)
.unwrap();
let include_key = v8::String::new(scope, "addExtension").unwrap();
chomp_val.set(scope, include_key.into(), include_fn.into());
let env_key = v8::String::new(scope, "ENV").unwrap();
let env_val = v8::Object::new(scope);
global.set(scope, env_key.into(), env_val.into());
for (key, value) in global_env {
let env_key = v8::String::new(scope, key).unwrap();
let env_key_val = v8::String::new(scope, value).unwrap();
env_val.set(scope, env_key.into(), env_key_val.into());
}
v8::Global::new(scope, context)
};
let extensions = Extensions::new();
isolate.set_slot(Rc::new(RefCell::new(extensions)));
ExtensionEnvironment {
isolate,
has_extensions: false,
global_context,
}
}
fn handle_scope(&mut self) -> v8::HandleScope<'_> {
v8::HandleScope::with_context(&mut self.isolate, self.global_context.clone())
}
pub fn get_tasks(&self) -> Vec<ChompTaskMaybeTemplatedJs> {
self.isolate
.get_slot::<Rc<RefCell<Extensions>>>()
.unwrap()
.borrow()
.tasks
.clone()
}
fn get_extensions(&self) -> &Rc<RefCell<Extensions>> {
self.isolate.get_slot::<Rc<RefCell<Extensions>>>().unwrap()
}
pub fn add_extension(
&mut self,
extension_source: &str,
filename: &str,
) -> Result<Option<Vec<String>>> {
self.has_extensions = true;
{
let mut handle_scope = self.handle_scope();
let code =
v8::String::new(&mut handle_scope, &format!("{{{}}}", extension_source)).unwrap();
let tc_scope = &mut v8::TryCatch::new(&mut handle_scope);
let resource_name = v8::String::new(tc_scope, filename).unwrap().into();
let source_map = v8::String::new(tc_scope, "").unwrap().into();
let origin = v8::ScriptOrigin::new(
tc_scope,
resource_name,
0,
0,
false,
123,
source_map,
true,
false,
false,
);
let script = match v8::Script::compile(tc_scope, code, Some(&origin)) {
Some(script) => script,
None => return Err(v8_exception(tc_scope)),
};
match script.run(tc_scope) {
Some(_) => {}
None => return Err(v8_exception(tc_scope)),
};
}
let mut extensions = self.get_extensions().borrow_mut();
if !extensions.includes.is_empty() {
Ok(Some(extensions.includes.drain(..).collect()))
} else {
Ok(None)
}
}
pub fn seal_extensions(&mut self) {
let mut extensions = self.get_extensions().borrow_mut();
extensions.can_register = false;
}
pub fn run_template(
&mut self,
name: &str,
task: &ChompTaskMaybeTemplatedJs,
) -> Result<Vec<ChompTaskMaybeTemplatedJs>> {
let template = {
let extensions = self.get_extensions().borrow();
match extensions.templates.get(name) {
Some(tpl) => Ok(tpl.clone()),
None => {
if name == "babel"
|| name == "cargo"
|| name == "jspm"
|| name == "npm"
|| name == "prettier"
|| name == "svelte"
|| name == "swc"
{
if self.has_extensions {
Err(anyhow!("Template '{}' has not been registered. To include the core template, add \x1b[1m'chomp@0.1:{}'\x1b[0m to the extensions list:\x1b[36m\n\n extensions = [..., 'chomp@0.1:{}']\n\n\x1b[0min the \x1b[1mchompfile.toml\x1b[0m.", &name, &name, &name))
} else {
Err(anyhow!("Template '{}' has not been registered. To include the core template, add:\x1b[36m\n\n extensions = ['chomp@0.1:{}']\n\n\x1b[0mto the \x1b[1mchompfile.toml\x1b[0m.", &name, &name))
}
} else {
Err(anyhow!("Template '{}' has not been registered. Make sure it is included in the \x1b[1mchompfile.toml\x1b[0m extensions.", &name))
}
}
}
}?;
let cb = template.open(&mut self.isolate);
let mut handle_scope = self.handle_scope();
let tc_scope = &mut v8::TryCatch::new(&mut handle_scope);
let this = v8::undefined(tc_scope).into();
let args: Vec<v8::Local<v8::Value>> =
vec![to_v8(tc_scope, task).expect("Unable to serialize template params")];
let result = match cb.call(tc_scope, this, args.as_slice()) {
Some(result) => result,
None => return Err(v8_exception(tc_scope)),
};
let task: Vec<ChompTaskMaybeTemplatedJs> = from_v8(tc_scope, result)
.expect("Unable to deserialize template task list due to invalid structure");
Ok(task)
}
pub fn has_batchers(&self) -> bool {
!self.get_extensions().borrow().batchers.is_empty()
}
pub fn run_batcher(
&mut self,
idx: usize,
batch: &HashSet<&CmdOp>,
running: &HashSet<&BatchCmd>,
) -> Result<(BatcherResult, Option<usize>)> {
let (name, batcher, batchers_len) = {
let extensions = self.get_extensions().borrow();
let (name, batcher) = extensions.batchers[idx].clone();
(name, batcher, extensions.batchers.len())
};
let cb = batcher.open(&mut self.isolate);
let mut handle_scope = self.handle_scope();
let tc_scope = &mut v8::TryCatch::new(&mut handle_scope);
let this = v8::undefined(tc_scope).into();
let args: Vec<v8::Local<v8::Value>> = vec![
to_v8(tc_scope, batch).expect("Unable to serialize batcher call"),
to_v8(tc_scope, running).expect("Unable to serialize batcher call"),
];
let result = match cb.call(tc_scope, this, args.as_slice()) {
Some(result) => result,
None => return Err(v8_exception(tc_scope)),
};
let result: Option<BatcherResult> = from_v8(tc_scope, result).unwrap_or_else(|_| panic!("Unable to deserialize batch for {} due to invalid structure",
name));
let next = if idx < batchers_len - 1 {
Some(idx + 1)
} else {
None
};
Ok((
result.unwrap_or(BatcherResult {
defer: None,
exec: None,
completion_map: None,
}),
next,
))
}
}
fn v8_exception(scope: &mut v8::TryCatch<v8::HandleScope>) -> Error {
let exception = scope.exception().unwrap();
if is_instance_of_error(scope, exception) {
let exception: v8::Local<v8::Object> = exception.try_into().unwrap();
let stack = get_property(scope, exception, "stack");
let stack: Option<v8::Local<v8::String>> = stack.and_then(|s| s.try_into().ok());
let stack = stack.map(|s| s.to_rust_string_lossy(scope));
let err_str = stack.unwrap();
if let Some(rest) = err_str.strip_prefix("Error: ") {
anyhow!("{}", rest)
} else if let Some(rest) = err_str.strip_prefix("TypeError: ") {
anyhow!("TypeError {}", rest)
} else if let Some(rest) = err_str.strip_prefix("SyntaxError: ") {
anyhow!("SyntaxError {}", rest)
} else if let Some(rest) = err_str.strip_prefix("ReferenceError: ") {
anyhow!("ReferenceError {}", rest)
} else {
anyhow!("{}", &err_str)
}
} else {
anyhow!("JS error: {}", exception.to_rust_string_lossy(scope))
}
}
fn get_property<'a>(
scope: &mut v8::HandleScope<'a>,
object: v8::Local<v8::Object>,
key: &str,
) -> Option<v8::Local<'a, v8::Value>> {
let key = v8::String::new(scope, key).unwrap();
object.get(scope, key.into())
}
fn is_instance_of_error<'s>(scope: &mut v8::HandleScope<'s>, value: v8::Local<v8::Value>) -> bool {
if !value.is_object() {
return false;
}
let message = v8::String::empty(scope);
let error_prototype = v8::Exception::error(scope, message)
.to_object(scope)
.unwrap()
.get_prototype(scope)
.unwrap();
let mut maybe_prototype = value.to_object(scope).unwrap().get_prototype(scope);
while let Some(prototype) = maybe_prototype {
if prototype.strict_equals(error_prototype) {
return true;
}
maybe_prototype = prototype
.to_object(scope)
.and_then(|o| o.get_prototype(scope));
}
false
}
================================================
FILE: src/http_client.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::{anyhow, Result};
use dirs::home_dir;
use http_body_util::{BodyExt, Empty};
use hyper::body::Bytes;
use hyper::Uri;
use hyper_tls::HttpsConnector;
use hyper_util::client::legacy::{connect::HttpConnector, Client};
use hyper_util::rt::TokioExecutor;
use sha2::{Digest, Sha256};
use std::path::PathBuf;
use tokio::fs;
fn chomp_cache_dir() -> PathBuf {
let mut path = home_dir().unwrap();
path.push(".chomp");
path.push("cache");
path
}
pub async fn clear_cache() -> std::io::Result<()> {
match fs::remove_dir_all(chomp_cache_dir()).await {
Ok(()) => Ok(()),
Err(e) => match e.kind() {
std::io::ErrorKind::NotFound => Ok(()),
_ => Err(e),
},
}
}
pub async fn prep_cache() -> Result<()> {
let _ = fs::create_dir_all(chomp_cache_dir()).await;
Ok(())
}
#[inline(always)]
fn u4_to_hex_char(c: u8) -> char {
(if c < 10 { c + 48 } else { c + 87 } as char)
}
pub fn hash(input: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(input);
let result = hasher.finalize();
let mut out_hash = String::with_capacity(64);
for c in result {
out_hash.push(u4_to_hex_char(c & 0xF));
out_hash.push(u4_to_hex_char(c >> 4));
}
out_hash
}
async fn from_cache(cache_key: &str) -> Option<String> {
let mut path = chomp_cache_dir();
path.push(cache_key);
match fs::read_to_string(&path).await {
Ok(cached) => Some(cached),
Err(e) => match e.kind() {
std::io::ErrorKind::NotFound => None,
_ => panic!("File error {}", e),
},
}
}
async fn write_cache(cache_key: &str, source: &str) -> Result<()> {
let mut path = chomp_cache_dir();
path.push(cache_key);
fs::write(&path, source).await?;
Ok(())
}
pub async fn fetch_uri_cached(uri_str: &str, uri: Uri) -> Result<String> {
let hash = hash(uri_str.as_bytes());
if let Some(cached) = from_cache(&hash).await {
return Ok(cached);
}
println!("\x1b[34;1mFetch\x1b[0m {}", &uri_str);
let https = HttpsConnector::new();
let client: Client<HttpsConnector<HttpConnector>, Empty<Bytes>> =
Client::builder(TokioExecutor::new()).build(https);
let res = client.get(uri).await?;
if res.status() != 200 {
return Err(anyhow!("{} for extension URL {}", res.status(), uri_str));
}
let body_bytes = res.into_body().collect().await?.to_bytes();
let result = String::from_utf8(body_bytes.to_vec()).unwrap();
write_cache(&hash, &result).await?;
Ok(result)
}
================================================
FILE: src/main.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#![allow(clippy::type_complexity, clippy::too_many_arguments)]
extern crate clap;
#[macro_use]
extern crate lazy_static;
use crate::chompfile::ChompTaskMaybeTemplated;
use crate::chompfile::Chompfile;
use crate::extensions::expand_template_tasks;
use crate::extensions::init_js_platform;
use crate::extensions::ExtensionEnvironment;
use crate::task::Runner;
use anyhow::{anyhow, Result};
use clap::{Arg, ArgAction, Command};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs;
use std::path::Path;
extern crate num_cpus;
use crate::engines::replace_env_vars_static;
use hyper::Uri;
use std::env;
use std::fs::canonicalize;
use tokio::sync::mpsc::unbounded_channel;
mod ansi_windows;
mod chompfile;
mod engines;
mod extensions;
mod http_client;
mod server;
mod task;
use std::path::PathBuf;
const CHOMP_CORE: &str = "https://ga.jspm.io/npm:@chompbuild/extensions@0.1.31/";
const CHOMP_INIT: &str = r#"version = 0.1
[[task]]
name = 'build'
run = 'echo \"Build script goes here\"'
"#;
const CHOMP_EMPTY: &str = "version = 0.1\n";
fn uri_parse(uri_str: &str) -> Option<Uri> {
let uri = uri_str.parse::<Uri>().ok()?;
uri.scheme_str()?;
Some(uri)
}
#[tokio::main]
async fn main() -> Result<()> {
#[cfg(not(debug_assertions))]
let version = "0.3.0";
#[cfg(debug_assertions)]
let version = "0.3.0-debug";
let matches = Command::new("Chomp")
.version(version)
.arg(
Arg::new("watch")
.short('w')
.long("watch")
.help("Watch the input files for changes")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("serve")
.short('s')
.long("serve")
.help("Run a local dev server")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("server-root")
.short('R')
.long("server-root")
.help("Server root path"),
)
.arg(
Arg::new("port")
.short('p')
.long("port")
.value_name("PORT")
.help("Custom port to serve"),
)
.arg(
Arg::new("jobs")
.short('j')
.long("jobs")
.value_name("N")
.value_parser(clap::value_parser!(usize))
.help("Maximum number of jobs to run in parallel"),
)
.arg(
Arg::new("config")
.short('c')
.long("config")
.value_name("CONFIG")
.default_value("chompfile.toml")
.help("Custom chompfile path"),
)
.arg(
Arg::new("list")
.short('l')
.long("list")
.help("List the available chompfile tasks")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("format")
.short('F')
.long("format")
.help("Format and save the chompfile.toml")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("eject_templates")
.long("eject")
.help("Ejects templates into tasks saving the rewritten chompfile.toml")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("init")
.short('i')
.long("init")
.help("Initialize a new chompfile.toml if it does not exist")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("import_scripts")
.short('I')
.long("import-scripts")
.help("Import from npm \"scripts\" into the chompfile.toml")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("clear_cache")
.short('C')
.long("clear-cache")
.help("Clear URL extension cache")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("rerun")
.short('r')
.long("rerun")
.help("Rerun the target tasks even if cached")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("force")
.short('f')
.long("force")
.help("Force rebuild targets")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("target")
.value_name("TARGET")
.help("Generate a target or list of targets")
.action(ArgAction::Append),
)
.arg(
Arg::new("arg")
.last(true)
.value_name("ARGS")
.help("Custom task args")
.action(ArgAction::Append),
)
.get_matches();
#[cfg(target_os = "windows")]
match ansi_windows::enable_ansi_support() {
Ok(()) => {}
Err(_) => {
// TODO: handling disabling of ansi codes
}
};
let mut targets: Vec<String> = Vec::new();
let mut use_default_target = true;
if let Some(target) = matches.get_many::<String>("target") {
for item in target {
targets.push(item.to_string());
}
}
let cfg_path = Path::new(matches.get_one::<String>("config").unwrap());
let cfg_dir = cfg_path.parent().unwrap().to_str().unwrap();
let mut cfg_file = canonicalize(if cfg_dir.is_empty() { "." } else { cfg_dir }).unwrap();
cfg_file.push(cfg_path.file_name().unwrap());
let mut created = false;
let chompfile_source = {
let is_dir: bool = match fs::metadata(&cfg_file) {
Ok(meta) => meta.is_dir(),
Err(_) => false,
};
if is_dir {
cfg_file.push("chompfile.toml");
}
match fs::read_to_string(&cfg_file) {
Ok(source) => source,
Err(_) => {
if matches.get_flag("init") {
created = true;
if matches.get_flag("import_scripts") {
String::from(CHOMP_EMPTY)
} else {
String::from(CHOMP_INIT)
}
} else {
if matches.get_flag("serve") {
String::from(CHOMP_EMPTY)
} else {
return Err(anyhow!(
"Unable to load the Chomp configuration {}. Pass the \x1b[1m--init\x1b[0m flag to create one, or try:\n\n\x1b[36mchomp --init --import-scripts\x1b[0m\n\nto create one and import from existing package.json scripts.",
&cfg_file.to_str().unwrap()
));
}
}
}
}
};
let mut chompfile: Chompfile = toml::from_str(&chompfile_source)?;
if chompfile.version != 0.1 {
return Err(anyhow!(
"Invalid chompfile version {}, only 0.1 is supported",
chompfile.version
));
}
let cwd = {
let mut parent: PathBuf = PathBuf::from(cfg_file.parent().unwrap());
if parent.to_str().unwrap().is_empty() {
parent = env::current_dir()?;
}
let unc_path = match canonicalize(&parent) {
Ok(path) => path,
Err(_) => {
return Err(anyhow!(
"Unable to load the Chomp configuration {}.\nMake sure it exists in the current directory, or use --config to set a custom path.",
&cfg_file.to_str().unwrap()
));
}
};
let unc_str = unc_path.to_str().unwrap();
if unc_str.starts_with(r"\\?\") {
PathBuf::from(String::from(&unc_path.to_str().unwrap()[4..]))
} else {
unc_path
}
};
assert!(env::set_current_dir(&cwd).is_ok());
if matches.get_flag("clear_cache") {
http_client::clear_cache().await?;
println!("\x1b[1;32m√\x1b[0m Cleared remote URL extension cache.");
if targets.is_empty() {
return Ok(());
}
}
init_js_platform();
let pool_size = match matches.get_one::<usize>("jobs") {
Some(&jobs) => jobs,
None => num_cpus::get(),
};
let mut global_env = BTreeMap::new();
for (key, value) in env::vars() {
global_env.insert(key.to_uppercase(), value);
}
for (key, value) in &chompfile.env {
global_env.insert(
key.to_uppercase(),
replace_env_vars_static(value, &global_env),
);
}
if matches.get_flag("eject_templates") {
global_env.insert("CHOMP_EJECT".to_string(), "1".to_string());
}
global_env.insert("CHOMP_POOL_SIZE".to_string(), pool_size.to_string());
// extend global env with the chompfile env as well
for (key, value) in &chompfile.env_default {
if !global_env.contains_key(&key.to_uppercase()) {
global_env.insert(
key.to_uppercase(),
replace_env_vars_static(value, &global_env),
);
}
}
let mut extension_env = ExtensionEnvironment::new(&global_env);
http_client::prep_cache().await?;
let mut extension_set: HashSet<String> = HashSet::new();
let mut extensions = chompfile.extensions.clone();
let mut i = 0;
while i < extensions.len() {
if extensions[i].starts_with("chomp:") {
return Err(anyhow!("Chomp core extensions must be versioned - try \x1b[36m'chomp@0.1:{}'\x1b[0m instead", &extensions[i][6..]));
}
let ext = if extensions[i].starts_with("chomp@0.1:") {
let mut s: String = match global_env.get("CHOMP_CORE") {
Some(path) => String::from(path),
None => String::from(CHOMP_CORE),
};
if !s.ends_with("/") && !s.ends_with("\\") {
s.push('/');
}
s.push_str(&extensions[i][10..]);
s.push_str(".js");
s
} else {
extensions[i].clone()
};
let (canonical, extension_source) = match uri_parse(ext.as_ref()) {
Some(uri) => {
if !extension_set.contains(&ext) {
extension_set.insert(ext.to_string());
(
extension_set.get(&ext).unwrap(),
Some(http_client::fetch_uri_cached(&ext, uri).await?),
)
} else {
(extension_set.get(&ext).unwrap(), None)
}
}
None => {
let canonical_str: String = match canonicalize(&ext) {
Ok(canonical) => canonical.to_str().unwrap().replace("\\", "/"),
Err(_) => {
return Err(anyhow!("Unable to read extension file '{}'.", &ext));
}
};
if !extension_set.contains(&canonical_str) {
extension_set.insert(canonical_str.to_string());
(
extension_set.get(&canonical_str).unwrap(),
Some(fs::read_to_string(&ext)?),
)
} else {
(extension_set.get(&canonical_str).unwrap(), None)
}
}
};
if let Some(extension_source) = extension_source {
if let Some(mut new_includes) = extension_env.add_extension(&extension_source, canonical)? {
for ext in new_includes.drain(..) {
// relative includes are relative to the parent
if let Some(rest) = ext.strip_prefix("./") {
let mut resolved_str =
canonical[0..canonical.rfind("/").unwrap() + 1].to_string();
resolved_str.push_str(rest);
extensions.push(resolved_str);
} else {
extensions.push(ext);
}
}
}
}
i += 1;
}
extension_env.seal_extensions();
// channel for watch events
let (watch_event_sender, watch_event_receiver) = unbounded_channel();
// channel for adding new files to watcher
let (watch_sender, watch_receiver) = unbounded_channel();
let mut serve_options = chompfile.server.clone();
{
if let Some(root) = matches.get_one::<String>("server-root") {
serve_options.root = root.to_string();
}
if let Some(port) = matches.get_one::<String>("port") {
serve_options.port = port.parse::<u16>().unwrap();
}
if matches.get_flag("serve") {
use_default_target = false;
tokio::spawn(server::serve(
serve_options,
watch_event_receiver,
watch_sender,
));
}
}
let mut args: Vec<String> = Vec::new();
if let Some(arg) = matches.get_many::<String>("arg") {
for item in arg {
args.push(item.to_string());
}
}
if matches.get_flag("import_scripts") {
if matches.get_flag("eject_templates") {
return Err(anyhow!(
"Cannot use --import-scripts and --eject-templates together."
));
}
let mut script_tasks = 0;
let pjson_source = match fs::read_to_string("package.json") {
Ok(source) => source,
Err(_) => {
return Err(anyhow!(
"No package.json to import found in the current project directory."
));
}
};
let pjson: serde_json::Value = serde_json::from_str(&pjson_source)?;
match &pjson["scripts"] {
serde_json::Value::Object(scripts) => {
for (name, val) in scripts.iter() {
if let serde_json::Value::String(run) = &val {
script_tasks += 1;
let mut task = ChompTaskMaybeTemplated::new();
task.name = Some(name.to_string());
task.run = Some(run.to_string());
chompfile.task.push(task);
}
}
}
_ => return Err(anyhow!("Unexpected \"scripts\" type in package.json.")),
};
fs::write(&cfg_file, toml::to_string_pretty(&chompfile)?)?;
println!(
"\x1b[1;32m√\x1b[0m \x1b[1m{}\x1b[0m {}.",
cfg_file.to_str().unwrap(),
if created {
format!(
"created with {} package.json script tasks imported",
script_tasks
)
} else {
format!(
"updated with {} package.json script tasks imported",
script_tasks
)
}
);
return Ok(());
}
let cwd_str = cwd.to_string_lossy().replace('\\', "/");
let (mut has_templates, mut template_tasks) =
expand_template_tasks(&chompfile, &mut extension_env, &cwd_str)?;
chompfile.task = Vec::new();
for task in extension_env.get_tasks().drain(..) {
has_templates = true;
chompfile.task.push(task.into());
}
chompfile.task.append(&mut template_tasks);
if matches.get_flag("list") {
if !targets.is_empty() {
return Err(anyhow!("--list does not take any arguments."));
}
if matches.get_flag("eject_templates")
|| matches.get_flag("format")
|| matches.get_flag("init")
{
return Err(anyhow!(
"Cannot use --list with --eject-templates, --format or --init."
));
}
for task in &chompfile.task {
if let Some(name) = &task.name {
let matches_some_target = if !targets.is_empty() {
let mut matches_some_target = false;
for target in &targets {
if name.starts_with(target) {
matches_some_target = true;
}
}
matches_some_target
} else {
true
};
if matches_some_target {
println!(" \x1b[1m▪\x1b[0m {}", name);
}
}
}
return Ok(());
}
if matches.get_flag("format") || matches.get_flag("eject_templates") || matches.get_flag("init")
{
use_default_target = false;
if matches.get_flag("eject_templates") {
if !has_templates {
return Err(anyhow!(
"\x1b[1m{}\x1b[0m has no templates to eject",
cfg_file.to_str().unwrap()
));
}
chompfile.extensions = Vec::new();
chompfile.template_options = HashMap::new();
}
fs::write(&cfg_file, toml::to_string_pretty(&chompfile)?)?;
if matches.get_flag("eject_templates") {
println!(
"\x1b[1;32m√\x1b[0m \x1b[1m{}\x1b[0m template tasks ejected.",
cfg_file.to_str().unwrap()
);
} else {
println!(
"\x1b[1;32m√\x1b[0m \x1b[1m{}\x1b[0m {}.",
cfg_file.to_str().unwrap(),
if created { "created" } else { "updated" }
);
}
if matches.get_flag("eject_templates") || targets.is_empty() {
return Ok(());
}
}
let targets = if targets.is_empty() && use_default_target {
vec![chompfile
.default_task
.to_owned()
.unwrap_or(String::from("build"))]
} else {
targets
};
let mut runner = Runner::new(
&chompfile,
&mut extension_env,
pool_size,
matches.get_flag("serve") || matches.get_flag("watch"),
)?;
let ok = runner
.run(
task::RunOptions {
watch: matches.get_flag("serve") || matches.get_flag("watch"),
force: matches.get_flag("force"),
rerun: matches.get_flag("rerun"),
args: if !args.is_empty() { Some(args) } else { None },
pool_size,
targets,
cfg_file,
},
watch_event_sender,
watch_receiver,
)
.await?;
if !ok {
eprintln!("Unable to complete all tasks.");
}
std::process::exit(if ok { 0 } else { 1 });
}
================================================
FILE: src/server.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
// const websocket = new WebSocket('ws://localhost:5776/watch'); websocket.onmessage = evt => console.log(evt.data);
use crate::chompfile::ServerOptions;
use crate::task::WatchEvent;
use bytes::Bytes;
use futures::{future, FutureExt, StreamExt};
use hyper::http::{header, Response, StatusCode};
use percent_encoding::percent_decode_str;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::fs;
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
use tokio::sync::{mpsc, RwLock};
use tokio_stream::wrappers::UnboundedReceiverStream;
use warp::ws::{Message, WebSocket, Ws};
use warp::Filter;
type ResponseBody = Bytes;
async fn client_connection(ws: WebSocket, state: State) {
let (sender, mut receiver) = ws.split();
let (client_sender, client_rcv) = mpsc::unbounded_channel();
let client_rcv = UnboundedReceiverStream::new(client_rcv);
tokio::task::spawn(client_rcv.forward(sender).map(|result| {
if let Err(e) = result {
eprintln!("error sending websocket msg: {}", e);
}
}));
client_sender.send(Ok(Message::text("Connected"))).unwrap();
let id = {
let clients_vec = &mut state.write().await.clients;
let id = if !clients_vec.is_empty() {
clients_vec.last().unwrap().id + 1
} else {
1
};
let client = Client {
sender: client_sender,
id,
};
clients_vec.push(client);
id
};
while let Some(body) = receiver.next().await {
let message = match body {
Ok(msg) => msg,
Err(e) => {
eprintln!("error reading message on websocket: {}", e);
break;
}
};
match message.to_str() {
Ok(msg) => {
println!("got message {}", msg);
}
_ => {
// println!("got non string message");
}
}
}
{
let clients_vec = &mut state.write().await.clients;
let idx = clients_vec
.iter()
.enumerate()
.find(|(_, client)| client.id == id)
.unwrap()
.0;
clients_vec.remove(idx);
}
}
pub struct Client {
sender: mpsc::UnboundedSender<std::result::Result<Message, warp::Error>>,
id: u32,
}
pub struct StateStruct {
clients: Vec<Client>,
file_hashes: BTreeMap<String, String>,
}
impl StateStruct {
fn new() -> StateStruct {
StateStruct {
clients: Vec::new(),
file_hashes: BTreeMap::new(),
}
}
}
pub type State = Arc<RwLock<StateStruct>>;
pub enum FileEvent {
WatchFile(PathBuf),
}
async fn check_watcher(mut rx: UnboundedReceiver<WatchEvent>, root: &PathBuf, state: State) {
loop {
if let Some(path) = rx.recv().await {
let path_str = match path.strip_prefix(root) {
Ok(path) => path.to_str().unwrap(),
Err(_) => continue,
};
let _ = revalidate(&path, path_str, state.clone(), true).await;
}
}
}
async fn revalidate(
path: &PathBuf,
path_str: &str,
state: State,
broadcast_updates: bool,
) -> (Option<String>, bool) {
let source = match fs::read(path).await {
Ok(src) => src,
Err(_) => return (None, true),
};
let hash = crate::http_client::hash(&source[0..]);
let mut state = state.write().await;
if let Some(existing_hash) = state.file_hashes.get(path_str) {
if hash.eq(existing_hash) {
return (Some(hash), false);
}
}
state
.file_hashes
.insert(path_str.to_string(), hash.to_string());
if broadcast_updates {
for client in state.clients.iter() {
client
.sender
.send(Ok(Message::text(path_str.replace('\\', "/"))))
.expect("error sending websocket");
}
}
(Some(hash), true)
}
fn not_found(resource: &str) -> Response<ResponseBody> {
Response::builder()
.status(StatusCode::NOT_FOUND)
.header(
header::CONTENT_TYPE,
header::HeaderValue::from_str("text/plain").unwrap(),
)
.body(Bytes::from(format!("\"{}\" Not Found", resource)))
.unwrap()
}
async fn file_serve(path: &PathBuf, root: &PathBuf, hash: Option<String>) -> Response<ResponseBody> {
if let Ok(contents) = fs::read(path).await {
let mut res = Response::new(Bytes::from(contents));
let guess = mime_guess::from_path(path);
if let Some(mime) = guess.first() {
let headers_mut = res.headers_mut();
headers_mut.insert(
header::CONTENT_TYPE,
header::HeaderValue::from_str(mime.essence_str()).unwrap(),
);
headers_mut.insert(
header::ETAG,
header::HeaderValue::from_str(&hash.unwrap()).unwrap(),
);
headers_mut.insert(
header::CACHE_CONTROL,
header::HeaderValue::from_str("must-revalidate").unwrap(),
);
}
return res;
}
not_found(
&path
.strip_prefix(root)
.expect("unexpected path")
.to_str()
.unwrap()
.replace('\\', "/"),
)
}
// TODO: gloss
async fn index_page(path: &mut PathBuf, root: &PathBuf) -> Option<Response<ResponseBody>> {
path.push("index.html");
match fs::metadata(&path).await {
Ok(_) => {}
Err(_) => {
path.pop();
let mut entries = std::fs::read_dir(&path)
.unwrap()
.map(|res| res.map(|e| e.path()))
.collect::<Result<Vec<_>, std::io::Error>>()
.unwrap();
entries.sort();
let mut listing = String::from("<!doctype html><body><ul>");
for entry in entries {
let name = entry
.strip_prefix(&path)
.unwrap()
.to_string_lossy()
.replace('\\', "/");
let relpath = entry
.strip_prefix(root)
.unwrap()
.to_string_lossy()
.replace('\\', "/");
let item = format!("<li><a href=\"{}\">{}</a></li>", relpath, name);
listing.push_str(&item);
}
listing.push_str("</ul>");
let mut res = Response::new(Bytes::from(listing));
*res.status_mut() = StatusCode::OK;
res.headers_mut().insert(
header::CONTENT_TYPE,
header::HeaderValue::from_str("text/html").unwrap(),
);
return Some(res);
}
};
None
}
pub async fn serve(
opts: ServerOptions,
watch_receiver: UnboundedReceiver<WatchEvent>,
watch_sender: UnboundedSender<FileEvent>,
) {
let state: State = Arc::new(RwLock::new(StateStruct::new()));
let watcher_state = state.clone();
let state_clone = state.clone();
let root = match fs::canonicalize(&opts.root).await {
Ok(canonical) => canonical,
Err(_) => {
eprintln!("Unable to find the root server path {}", &opts.root);
return;
}
};
let root_str = root.to_str().unwrap();
let root = if let Some(rest) = root_str.strip_prefix(r"\\?\") {
PathBuf::from(String::from(rest))
} else {
root
};
let watcher_root = root.clone();
let static_assets = warp::path::tail()
.and(warp::any().map(move || root.clone()))
.and(warp::any().map(move || state.clone()))
.and(warp::any().map(move || watch_sender.clone()))
.and(warp::filters::header::optional::<String>("if-none-match"))
.then(
|path: warp::path::Tail,
root: PathBuf,
state: State,
sender: UnboundedSender<FileEvent>,
validate_hash: Option<String>| async move {
let subpath = percent_decode_str(path.as_str())
.decode_utf8_lossy()
.into_owned();
let mut path = PathBuf::from(&root);
path.push(&subpath);
let is_dir = match fs::metadata(&path).await {
Ok(metadata) => metadata.is_dir(),
Err(_) => {
if !path.ends_with(".html") {
path.set_extension("html");
match fs::metadata(&path).await {
Ok(metadata) => metadata.is_dir(),
Err(_) => false,
}
} else {
false
}
}
};
if is_dir {
if let Some(res) = index_page(&mut path, &root).await {
return res;
}
}
let (hash, add_watch) = revalidate(&path, &subpath, state, false).await;
if add_watch {
let _ = sender.send(FileEvent::WatchFile(path.clone())).is_ok();
}
let (cached, etag) = match hash {
Some(hash) => match validate_hash {
Some(validate_hash) => (validate_hash == hash, Some(hash)),
None => (false, Some(hash)),
},
None => (false, None),
};
if cached {
let mut res = Response::new(Bytes::new());
*res.status_mut() = StatusCode::NOT_MODIFIED;
res
} else {
file_serve(&path, &root, etag).await
}
},
);
let websocket = warp::path("watch")
.and(warp::ws())
.and(warp::any().map(move || state_clone.clone()))
.map(|ws: Ws, state: State| ws.on_upgrade(move |socket| client_connection(socket, state)));
let routes = websocket
.or(static_assets)
.with(warp::cors().allow_any_origin())
.boxed();
println!(
"Serving \x1b[1m{}\x1b[0m on \x1b[36mhttp://localhost:{}\x1b[0m...",
opts.root, opts.port
);
future::join(
check_watcher(watch_receiver, &watcher_root, watcher_state),
warp::serve(routes).run(([127, 0, 0, 1], opts.port)),
)
.await;
}
================================================
FILE: src/task.rs
================================================
// Chomp Task Runner
// Copyright (C) 2022 Guy Bedford
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::chompfile::{
resolve_path, ChompTaskMaybeTemplated, Chompfile, InvalidationCheck, TaskDisplay,
ValidationCheck, WatchInvalidation,
};
use crate::engines::CmdPool;
use crate::server::FileEvent;
use crate::ExtensionEnvironment;
use async_recursion::async_recursion;
use capturing_glob::{glob, Pattern};
use futures::future::Shared;
use futures::future::{select_all, Future, FutureExt};
use notify_debouncer_mini::{new_debouncer, DebounceEventResult};
use pathdiff::diff_paths;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::env::current_dir;
use std::fs::canonicalize;
use std::io::ErrorKind::NotFound;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::mpsc::{Receiver, TryRecvError};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::mpsc::UnboundedReceiver;
use tokio::sync::mpsc::UnboundedSender;
extern crate notify;
// Path-only event from the file watcher. The notify-debouncer-mini debouncer collapses
// rapid filesystem events down to a single per-path notification, so the kind is irrelevant
// to chomp — we just need to know which path changed.
pub type WatchEvent = PathBuf;
use crate::engines::replace_env_vars_static;
use crate::engines::ExecState;
use anyhow::{anyhow, Result};
use derivative::Derivative;
use futures::executor;
use notify::{RecursiveMode, Watcher};
use std::sync::mpsc::channel;
use tokio::fs;
use tokio::time;
#[derive(Debug)]
pub struct Task<'a> {
name: Option<String>,
targets: Vec<String>,
deps: Vec<String>,
env: BTreeMap<String, String>,
chomp_task: &'a ChompTaskMaybeTemplated,
}
#[allow(dead_code)]
pub struct RunOptions {
pub args: Option<Vec<String>>,
pub cfg_file: PathBuf,
pub pool_size: usize,
pub targets: Vec<String>,
pub watch: bool,
pub rerun: bool,
pub force: bool,
}
#[derive(Hash, Eq, PartialEq, Debug, Clone, Copy)]
enum JobState {
Sentinel,
Uninitialized,
Initialized,
Checking,
Pending,
Running,
Fresh,
Failed,
}
#[derive(Derivative)]
#[derivative(Debug)]
struct Job {
interpolate: Option<String>,
task: usize,
deps: Vec<usize>,
parents: Vec<usize>,
live: bool,
state: JobState,
mtime: Option<Duration>,
#[derivative(Debug = "ignore")]
mtime_future: Option<Shared<Pin<Box<dyn Future<Output = Option<Duration>>>>>>,
targets: Vec<String>,
cmd_num: Option<usize>,
}
#[derive(Debug)]
enum Node {
Job(Job),
File(File),
}
#[derive(Hash, Eq, PartialEq, Debug, Clone)]
enum FileState {
Uninitialized,
Initialized,
Checking,
Found,
NotFound,
}
#[derive(Derivative)]
#[derivative(Debug)]
struct File {
name: String,
parents: Vec<usize>,
state: FileState,
mtime: Option<Duration>,
#[derivative(Debug = "ignore")]
mtime_future: Option<Shared<Pin<Box<dyn Future<Output = Option<Duration>>>>>>,
}
impl File {
fn new(name: String) -> File {
File {
name,
mtime: None,
parents: Vec::new(),
state: FileState::Uninitialized,
mtime_future: None,
}
}
fn init(&mut self, watcher: Option<&mut dyn Watcher>) {
self.state = FileState::Initialized;
if let Some(watcher) = watcher {
#[cfg(target_os = "windows")]
let name = self.name.replace('/', "\\");
#[cfg(not(target_os = "windows"))]
let name = &self.name;
match watcher.watch(Path::new(&name), RecursiveMode::Recursive) {
Ok(_) => {}
Err(_) => {
// eprintln!("Unable to watch {}", self.name);
}
};
}
}
}
fn find_interpolate(s: &str) -> Result<Option<(usize, bool)>> {
match s.find("##") {
Some(idx) => {
if s.find('#').unwrap() != idx || s[idx + 2..].find('#').is_some() {
return Err(anyhow!("Multiple interpolates in '{}' not supported", s));
}
Ok(Some((idx, true)))
}
None => match s.find('#') {
Some(idx) => {
if s[idx + 1..].find('#').is_some() {
return Err(anyhow!("Multiple interpolates in '{}' not supported", s));
}
Ok(Some((idx, false)))
}
None => Ok(None),
},
}
}
fn get_interpolate_match(interpolate: &str, path: &str) -> String {
let prefix_len = interpolate.find('#').unwrap();
let suffix_len = interpolate.len() - interpolate.rfind('#').unwrap() - 1;
path[prefix_len..path.len() - suffix_len].to_string()
}
fn check_interpolate_exclude(task: &Task, path: &str) -> bool {
// If the interpolated dependency matches its own task's target glob space, then we exclude it
// We can enable further custom ignores here in future
if let Some(interpolation_target) = task.targets.iter().find(|&t| t.contains('#')) {
let target_glob = if interpolation_target.contains("##") {
interpolation_target.replace("##", "(**/*)")
} else {
interpolation_target.replace('#', "(*)")
};
if Pattern::new(&target_glob).unwrap().matches(path) {
return true;
}
}
false
}
fn replace_interpolate(s: &str, replacement: &str) -> String {
if let Some((_, double)) = find_interpolate(s).unwrap() {
if double {
s.replace("##", replacement)
} else {
s.replace('#', replacement)
}
} else {
String::from(s)
}
}
pub struct Runner<'a> {
// ui: &'a ChompUI,
cwd: String,
cmd_pool: CmdPool<'a>,
chompfile: &'a Chompfile,
watch: bool,
tasks: Vec<Task<'a>>,
nodes: Vec<Node>,
task_jobs: HashMap<String, usize>,
file_nodes: HashMap<String, usize>,
interpolate_nodes: Vec<usize>,
}
impl<'a> Job {
fn new(task: usize, interpolate: Option<String>) -> Job {
Job {
interpolate,
task,
deps: Vec::new(),
live: false,
parents: Vec::new(),
state: JobState::Uninitialized,
targets: Vec::new(),
mtime: None,
cmd_num: None,
mtime_future: None,
}
}
fn display_name(&self, tasks: &[Task<'a>], cwd: &str) -> String {
let task = &tasks[self.task];
let mut skip_relative_path = true;
let name = if let Some(interpolate) = self.interpolate.as_ref() {
skip_relative_path = false;
if !task.targets.is_empty() {
match task.targets.iter().find(|&t| t.contains('#')) {
Some(interpolate_target) => replace_interpolate(interpolate_target, interpolate),
None => replace_interpolate(
task.deps.iter().find(|&d| d.contains('#')).unwrap(),
interpolate,
),
}
} else {
replace_interpolate(
task.deps.iter().find(|&d| d.contains('#')).unwrap(),
interpolate,
)
}
} else if !self.targets.is_empty() {
skip_relative_path = false;
self.targets.first().unwrap().to_string()
} else if let Some(name) = &task.name {
format!(":{}", name)
} else if let Some(run) = &task.chomp_task.run {
run.to_string()
} else {
format!("[task {}]", self.task)
};
if skip_relative_path {
name
} else {
relative_path(&name, cwd)
}
}
}
#[derive(Hash, Eq, PartialEq, Debug, Clone)]
enum JobOrFileState {
Job(JobState),
File(FileState),
}
#[derive(Hash, Eq, PartialEq, Debug, Clone)]
struct StateTransition {
node_num: usize,
cmd_num: Option<usize>,
state: JobOrFileState,
}
impl StateTransition {
fn from_job(node_num: usize, state: JobState, cmd_num: Option<usize>) -> Self {
StateTransition {
node_num,
cmd_num,
state: JobOrFileState::Job(state),
}
}
fn from_file(node_num: usize, state: FileState, cmd_num: Option<usize>) -> Self {
StateTransition {
node_num,
cmd_num,
state: JobOrFileState::File(state),
}
}
}
#[derive(Debug)]
struct QueuedStateTransitions {
state_transitions: HashSet<StateTransition>,
}
impl QueuedStateTransitions {
fn new() -> Self {
Self {
state_transitions: HashSet::new(),
}
}
fn insert_job(
&mut self,
node_num: usize,
state: JobState,
cmd_num: Option<usize>,
) -> Option<StateTransition> {
let transition = StateTransition::from_job(node_num, state, cmd_num);
if self.state_transitions.insert(transition.clone()) {
Some(transition)
} else {
None
}
}
fn insert_file(
&mut self,
node_num: usize,
state: FileState,
cmd_num: Option<usize>,
) -> Option<StateTransition> {
let transition = StateTransition::from_file(node_num, state, cmd_num);
if self.state_transitions.insert(transition.clone()) {
Some(transition)
} else {
None
}
}
fn remove_job(&mut self, node_num: usize, state: JobState, cmd_num: Option<usize>) -> bool {
let transition = StateTransition::from_job(node_num, state, cmd_num);
self.state_transitions.remove(&transition)
}
}
// None = NotFound
pub async fn check_target_mtimes(targets: Vec<String>, default_latest: bool) -> Option<Duration> {
if targets.is_empty() {
if default_latest {
return Some(now());
} else {
return None;
}
}
let mut futures = Vec::new();
for target in &targets {
let target_path = Path::new(target);
futures.push(
async move {
match fs::metadata(target_path).await {
Ok(n) => Some(
n.modified()
.expect("No modified implementation")
.duration_since(UNIX_EPOCH)
.unwrap(),
),
Err(e) => match e.kind() {
NotFound => None,
_ => panic!("Unknown file error"),
},
}
}
.boxed_local(),
);
}
let mut has_missing = false;
let mut last_mtime = None;
while !futures.is_empty() {
let (mtime, _, new_futures) = select_all(futures).await;
futures = new_futures;
if mtime.is_none() {
has_missing = true;
last_mtime = None;
} else if !has_missing && mtime > last_mtime {
last_mtime = mtime;
}
}
last_mtime
}
fn has_glob_chars(s: &str) -> bool {
s.contains('(') || s.contains('[') || s.contains('?') || s.contains('*')
}
fn now() -> std::time::Duration {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap()
}
// On Windows, we need to explicitly redefine wanted system-defined
// env vars since these are specifically promoted to local variables
// for the powershell exec
#[cfg(target_os = "windows")]
fn create_task_env(
task: &ChompTaskMaybeTemplated,
chompfile: &Chompfile,
replacements: bool,
) -> BTreeMap<String, String> {
let mut env = BTreeMap::new();
for (item, value) in &chompfile.env {
env.insert(
item.to_uppercase(),
if replacements {
replace_env_vars_static(value, &env)
} else {
value.to_string()
},
);
}
for (item, value) in &chompfile.env_default {
if !env.contains_key(item) {
if let Some(val) = std::env::var_os(item) {
env.insert(item.to_uppercase(), String::from(val.to_str().unwrap()));
} else {
env.insert(
item.to_uppercase(),
if replacements {
replace_env_vars_static(value, &env)
} else {
value.to_string()
},
);
}
}
}
if let Some(ref task_env) = task.env {
for (item, value) in task_env {
env.insert(
item.to_uppercase(),
if replacements {
replace_env_vars_static(value, &env)
} else {
value.to_string()
},
);
}
}
if let Some(ref task_env_default) = task.env_default {
for (item, value) in task_env_default {
if !env.contains_key(item) {
if let Some(val) = std::env::var_os(item) {
env.insert(item.to_uppercase(), String::from(val.to_str().unwrap()));
} else {
env.insert(
item.to_uppercase(),
if replacements {
replace_env_vars_static(value, &env)
} else {
value.to_string()
},
);
}
}
}
}
env
}
#[cfg(not(target_os = "windows"))]
fn create_task_env<'a>(
task: &ChompTaskM
gitextract__re7x5ob/
├── .github/
│ └── workflows/
│ ├── release.yml
│ └── test.yml
├── .gitignore
├── .gitmodules
├── Cargo.toml
├── LICENSE
├── README.md
├── chompfile.toml
├── docs/
│ ├── chompfile.md
│ ├── cli.md
│ ├── extensions.md
│ └── task.md
├── node-chomp/
│ ├── README.md
│ ├── index.js
│ └── package.json
├── src/
│ ├── ansi_windows.rs
│ ├── chompfile.rs
│ ├── engines/
│ │ ├── cmd.rs
│ │ ├── deno.rs
│ │ ├── mod.rs
│ │ └── node.rs
│ ├── extensions.rs
│ ├── http_client.rs
│ ├── main.rs
│ ├── server.rs
│ └── task.rs
└── test/
├── chompfile.toml
├── fixtures/
│ ├── app.js
│ ├── many/
│ │ ├── one/
│ │ │ └── config.yml
│ │ └── two/
│ │ └── config.yml
│ └── src/
│ ├── app.ts
│ └── dep.ts
└── unit/
└── ok-node.mjs
SYMBOL INDEX (155 symbols across 11 files)
FILE: src/ansi_windows.rs
function enable_ansi_support (line 10) | pub fn enable_ansi_support() -> Result<(), u32> {
FILE: src/chompfile.rs
type ChompEngine (line 29) | pub enum ChompEngine {
type TaskDisplay (line 39) | pub enum TaskDisplay {
type TaskStdio (line 52) | pub enum TaskStdio {
type Chompfile (line 65) | pub struct Chompfile {
type ServerOptions (line 85) | pub struct ServerOptions {
function default_root (line 92) | fn default_root() -> String {
function default_port (line 96) | fn default_port() -> u16 {
method default (line 101) | fn default() -> Self {
type InvalidationCheck (line 112) | pub enum InvalidationCheck {
type ValidationCheck (line 122) | pub enum ValidationCheck {
type WatchInvalidation (line 136) | pub enum WatchInvalidation {
type ChompTaskMaybeTemplated (line 145) | pub struct ChompTaskMaybeTemplated {
method new (line 170) | pub fn new() -> Self {
method targets_vec (line 195) | pub fn targets_vec(&self, cwd: &str) -> Result<Vec<String>> {
method deps_vec (line 206) | pub fn deps_vec(&self, chompfile: &Chompfile, cwd: &str) -> Result<Vec...
method from (line 273) | fn from(val: ChompTaskMaybeTemplatedJs) -> Self {
function skip_special_chars (line 238) | fn skip_special_chars(s: &str) -> bool {
function is_default (line 242) | fn is_default<T: Default + PartialEq>(t: &T) -> bool {
type ChompTaskMaybeTemplatedJs (line 248) | pub struct ChompTaskMaybeTemplatedJs {
function resolve_path (line 300) | pub fn resolve_path(target: &str, cwd: &str) -> String {
function path_from (line 307) | pub fn path_from<P: AsRef<Path>>(base_dir: P, input: &str) -> PathBuf {
function normalize_path (line 339) | pub fn normalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
FILE: src/engines/cmd.rs
function replace_env_vars (line 27) | fn replace_env_vars(arg: &str, env: &BTreeMap<String, String>) -> String {
function set_cmd_stdio (line 80) | fn set_cmd_stdio(command: &mut Command, stdio: TaskStdio) {
function create_cmd (line 103) | pub fn create_cmd(
function create_cmd (line 263) | pub fn create_cmd(
FILE: src/engines/deno.rs
constant DENO_CMD (line 28) | const DENO_CMD: &str = "deno run -A --unstable --no-check $CHOMP_MAIN";
function deno_runner (line 30) | pub fn deno_runner(cmd_pool: &mut CmdPool, mut cmd: BatchCmd, targets: V...
FILE: src/engines/mod.rs
function replace_env_vars_static (line 47) | pub fn replace_env_vars_static(arg: &str, env: &BTreeMap<String, String>...
type CmdPool (line 77) | pub struct CmdPool<'a> {
type CmdOp (line 93) | pub struct CmdOp {
type BatchCmd (line 106) | pub struct BatchCmd {
type ExecState (line 119) | pub enum ExecState {
type Exec (line 128) | pub struct Exec<'a> {
function new (line 137) | pub fn new(
function terminate (line 177) | pub fn terminate(&mut self, cmd_num: usize, name: &str) {
function get_exec_future (line 190) | pub fn get_exec_future(
function create_batch_future (line 220) | fn create_batch_future(&mut self) {
function new_exec (line 315) | async fn new_exec(&mut self, mut cmd: BatchCmd) {
function batch (line 393) | pub fn batch(
FILE: src/engines/node.rs
constant NODE_LOADER (line 31) | const NODE_LOADER: &str = "let s;export function resolve(u,c,d){if(c.par...
function node_runner (line 33) | pub fn node_runner(cmd_pool: &mut CmdPool, mut cmd: BatchCmd, targets: V...
FILE: src/extensions.rs
type ExtensionEnvironment (line 34) | pub struct ExtensionEnvironment {
method new (line 295) | pub fn new(global_env: &BTreeMap<String, String>) -> Self {
method handle_scope (line 370) | fn handle_scope(&mut self) -> v8::HandleScope<'_> {
method get_tasks (line 374) | pub fn get_tasks(&self) -> Vec<ChompTaskMaybeTemplatedJs> {
method get_extensions (line 383) | fn get_extensions(&self) -> &Rc<RefCell<Extensions>> {
method add_extension (line 387) | pub fn add_extension(
method seal_extensions (line 429) | pub fn seal_extensions(&mut self) {
method run_template (line 434) | pub fn run_template(
method has_batchers (line 480) | pub fn has_batchers(&self) -> bool {
method run_batcher (line 484) | pub fn run_batcher(
type BatcherResult (line 42) | pub struct BatcherResult {
type Extensions (line 48) | struct Extensions {
method new (line 57) | fn new() -> Self {
function create_template_options (line 68) | fn create_template_options(
function expand_template_tasks (line 97) | pub fn expand_template_tasks(
function init_js_platform (line 182) | pub fn init_js_platform() {
function chomp_log (line 188) | fn chomp_log(
function chomp_include (line 208) | fn chomp_include(
function chomp_register_task (line 227) | fn chomp_register_task(
function chomp_register_template (line 246) | fn chomp_register_template(
function chomp_register_batcher (line 266) | fn chomp_register_batcher(
function v8_exception (line 529) | fn v8_exception(scope: &mut v8::TryCatch<v8::HandleScope>) -> Error {
function get_property (line 554) | fn get_property<'a>(
function is_instance_of_error (line 563) | fn is_instance_of_error<'s>(scope: &mut v8::HandleScope<'s>, value: v8::...
FILE: src/http_client.rs
function chomp_cache_dir (line 29) | fn chomp_cache_dir() -> PathBuf {
function clear_cache (line 36) | pub async fn clear_cache() -> std::io::Result<()> {
function prep_cache (line 46) | pub async fn prep_cache() -> Result<()> {
function u4_to_hex_char (line 52) | fn u4_to_hex_char(c: u8) -> char {
function hash (line 56) | pub fn hash(input: &[u8]) -> String {
function from_cache (line 68) | async fn from_cache(cache_key: &str) -> Option<String> {
function write_cache (line 80) | async fn write_cache(cache_key: &str, source: &str) -> Result<()> {
function fetch_uri_cached (line 87) | pub async fn fetch_uri_cached(uri_str: &str, uri: Uri) -> Result<String> {
FILE: src/main.rs
constant CHOMP_CORE (line 50) | const CHOMP_CORE: &str = "https://ga.jspm.io/npm:@chompbuild/extensions@...
constant CHOMP_INIT (line 52) | const CHOMP_INIT: &str = r#"version = 0.1
constant CHOMP_EMPTY (line 59) | const CHOMP_EMPTY: &str = "version = 0.1\n";
function uri_parse (line 61) | fn uri_parse(uri_str: &str) -> Option<Uri> {
function main (line 68) | async fn main() -> Result<()> {
FILE: src/server.rs
type ResponseBody (line 35) | type ResponseBody = Bytes;
function client_connection (line 37) | async fn client_connection(ws: WebSocket, state: State) {
type Client (line 90) | pub struct Client {
type StateStruct (line 95) | pub struct StateStruct {
method new (line 101) | fn new() -> StateStruct {
type State (line 109) | pub type State = Arc<RwLock<StateStruct>>;
type FileEvent (line 111) | pub enum FileEvent {
function check_watcher (line 115) | async fn check_watcher(mut rx: UnboundedReceiver<WatchEvent>, root: &Pat...
function revalidate (line 127) | async fn revalidate(
function not_found (line 158) | fn not_found(resource: &str) -> Response<ResponseBody> {
function file_serve (line 169) | async fn file_serve(path: &PathBuf, root: &PathBuf, hash: Option<String>...
function index_page (line 201) | async fn index_page(path: &mut PathBuf, root: &PathBuf) -> Option<Respon...
function serve (line 241) | pub async fn serve(
FILE: src/task.rs
type WatchEvent (line 49) | pub type WatchEvent = PathBuf;
type Task (line 62) | pub struct Task<'a> {
type RunOptions (line 71) | pub struct RunOptions {
type JobState (line 82) | enum JobState {
type Job (line 95) | struct Job {
method new (line 233) | fn new(task: usize, interpolate: Option<String>) -> Job {
method display_name (line 248) | fn display_name(&self, tasks: &[Task<'a>], cwd: &str) -> String {
type Node (line 110) | enum Node {
type FileState (line 116) | enum FileState {
type File (line 126) | struct File {
method new (line 136) | fn new(name: String) -> File {
method init (line 146) | fn init(&mut self, watcher: Option<&mut dyn Watcher>) {
function find_interpolate (line 163) | fn find_interpolate(s: &str) -> Result<Option<(usize, bool)>> {
function get_interpolate_match (line 183) | fn get_interpolate_match(interpolate: &str, path: &str) -> String {
function check_interpolate_exclude (line 189) | fn check_interpolate_exclude(task: &Task, path: &str) -> bool {
function replace_interpolate (line 205) | fn replace_interpolate(s: &str, replacement: &str) -> String {
type Runner (line 217) | pub struct Runner<'a> {
type JobOrFileState (line 287) | enum JobOrFileState {
type StateTransition (line 293) | struct StateTransition {
method from_job (line 300) | fn from_job(node_num: usize, state: JobState, cmd_num: Option<usize>) ...
method from_file (line 307) | fn from_file(node_num: usize, state: FileState, cmd_num: Option<usize>...
type QueuedStateTransitions (line 317) | struct QueuedStateTransitions {
method new (line 322) | fn new() -> Self {
method insert_job (line 327) | fn insert_job(
method insert_file (line 340) | fn insert_file(
method remove_job (line 353) | fn remove_job(&mut self, node_num: usize, state: JobState, cmd_num: Op...
function check_target_mtimes (line 360) | pub async fn check_target_mtimes(targets: Vec<String>, default_latest: b...
function has_glob_chars (line 404) | fn has_glob_chars(s: &str) -> bool {
function now (line 408) | fn now() -> std::time::Duration {
function create_task_env (line 416) | fn create_task_env(
function create_task_env (line 482) | fn create_task_env<'a>(
function new (line 540) | pub fn new(
function add_job (line 583) | fn add_job(&mut self, task_num: usize, interpolate: Option<String>) -> R...
function add_file (line 686) | fn add_file(&mut self, file: String) -> Result<usize> {
function get_job (line 700) | fn get_job(&self, num: usize) -> Option<&Job> {
function get_job_mut (line 708) | fn get_job_mut(&mut self, num: usize) -> Option<&mut Job> {
function get_file_mut (line 716) | fn get_file_mut(&mut self, num: usize) -> Option<&mut File> {
function mark_complete (line 723) | fn mark_complete(
function invalidate_job (line 800) | fn invalidate_job(
function invalidate_path (line 860) | fn invalidate_path(
function expand_job_deps (line 885) | fn expand_job_deps(&self, job_num: usize, deps: &mut Vec<String>) {
function run_job (line 914) | fn run_job(
function drive_all (line 1193) | fn drive_all(
function drive_completion (line 1424) | fn drive_completion(
function lookup_task (line 1543) | fn lookup_task(&mut self, task: usize) -> Option<usize> {
function lookup_task_name (line 1558) | async fn lookup_task_name(
function get_interpolate_target (line 1599) | fn get_interpolate_target(&self, interpolate_job: usize) -> Option<&Stri...
function match_interpolate_target (line 1610) | fn match_interpolate_target(&self, target: &str) -> Result<Option<(usize...
function interpolate_dep_input (line 1637) | fn interpolate_dep_input(&self, task_num: usize, interpolate: &str) -> O...
function lookup_target (line 1646) | async fn lookup_target(
function lookup_glob_target (line 1687) | async fn lookup_glob_target(
function check_acyclic (line 1854) | fn check_acyclic(&self, root: usize) -> Result<()> {
function node_display (line 1893) | fn node_display(&self, node: usize) -> String {
function expand_target (line 1901) | async fn expand_target(
function expand_job (line 1921) | async fn expand_job(
function expand_interpolate (line 2050) | async fn expand_interpolate(
function expand_interpolate_match (line 2106) | async fn expand_interpolate_match(
function drive_jobs (line 2231) | async fn drive_jobs(
function watcher_interval (line 2306) | async fn watcher_interval() -> StateTransition {
function check_watcher (line 2315) | async fn check_watcher(
function run (line 2363) | pub async fn run(
function relative_path (line 2460) | pub fn relative_path(name: &str, cwd: &str) -> String {
Condensed preview — 33 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (296K chars).
[
{
"path": ".github/workflows/release.yml",
"chars": 2800,
"preview": "on:\n push:\n tags: '*'\n\nname: Create Release\n\njobs:\n publish-crate:\n name: Publish to crates.io\n runs-on: ubun"
},
{
"path": ".github/workflows/test.yml",
"chars": 1468,
"preview": "name: Test\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n\nenv:\n CARGO_TERM_COLOR: always\n "
},
{
"path": ".gitignore",
"chars": 83,
"preview": "node_modules\ntarget\ntest/package.json\npackage-lock.json\nsandbox\ntest/output\nvendor\n"
},
{
"path": ".gitmodules",
"chars": 0,
"preview": ""
},
{
"path": "Cargo.toml",
"chars": 1500,
"preview": "[package]\nname = \"chompbuild\"\nversion = \"0.3.0\"\nauthors = [\"Guy Bedford <guybedford@gmail.com>\"]\nedition = \"2021\"\nlicens"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": "\n Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 11326,
"preview": "# CHOMP\n\n[](https://crates.io/crates/chompbuild"
},
{
"path": "chompfile.toml",
"chars": 1036,
"preview": "version = 0.1\n\ndefault-task = 'build'\n\n[[task]]\nname = 'build'\ndeps = ['src/**/*.rs']\nrun = 'cargo build'\n\n[[task]]\nname"
},
{
"path": "docs/chompfile.md",
"chars": 2207,
"preview": "# Chompfile\n\nChomp projects are defined by a `chompfile.toml`, with Chompfiles defined using the [TOML configuration for"
},
{
"path": "docs/cli.md",
"chars": 4847,
"preview": "# CLI Flags\n\nUsage:\n\n```\nchomp [FLAGS/OPTIONS] <TARGET>...\n```\n\nChomp takes the following arguments and flags:\n\n* [`<TAR"
},
{
"path": "docs/extensions.md",
"chars": 14816,
"preview": "# Extensions\n\n## Overview\n\nExecutions are loaded through the embedded V8 environment in Chomp, which includes a very sim"
},
{
"path": "docs/task.md",
"chars": 25169,
"preview": "# Chomp Tasks\n\nThe [`chompfile.toml`](chompfile.md) defines Chomp tasks as a list of Task objects of the form:\n\n_chompfi"
},
{
"path": "node-chomp/README.md",
"chars": 434,
"preview": "# Chomp Node\n\nNode.js wrapper for [Chomp](https://chompbuild.com)\n\nUsable via your favourite JS package manager:\n\n```\nnp"
},
{
"path": "node-chomp/index.js",
"chars": 886,
"preview": "#!/usr/bin/env node\n\nimport BinWrapper from 'bin-wrapper';\nimport { readFileSync } from 'fs';\nimport { spawn } from 'chi"
},
{
"path": "node-chomp/package.json",
"chars": 698,
"preview": "{\n \"name\": \"chomp\",\n \"version\": \"0.3.0\",\n \"description\": \"'JS Make' - parallel task runner CLI for the frontend ecosy"
},
{
"path": "src/ansi_windows.rs",
"chars": 2435,
"preview": "/// Enables ANSI code support on Windows 10.\n///\n/// This uses Windows API calls to alter the properties of the console "
},
{
"path": "src/chompfile.rs",
"chars": 11068,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/engines/cmd.rs",
"chars": 12363,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/engines/deno.rs",
"chars": 3356,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/engines/mod.rs",
"chars": 15540,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/engines/node.rs",
"chars": 3899,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/extensions.rs",
"chars": 21143,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/http_client.rs",
"chars": 3276,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/main.rs",
"chars": 19527,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/server.rs",
"chars": 11295,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "src/task.rs",
"chars": 96596,
"preview": "// Chomp Task Runner\n// Copyright (C) 2022 Guy Bedford\n\n// This program is free software: you can redistribute it and/o"
},
{
"path": "test/chompfile.toml",
"chars": 5330,
"preview": "version = 0.1\nextensions = ['chomp@0.1:assert', 'chomp@0.1:npm']\ndefault-task = 'test'\n\n[env]\nVAL = 'C'\n\n[env-default]\nD"
},
{
"path": "test/fixtures/app.js",
"chars": 18,
"preview": "export var p = 5;\n"
},
{
"path": "test/fixtures/many/one/config.yml",
"chars": 10,
"preview": "name: one\n"
},
{
"path": "test/fixtures/many/two/config.yml",
"chars": 10,
"preview": "name: two\n"
},
{
"path": "test/fixtures/src/app.ts",
"chars": 78,
"preview": "import { dep } from './dep.js';\n\nconsole.log(dep);\n\nexport var p: number = 5;\n"
},
{
"path": "test/fixtures/src/dep.ts",
"chars": 34,
"preview": "export const dep: string = 'dep';\n"
},
{
"path": "test/unit/ok-node.mjs",
"chars": 148,
"preview": "import { writeFileSync } from 'fs';\n\nwriteFileSync('output/unittest.txt', 'UNIT OK');\n\nconsole.log('THIS SHOULD NEVER DI"
}
]
About this extraction
This page contains the full source code of the guybedford/chomp GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 33 files (278.1 KB), approximately 63.5k tokens, and a symbol index with 155 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.