Repository: NeowayLabs/nash Branch: master Commit: a227041ffd50 Files: 174 Total size: 642.6 KB Directory structure: gitextract_5jvfgdox/ ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── _disabled_appveyor.yml ├── ast/ │ ├── doc_test.go │ ├── node.go │ ├── node_args.go │ ├── node_fmt.go │ ├── node_fmt_test.go │ ├── nodetype_string.go │ ├── tree.go │ └── tree_test.go ├── cmd/ │ ├── nash/ │ │ ├── cli.go │ │ ├── completer.go │ │ ├── env.go │ │ ├── env_test.go │ │ ├── example.sh │ │ ├── install.go │ │ ├── install_test.go │ │ ├── main.go │ │ ├── rpc.go │ │ ├── vendor/ │ │ │ └── github.com/ │ │ │ └── chzyer/ │ │ │ └── readline/ │ │ │ ├── .travis.yml │ │ │ ├── CHANGELOG.md │ │ │ ├── LICENSE │ │ │ ├── README.md │ │ │ ├── ansi_windows.go │ │ │ ├── complete.go │ │ │ ├── complete_helper.go │ │ │ ├── complete_segment.go │ │ │ ├── complete_segment_test.go │ │ │ ├── doc/ │ │ │ │ └── shortcut.md │ │ │ ├── example/ │ │ │ │ ├── readline-demo/ │ │ │ │ │ └── readline-demo.go │ │ │ │ ├── readline-im/ │ │ │ │ │ ├── README.md │ │ │ │ │ └── readline-im.go │ │ │ │ ├── readline-multiline/ │ │ │ │ │ └── readline-multiline.go │ │ │ │ ├── readline-pass-strength/ │ │ │ │ │ └── readline-pass-strength.go │ │ │ │ └── readline-remote/ │ │ │ │ ├── readline-remote-client/ │ │ │ │ │ └── client.go │ │ │ │ └── readline-remote-server/ │ │ │ │ └── server.go │ │ │ ├── history.go │ │ │ ├── operation.go │ │ │ ├── password.go │ │ │ ├── rawreader_windows.go │ │ │ ├── readline.go │ │ │ ├── readline_test.go │ │ │ ├── remote.go │ │ │ ├── runebuf.go │ │ │ ├── runes/ │ │ │ │ ├── runes.go │ │ │ │ └── runes_test.go │ │ │ ├── runes.go │ │ │ ├── runes_test.go │ │ │ ├── std.go │ │ │ ├── std_windows.go │ │ │ ├── term.go │ │ │ ├── term_bsd.go │ │ │ ├── term_linux.go │ │ │ ├── term_windows.go │ │ │ ├── terminal.go │ │ │ ├── utils.go │ │ │ ├── utils_test.go │ │ │ ├── utils_unix.go │ │ │ ├── utils_windows.go │ │ │ ├── vim.go │ │ │ └── windows_api.go │ │ └── vendor.sh │ └── nashfmt/ │ └── main.go ├── docs/ │ ├── interactive.md │ ├── reference.md │ └── stdlib/ │ └── fmt.md ├── errors/ │ └── error.go ├── examples/ │ ├── append.sh │ ├── args.sh │ ├── init │ └── len.sh ├── examples_test.go ├── fuzz.go ├── hack/ │ ├── check.sh │ ├── install/ │ │ └── unix.sh │ └── releaser.sh ├── internal/ │ ├── sh/ │ │ ├── builtin/ │ │ │ ├── append.go │ │ │ ├── append_test.go │ │ │ ├── chdir.go │ │ │ ├── doc.go │ │ │ ├── exec_test.go │ │ │ ├── exit.go │ │ │ ├── exit_test.go │ │ │ ├── format.go │ │ │ ├── format_test.go │ │ │ ├── glob.go │ │ │ ├── glob_test.go │ │ │ ├── len.go │ │ │ ├── len_test.go │ │ │ ├── loader.go │ │ │ ├── print.go │ │ │ ├── print_test.go │ │ │ ├── split.go │ │ │ ├── split_test.go │ │ │ └── testdata/ │ │ │ ├── exit.sh │ │ │ ├── split.sh │ │ │ └── splitfunc.sh │ │ ├── builtin.go │ │ ├── cmd.go │ │ ├── cmd_test.go │ │ ├── fncall.go │ │ ├── fndef.go │ │ ├── functions_test.go │ │ ├── internal/ │ │ │ └── fixture/ │ │ │ └── fixture.go │ │ ├── ioutils_test.go │ │ ├── log.go │ │ ├── rfork.go │ │ ├── rfork_linux.go │ │ ├── rfork_linux_test.go │ │ ├── rfork_plan9.go │ │ ├── shell.go │ │ ├── shell_import_test.go │ │ ├── shell_linux_test.go │ │ ├── shell_regression_test.go │ │ ├── shell_test.go │ │ ├── shell_var_test.go │ │ ├── util.go │ │ └── util_test.go │ └── testing/ │ └── fixture/ │ └── io.go ├── nash.go ├── nash_test.go ├── parser/ │ ├── parse.go │ ├── parse_fmt_test.go │ ├── parse_regression_test.go │ └── parse_test.go ├── proposal/ │ ├── 1-scope-management.md │ └── 2-concurrency.md ├── scanner/ │ ├── examples_test.go │ ├── lex.go │ ├── lex_regression_test.go │ └── lex_test.go ├── sh/ │ ├── obj.go │ ├── objtype_string.go │ └── shell.go ├── spec.ebnf ├── spec_test.go ├── stdbin/ │ ├── mkdir/ │ │ ├── main.go │ │ └── mkdir_test.go │ ├── pwd/ │ │ └── main.go │ ├── strings/ │ │ ├── main.go │ │ ├── strings.go │ │ └── strings_test.go │ └── write/ │ ├── common_test.sh │ ├── fd.go │ ├── fd_windows.go │ ├── main.go │ ├── write.go │ ├── write_linux_test.sh │ └── write_test.sh ├── stdlib/ │ ├── io.sh │ ├── io_example.sh │ ├── io_test.sh │ ├── map.sh │ └── map_test.sh ├── testfiles/ │ ├── ex1.sh │ ├── fibonacci.sh │ └── sieve.sh ├── tests/ │ ├── cfg.go │ ├── doc.go │ ├── internal/ │ │ ├── assert/ │ │ │ ├── doc.go │ │ │ ├── equal.go │ │ │ └── error.go │ │ ├── sh/ │ │ │ └── shell.go │ │ └── tester/ │ │ └── tester.go │ ├── listindex_test.go │ └── stringindex_test.go ├── token/ │ └── token.go └── vendor/ └── golang.org/ └── x/ └── exp/ └── ebnf/ ├── ebnf.go └── parser.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ ./cmd/cnt/cnt /cmd/nash/nash /coverage.txt cmd/nashfmt/nashfmt dist *.exe stdbin/mkdir/mkdir stdbin/pwd/pwd stdbin/strings/strings stdbin/write/write coverage.html ================================================ FILE: .travis.yml ================================================ os: - linux language: go sudo: false go: - "tip" - "1.12" install: - go get -v golang.org/x/exp/ebnf - make build script: - go get github.com/axw/gocov/gocov - go get github.com/mattn/goveralls - go get golang.org/x/tools/cmd/cover - mkdir $HOME/nashroot - make test - make build - ./cmd/nash/nash ./hack/releaser.sh testci after_success: - bash <(curl -s https://codecov.io/bash) notifications: webhooks: urls: - https://webhooks.gitter.im/e/52ad02845e880cdca2cf on_success: change on_failure: always on_start: never email: - tiago.natel@neoway.com.br - tiagokatcipis@gmail.com ================================================ FILE: Dockerfile ================================================ FROM golang:1.12 ADD . /go/src/github.com/madlambda/nash ENV NASHPATH /nashpath ENV NASHROOT /nashroot RUN cd /go/src/github.com/madlambda/nash && \ make install CMD ["/nashroot/bin/nash"] ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2016 Neoway Business Solution Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ ifndef version version=$(shell git rev-list -1 HEAD) endif buildargs = -ldflags "-X main.VersionString=$(version)" -v all: build test install build: cd cmd/nash && go build $(buildargs) cd cmd/nashfmt && go build $(buildargs) cd stdbin/mkdir && go build $(buildargs) cd stdbin/pwd && go build $(buildargs) cd stdbin/write && go build $(buildargs) cd stdbin/strings && go build $(buildargs) NASHPATH?=$(HOME)/nash NASHROOT?=$(HOME)/nashroot # FIXME: binaries install do not work on windows this way (the .exe) install: build @echo @echo "Installing nash at: "$(NASHROOT) mkdir -p $(NASHROOT)/bin rm -f $(NASHROOT)/bin/nash rm -f $(NASHROOT)/bin/nashfmt cp -p ./cmd/nash/nash $(NASHROOT)/bin cp -p ./cmd/nashfmt/nashfmt $(NASHROOT)/bin rm -rf $(NASHROOT)/stdlib cp -pr ./stdlib $(NASHROOT)/stdlib cp -pr ./stdbin/mkdir/mkdir $(NASHROOT)/bin/mkdir cp -pr ./stdbin/pwd/pwd $(NASHROOT)/bin/pwd cp -pr ./stdbin/write/write $(NASHROOT)/bin/write cp -pr ./stdbin/strings/strings $(NASHROOT)/bin/strings docsdeps: go get github.com/madlambda/mdtoc/cmd/mdtoc docs: docsdeps mdtoc -w ./README.md mdtoc -w ./docs/interactive.md mdtoc -w ./docs/reference.md mdtoc -w ./docs/stdlib/fmt.md test: build ./hack/check.sh update-vendor: cd cmd/nash && nash ./vendor.sh release: clean ./hack/releaser.sh $(version) coverage-html: test go tool cover -html=coverage.txt -o coverage.html @echo "coverage file: coverage.html" coverage-show: coverage-html xdg-open coverage.html clean: rm -f cmd/nash/nash rm -f cmd/nashfmt/nashfmt rm -rf dist ================================================ FILE: README.md ================================================ # Table of Contents - [nash](#nash) - [Show time!](#show-time) - [Useful stuff](#useful-stuff) - [Why nash scripts are reliable?](#why-nash-scripts-are-reliable) - [Installation](#installation) - [Release](#release) - [Linux](#linux) - [Master](#master) - [Getting started](#getting-started) - [Accessing command line args](#accessing-command-line-args) - [Namespace features](#namespace-features) - [OK, but how scripts should look like?](#ok-but-how-scripts-should-look-like) - [Didn't work?](#didnt-work) - [Language specification](#language-specification) - [Some Bash comparisons](#some-bash-comparisons) - [Security](#security) - [Installing libraries](#installing-libraries) - [Releasing](#releasing) - [Want to contribute?](#want-to-contribute) # nash [![Join the chat at https://gitter.im/madlambda/nash](https://badges.gitter.im/madlambda/nash.svg)](https://gitter.im/madlambda/nash?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![GoDoc](https://godoc.org/github.com/madlambda/nash?status.svg)](https://godoc.org/github.com/madlambda/nash) [![Build Status](https://travis-ci.org/madlambda/nash.svg?branch=master)](https://travis-ci.org/madlambda/nash) [![Go Report Card](https://goreportcard.com/badge/github.com/madlambda/nash)](https://goreportcard.com/report/github.com/madlambda/nash) Nash is a system shell, inspired by plan9 `rc`, that makes it easy to create reliable and safe scripts taking advantages of operating systems namespaces (on linux and plan9) in an idiomatic way. ## Useful stuff - nashfmt: Formats nash code (like gofmt) but no code styling defined yet (see Installation section). - [nashcomplete](https://github.com/madlambda/nashcomplete): Autocomplete done in nash script. - [Dotnash](https://github.com/lborguetti/dotnash): Nash profile customizations (e.g: prompt, aliases, etc) - [nash-mode](https://github.com/tiago4orion/nash-mode.el): Emacs major mode integrated with `nashfmt`. ## Why nash scripts are reliable? 1. Nash aborts at first non-success status of commands; 2. Nash aborts at first unbound variable; 3. It's possible to check the result status of every component of a pipe; 4. **no eval**; 5. Strings are pure strings (no evaluation of variables); 6. No wildcards (globbing) of files; ('rm \*' removes a file called '\*'); - On windows, the terminal does the globbing when in interactive mode. - On unix there's libs/completions to achieve something similar. 7. No [obscure](http://explainshell.com/) syntax; 8. Support tooling for indent/format and statically analyze the scripts; ## Installation Nash uses two environment variables: **NASHROOT** to find the standard nash library and **NASHPATH** to find libraries in general (like user's code). It is important to have two different paths since this will allow you to upgrade nash (overwrite nash stdlib) without risking lost your code. If **NASHPATH** is not set, a default of $HOME/nash will be assumed ($HOMEPATH/nash on windows). If **NASHROOT** is not set, a default of $HOME/nashroot will be assumed ($HOMEPATH/nashroot on windows). The libraries lookup dir will be $NASHPATH/lib. The standard library lookup dir will be $NASHROOT/stdlib. After installing the nash binary will be located at $NASHROOT/bin. ### Installing a Release Installing is so stupid that we provide small scripts to do it. If your platform is not supported take a look at the existent ones and send a MR with the script for your platform. #### Unix If you run a unix based machine (Linux, Darwin/OSX, *BSD, etc) you can use the script below: Run: ``` ./hack/install/unix.sh ``` ### Master Run: ``` make install ``` ## Getting started Nash syntax resembles a common shell: ``` nash λ> echo "hello world" hello world ``` Pipes works like borne shell and derivations: ```sh λ> cat spec.ebnf | wc -l 108 ``` Output redirection works like Plan9 rc, but not only for filenames. It supports output redirection to tcp, udp and unix network protocols (unix sockets are not supported on windows). ```sh # stdout to log.out, stderr to log.err λ> ./daemon >[1] log.out >[2] log.err # stderr pointing to stdout λ> ./daemon-logall >[2=1] # stdout to /dev/null λ> ./daemon-quiet >[1=] # stdout and stderr to tcp address λ> ./daemon >[1] "udp://syslog:6666" >[2=1] # stdout to unix file λ> ./daemon >[1] "unix:///tmp/syslog.sock" ``` **For safety, there's no `eval` or `string/tilde expansion` or `command substitution` in Nash.** To assign command output to a variable exists the '<=' operator. See the example below: ```sh var fullpath <= realpath $path | xargs -n echo echo $fullpath ``` The symbol '<=' redirects the stdout of the command or function invocation in the right-hand side to the variable name specified. If you want the command output splited into an array, then you'll need to store it in a temporary variable and then use the builtin `split` function. ```sh var out <= find . var files <= split($out, "\n") for f in $files { echo "File: " + $f } ``` To avoid problems with spaces in variables being passed as multiple arguments to commands, nash pass the contents of each variable as a single argument to the command. It works like enclosing every variable with quotes before executing the command. Then the following example do the right thing: ```sh var fullname = "John Nash" ./ci-register --name $fullname --option somevalue ``` On bash you need to enclose the `$fullname` variable in quotes to avoid problems. Nash syntax does not support shell expansion from strings. There's no way to do things like the following in nash: ```bash echo "The date is: $(date +%D)" # DOESNT WORKS! ``` Instead you need to assign each command output to a proper variable and then concat it with another string when needed (see the [reference docs](./docs/reference.md)). In the same way, nash doesn't support shell expansion at `if` condition. For check if a directory exists you must use: ```sh -test -d $rootfsDir # if you forget '-', the script will be aborted here # if path not exists if $status != "0" { echo "RootFS does not exists." exit $status } ``` Nash stops executing the script at first error found and, in the majority of times, it is what you want (specially for deploys). But Commands have an explicitly way to bypass such restriction by prepending a dash '-' to the command statement. For example: ```sh fn cleanup() -rm -rf $buildDir -rm -rf $tmpDir } ``` The dash '-' works only for operating system commands, other kind of errors are impossible to bypass. For example, trying to evaluate an unbound variable aborts the program with error. ```sh λ> echo $PATH /bin:/sbin:/usr/bin:/usr/local/bin:/home/user/.local/bin:/home/user/bin:/home/user/.gvm/pkgsets/go1.5.3/global/bin:/home/user/projects/3rdparty/plan9port/bin:/home/user/.gvm/gos/go1.5.3/bin λ> echo $bleh ERROR: Variable '$bleh' not set ``` Long commands can be split in multiple lines: ```sh λ> (aws ec2 attach-internet-gateway --internet-gateway-id $igwid --vpc-id $vpcid) λ> var instanceId <= ( aws ec2 run-instances --image-id ami-xxxxxxxx --count 1 --instance-type t1.micro --key-name MyKeyPair --security-groups my-sg | jq ".Instances[0].InstanceId" ) λ> echo $instanceId ``` # Accessing command line args When you run a nash script like: ``` λ> nash ./examples/args.sh --arg value ``` You can get the args using the **ARGS** variable, that is a list: ``` #!/usr/bin/env nash echo "iterating through the arguments list" echo "" for arg in $ARGS { echo $arg } ``` # Namespace features Nash is built with namespace support only on Linux (Plan9 soon). If you use OSX, BSD or Windows, then the `rfork` keyword will fail. *The examples below assume you are on a Linux box.* Below are some facilities for namespace management inside nash. Make sure you have USER namespaces enabled in your kernel: ```sh zgrep CONFIG_USER_NS /proc/config.gz CONFIG_USER_NS=y ``` If it's not enabled you will need root privileges to execute every example below... Creating a new process in a new USER namespace (u): ```sh λ> id uid=1000(user) gid=1000(user) groups=1000(user),98(qubes) λ> rfork u { id } uid=0(root) gid=0(root) groups=0(root),65534 ``` Yes, Linux supports creation of containers by unprivileged users. Tell this to the customer success of your container-infrastructure-vendor. :-) The default UID mapping is: Current UID (getuid) => 0 (no range support). I'll look into more options for this in the future. Yes, you can create multiple nested user namespaces. But kernel limits the number of nested user namespace clones to 32. ```sh λ> rfork u { echo "inside first container" id rfork u { echo "inside second namespace..." id } } ``` You can verify that other types of namespace still requires root capabilities, see for PID namespaces (p). ```sh λ> rfork p { id } ERROR: fork/exec ./nash: operation not permitted ``` The same happens for mount (m), ipc (i) and uts (s) if used without user namespace (u) flag. The `c` flag stands for "container" and is an alias for upmnis (all types of namespaces). If you want another shell (maybe bash) inside the namespace: ```sh λ> rfork c { bash } [root@stay-away nash]# id uid=0(root) gid=0(root) groups=0(root),65534 [root@stay-away nash]# mount -t proc proc /proc [root@stay-away nash]# [root@stay-away nash]# ps aux USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND root 1 0.0 0.0 34648 2748 pts/4 Sl 17:32 0:00 -rcd- -addr /tmp/nash.qNQa.sock root 5 0.0 0.0 16028 3840 pts/4 S 17:32 0:00 /usr/bin/bash root 23 0.0 0.0 34436 3056 pts/4 R+ 17:34 0:00 ps aux ``` Everything except the `rfork` is like a common shell. Rfork will spawn a new process with the namespace flags and executes the commands inside the block on this namespace. It has the form: ```sh rfork { } ``` # OK, but how scripts should look like? See the project [nash-app-example](https://github.com/madlambda/nash-app-example). # Didn't work? I've tested in the following environments: Linux 4.7-rc7 Archlinux Linux 4.5.5 (amd64) Archlinux Linux 4.3.3 (amd64) Archlinux Linux 4.1.13 (amd64) Fedora release 23 Linux 4.1.13 (amd64) Debian 8 # Language specification The specification isn't complete yet, but can be found [here](https://github.com/madlambda/nash/blob/master/spec.ebnf). The file `spec_test.go` makes sure it is sane. # Some Bash comparisons | Bash | Nash | Description | | --- | --- | --- | | `GOPATH=/home/user/gopath` | `GOPATH="/home/user/gopath"` | Nash enforces quoted strings | | `GOPATH="$HOME/gopath"` | `GOPATH=$HOME+"/gopath"` | Nash doesn't do string expansion | | `export PATH=/usr/bin` | `PATH="/usr/bin"`
`setenv PATH` | setenv operates only on valid variables | | `export` | `showenv` | | | `ls -la` | `ls -la` | Simple commads are identical | | `ls -la "$GOPATH"` | `ls -la $GOPATH` | Nash variables shouldn't be enclosed in quotes, because it's default behaviour | | `./worker 2>log.err 1>log.out` | `./worker >[2] log.err >[1] log.out` | Nash redirection works like plan9 rc | | `./worker 2>&1` | `./worker >[2=1]` | Redirection map only works for standard file descriptors (0,1,2) | # Security The PID 1 of every namespace created by `nash` is the same nash binary reading commands from the parent shell via unix socket. It allows the parent namespace (the script that creates the namespace) to issue commands inside the child namespace. In the current implementation the unix socket communication is not secure yet. # Installing libraries Lets say you have a nash library and you want to install it. For example you have the following: ``` awesome/code.sh ``` And you want to install it so you can write code like this: ``` import awesome/code code_do_awesome_stuff() ``` All you have to do is run: ``` nash -install ./awesome ``` Or: ``` nash -install /absolute/path/awesome ``` The entire awesome dir (and its subdirs) will be copied where nash searches for libraries (dependent on environment variables). This is the recommended way of installing nash libraries (althought you can do it manually if you want). Single files can also be installed as packages, for example: ``` nash -install ./awesome/code.sh ``` Will enable you to import like this: ``` import code ``` If there is already a package with the given name it will be overwritten. # Releasing To generate a release basically: * Generate the release on github * Clone the generated tag * Run: ``` make release "version=" ``` Where **** must match the version of the git tag. # Want to contribute? Open issues and PR :) The project is in an early stage, be patient because things can change in the future. > "What I cannot create, I do not understand." > > -- Richard Feynman ================================================ FILE: _disabled_appveyor.yml ================================================ version: "1.0.0.{build}" platform: x64 clone_folder: "c:\\gopath\\src\\github.com\\madlambda\\nash" environment: GOPATH: "c:\\gopath" install: - "echo %PATH%" - "echo %GOPATH%" - "set PATH=%GOPATH%\\bin;c:\\go\\bin;c:\\MinGW\\bin;%PATH%" - "go version" - "go env" - copy c:\MinGW\bin\mingw32-make.exe c:\MinGW\bin\make.exe - choco install cygwin - set PATH=C:\\cygwin64\\bin;%PATH% build_script: - make build - make test notifications: - provider: GitHubPullRequest auth_token: secure: QuTLyXQp/4bQNeeEe5DLt9NIt/TzmZkn87s6wfOWpELX1L5UJyRCKV8AJitZWgwv template: "{{#passed}}:white_check_mark:{{/passed}}{{#failed}}:x:{{/failed}} [Build {{&projectName}} {{buildVersion}} {{status}}]({{buildUrl}}) (commit {{commitUrl}} by @{{&commitAuthorUsername}})" ================================================ FILE: ast/doc_test.go ================================================ package ast_test import ( "fmt" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/token" ) func Example_AssignmentNode() { one := ast.NewNameNode(token.NewFileInfo(1, 0), "one", nil) two := ast.NewNameNode(token.NewFileInfo(1, 4), "two", nil) value1 := ast.NewStringExpr(token.NewFileInfo(1, 8), "1", true) value2 := ast.NewStringExpr(token.NewFileInfo(1, 10), "2", true) assign := ast.NewAssignNode(token.NewFileInfo(1, 0), []*ast.NameNode{one, two}, []ast.Expr{value1, value2}, ) fmt.Printf("%s", assign) // Output: one, two = "1", "2" } func Example_AssignmentNode_Single() { operatingSystems := ast.NewNameNode(token.NewFileInfo(1, 0), "operatingSystems", nil) values := []ast.Expr{ ast.NewStringExpr(token.NewFileInfo(1, 19), "plan9 from bell labs", true), ast.NewStringExpr(token.NewFileInfo(2, 19), "unix", true), ast.NewStringExpr(token.NewFileInfo(3, 19), "linux", true), ast.NewStringExpr(token.NewFileInfo(4, 19), "oberon", true), ast.NewStringExpr(token.NewFileInfo(5, 19), "windows", true), } list := ast.NewListExpr(token.NewFileInfo(0, 18), values) assign := ast.NewSingleAssignNode(token.NewFileInfo(1, 0), operatingSystems, list, ) fmt.Printf("%s", assign) // Output: operatingSystems = ( // "plan9 from bell labs" // "unix" // "linux" // "oberon" // "windows" // ) } ================================================ FILE: ast/node.go ================================================ package ast import ( "errors" "fmt" "github.com/madlambda/nash/token" ) const ( // RedirMapNoValue indicates the pipe has not redirection RedirMapNoValue = -1 // RedirMapSupress indicates the rhs of map was suppressed RedirMapSupress = -2 RforkFlags = "umnips" ) type ( // Node represents nodes in the grammar Node interface { Type() NodeType IsEqual(Node) bool // Line of node in the file Line() int // Column of the node in the file Column() int // String representation of the node. // Note that it could not match the correspondent node in // the source code. String() string } assignable interface { names() []*NameNode setEqSpace(int) getEqSpace() int string() (string, bool) } egalitarian struct{} // Expr is the interface of expression nodes. Expr Node // NodeType is the types of grammar NodeType int // BlockNode is the block BlockNode struct { NodeType token.FileInfo egalitarian Nodes []Node } // An ImportNode represents the node for an "import" keyword. ImportNode struct { NodeType token.FileInfo egalitarian Path *StringExpr // Import path } // A SetenvNode represents the node for a "setenv" keyword. SetenvNode struct { NodeType token.FileInfo egalitarian Name string assign Node } NameNode struct { NodeType token.FileInfo egalitarian Ident string Index Expr } // AssignNode is a node for variable assignments AssignNode struct { NodeType token.FileInfo egalitarian Names []*NameNode Values []Expr eqSpace int } // ExecAssignNode represents the node for execution assignment. ExecAssignNode struct { NodeType token.FileInfo egalitarian Names []*NameNode cmd Node eqSpace int } // A CommandNode is a node for commands CommandNode struct { NodeType token.FileInfo egalitarian name string args []Expr redirs []*RedirectNode multi bool } // PipeNode represents the node for a command pipeline. PipeNode struct { NodeType token.FileInfo egalitarian cmds []*CommandNode multi bool } // StringExpr is a string argument StringExpr struct { NodeType token.FileInfo egalitarian str string quoted bool } // IntExpr is a integer used at indexing IntExpr struct { NodeType token.FileInfo egalitarian val int } // ListExpr is a list argument ListExpr struct { NodeType token.FileInfo egalitarian List []Expr IsVariadic bool } // ConcatExpr is a concatenation of arguments ConcatExpr struct { NodeType token.FileInfo egalitarian concat []Expr } // VarExpr is a variable argument VarExpr struct { NodeType token.FileInfo egalitarian Name string IsVariadic bool } // IndexExpr is a indexed variable IndexExpr struct { NodeType token.FileInfo egalitarian Var *VarExpr Index Expr IsVariadic bool } // RedirectNode represents the output redirection part of a command RedirectNode struct { NodeType token.FileInfo egalitarian rmap RedirMap location Expr } // RforkNode is a builtin node for rfork RforkNode struct { NodeType token.FileInfo egalitarian arg *StringExpr tree *Tree } // CommentNode is the node for comments CommentNode struct { NodeType token.FileInfo egalitarian val string } // RedirMap is the map of file descriptors of the redirection RedirMap struct { lfd int rfd int } // IfNode represents the node for the "if" keyword. IfNode struct { NodeType token.FileInfo egalitarian lvalue Expr rvalue Expr op string elseIf bool ifTree *Tree elseTree *Tree } // VarAssignDeclNode is a "var" declaration to assign values VarAssignDeclNode struct { NodeType token.FileInfo egalitarian Assign *AssignNode } // VarExecAssignDeclNode is a var declaration to assign output of fn/cmd VarExecAssignDeclNode struct { NodeType token.FileInfo egalitarian ExecAssign *ExecAssignNode } // FnArgNode represents function arguments FnArgNode struct { NodeType token.FileInfo egalitarian Name string IsVariadic bool } // A FnDeclNode represents a function declaration. FnDeclNode struct { NodeType token.FileInfo egalitarian name string args []*FnArgNode tree *Tree } // A FnInvNode represents a function invocation statement. FnInvNode struct { NodeType token.FileInfo egalitarian name string args []Expr } // A ReturnNode represents the "return" keyword. ReturnNode struct { NodeType token.FileInfo egalitarian Returns []Expr } // A BindFnNode represents the "bindfn" keyword. BindFnNode struct { NodeType token.FileInfo egalitarian name string cmdname string } // A ForNode represents the "for" keyword. ForNode struct { NodeType token.FileInfo egalitarian identifier string inExpr Expr tree *Tree } ) //go:generate stringer -type=NodeType const ( // NodeSetenv is the type for "setenv" builtin keyword NodeSetenv NodeType = iota + 1 // NodeBlock represents a program scope. NodeBlock // NodeName represents an identifier NodeName // NodeAssign is the type for variable assignment NodeAssign // NodeExecAssign is the type for command/function assignment NodeExecAssign // NodeImport is the type for "import" builtin keyword NodeImport execBegin // NodeCommand is the type for command execution NodeCommand // NodePipe is the type for pipeline execution NodePipe // NodeRedirect is the type for redirection nodes NodeRedirect // NodeFnInv is the type for function invocation NodeFnInv execEnd expressionBegin // NodeStringExpr is the type of string expression (quoted or not). NodeStringExpr // NodeIntExpr is the type of integer expression (commonly list indexing) NodeIntExpr // NodeVarExpr is the type of variable expressions. NodeVarExpr // NodeListExpr is the type of list expression. NodeListExpr // NodeIndexExpr is the type of indexing expressions. NodeIndexExpr // NodeConcatExpr is the type of concatenation expressions. NodeConcatExpr expressionEnd // NodeString are nodes for argument strings NodeString // NodeRfork is the type for rfork statement NodeRfork // NodeRforkFlags are nodes for rfork flags NodeRforkFlags // NodeIf is the type for if statements NodeIf // NodeComment are nodes for comment NodeComment NodeFnArg // NodeVarAssignDecl is the type for var declaration of values NodeVarAssignDecl // NodeVarExecAssignDecl NodeVarExecAssignDecl // NodeFnDecl is the type for function declaration NodeFnDecl // NodeReturn is the type for return statement NodeReturn // NodeBindFn is the type for bindfn statements NodeBindFn // NodeFor is the type for "for" statements NodeFor ) var ( DebugCmp bool ) func debug(format string, args ...interface{}) { if DebugCmp { fmt.Printf("[debug] "+format+"\n", args...) } } // Type returns the type of the node func (t NodeType) Type() NodeType { return t } // IsExpr returns if the node is an expression. func (t NodeType) IsExpr() bool { return t > expressionBegin && t < expressionEnd } // IsExecutable returns if the node is executable func (t NodeType) IsExecutable() bool { return t > execBegin && t < execEnd } func (e egalitarian) equal(node, other Node) bool { if node == other { return true } if node == nil { return false } if !cmpInfo(node, other) { return false } return true } // NewBlockNode creates a new block func NewBlockNode(info token.FileInfo) *BlockNode { return &BlockNode{ NodeType: NodeBlock, FileInfo: info, } } // Push adds a new node for a block of nodes func (l *BlockNode) Push(n Node) { l.Nodes = append(l.Nodes, n) } // IsEqual returns if it is equal to the other node. func (l *BlockNode) IsEqual(other Node) bool { if !l.equal(l, other) { return false } o, ok := other.(*BlockNode) if !ok { debug("Failed to cast other node to BlockNode") return false } if len(l.Nodes) != len(o.Nodes) { debug("Nodes differs in length") return false } for i := 0; i < len(l.Nodes); i++ { if !l.Nodes[i].IsEqual(o.Nodes[i]) { debug("List entry %d differ... '%s' != '%s'", i, l.Nodes[i], o.Nodes[i]) return false } } return true } // NewImportNode creates a new ImportNode object func NewImportNode(info token.FileInfo, path *StringExpr) *ImportNode { return &ImportNode{ NodeType: NodeImport, FileInfo: info, Path: path, } } // IsEqual returns if it is equal to the other node. func (n *ImportNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*ImportNode) if !ok { debug("Failed to cast to ImportNode") return false } if n.Path != o.Path { if n.Path != nil { return n.Path.IsEqual(o.Path) } } return false } // NewSetenvNode creates a new assignment node func NewSetenvNode(info token.FileInfo, name string, assign Node) (*SetenvNode, error) { if assign != nil && assign.Type() != NodeAssign && assign.Type() != NodeExecAssign { return nil, errors.New("Invalid assignment in setenv") } return &SetenvNode{ NodeType: NodeSetenv, FileInfo: info, Name: name, assign: assign, }, nil } // Assignment returns the setenv assignment (if any) func (n *SetenvNode) Assignment() Node { return n.assign } // IsEqual returns if it is equal to the other node. func (n *SetenvNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*SetenvNode) if !ok { debug("Failed to convert to SetenvNode") return false } if n.assign != o.assign { if !n.assign.IsEqual(o.assign) { return false } } return n.Name == o.Name } func NewNameNode(info token.FileInfo, ident string, index Expr) *NameNode { return &NameNode{ NodeType: NodeName, FileInfo: info, Ident: ident, Index: index, } } func (n *NameNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*NameNode) if !ok { debug("Failed to convert to NameNode") return false } if n.Ident != o.Ident { return false } if n.Index == o.Index { return true } if n.Index != nil { return n.Index.IsEqual(o.Index) } return false } // NewAssignNode creates a new tuple assignment (multiple variable // assigned in a single statement). // For single assignment see NewSingleAssignNode. func NewAssignNode(info token.FileInfo, names []*NameNode, values []Expr) *AssignNode { return &AssignNode{ NodeType: NodeAssign, FileInfo: info, eqSpace: -1, Names: names, Values: values, } } // NewSingleAssignNode creates an assignment of a single variable. Eg.: // name = "hello" // To make an assignment of multiple variables in the same statement // use `NewAssignNode`. func NewSingleAssignNode(info token.FileInfo, name *NameNode, value Expr) *AssignNode { return NewAssignNode(info, []*NameNode{name}, []Expr{value}) } // TODO(i4k): fix that func (n *AssignNode) names() []*NameNode { return n.Names } func (n *AssignNode) getEqSpace() int { return n.eqSpace } func (n *AssignNode) setEqSpace(value int) { n.eqSpace = value } // IsEqual returns if it is equal to the other node. func (n *AssignNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*AssignNode) if !ok { debug("Failed to convert to AssignNode") return false } if len(n.Names) == len(o.Names) { for i := 0; i < len(n.Names); i++ { if !n.Names[i].IsEqual(o.Names[i]) { debug("Assignment identifier doesn't match: '%s' != '%s'", n.Names[i], o.Names[i]) return false } } } else { return false } if len(n.Values) == len(o.Values) { for i := 0; i < len(n.Values); i++ { if !n.Values[i].IsEqual(o.Values[i]) { return false } } } else { return false } return true } // NewExecAssignNode creates a new node for executing something and store the // result on a new variable. The assignment could be made using an operating system // command, a pipe of commands or a function invocation. // It returns a *ExecAssignNode ready to be executed or error when n is not a valid // node for execution. // TODO(i4k): Change the API to specific node types. Eg.: NewExecAssignCmdNode and // so on. func NewExecAssignNode(info token.FileInfo, names []*NameNode, n Node) (*ExecAssignNode, error) { if !n.Type().IsExecutable() { return nil, errors.New("NewExecAssignNode expects a CommandNode, PipeNode or FninvNode") } return &ExecAssignNode{ NodeType: NodeExecAssign, FileInfo: info, Names: names, cmd: n, eqSpace: -1, }, nil } func (n *ExecAssignNode) names() []*NameNode { return n.Names } func (n *ExecAssignNode) getEqSpace() int { return n.eqSpace } func (n *ExecAssignNode) setEqSpace(value int) { n.eqSpace = value } // Command returns the command (or r-value). Command could be a CommandNode or FnNode func (n *ExecAssignNode) Command() Node { return n.cmd } // SetCommand set the command part (NodeCommand or NodeFnDecl) func (n *ExecAssignNode) SetCommand(c Node) { n.cmd = c } func (n *ExecAssignNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*ExecAssignNode) if !ok { debug("Failed to convert to ExecAssignNode") return false } if len(n.Names) != len(o.Names) { return false } for i := 0; i < len(n.Names); i++ { if n.Names[i] != nil { if !n.Names[i].IsEqual(o.Names[i]) { debug("Exec assignment name differs") return false } } } if n.cmd == o.cmd { return true } else if n.cmd != nil { return n.cmd.IsEqual(o.cmd) } return false } // NewCommandNode creates a new node for commands func NewCommandNode(info token.FileInfo, name string, multiline bool) *CommandNode { return &CommandNode{ NodeType: NodeCommand, FileInfo: info, name: name, multi: multiline, } } func (n *CommandNode) IsMulti() bool { return n.multi } func (n *CommandNode) SetMulti(b bool) { n.multi = b } // AddArg adds a new argument to the command func (n *CommandNode) AddArg(a Expr) { n.args = append(n.args, a) } // SetArgs sets an array of args to command func (n *CommandNode) SetArgs(args []Expr) { n.args = args } // Args returns the list of arguments supplied to command. func (n *CommandNode) Args() []Expr { return n.args } // AddRedirect adds a new redirect node to command func (n *CommandNode) AddRedirect(redir *RedirectNode) { n.redirs = append(n.redirs, redir) } // Redirects return the list of redirect maps of the command. func (n *CommandNode) Redirects() []*RedirectNode { return n.redirs } // Name returns the program name func (n *CommandNode) Name() string { return n.name } // IsEqual returns if it is equal to the other node. func (n *CommandNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*CommandNode) if !ok { debug("Failed to convert to CommandNode") return false } if n.multi != o.multi { debug("Command multiline differs.") return false } if len(n.args) != len(o.args) { debug("Command argument length differs: %d (%+v) != %d (%+v)", len(n.args), n.args, len(o.args), o.args) return false } for i := 0; i < len(n.args); i++ { if !n.args[i].IsEqual(o.args[i]) { debug("Argument %d differs. '%s' != '%s'", i, n.args[i], o.args[i]) return false } } if len(n.redirs) != len(o.redirs) { debug("Number of redirects differs. %d != %d", len(n.redirs), len(o.redirs)) return false } for i := 0; i < len(n.redirs); i++ { if n.redirs[i] == o.redirs[i] { continue } else if n.redirs[i] != nil && !n.redirs[i].IsEqual(o.redirs[i]) { debug("Redirect differs... %s != %s", n.redirs[i], o.redirs[i]) return false } } return n.name == o.name } // NewPipeNode creates a new command pipeline func NewPipeNode(info token.FileInfo, multi bool) *PipeNode { return &PipeNode{ NodeType: NodePipe, FileInfo: info, multi: multi, } } func (n *PipeNode) IsMulti() bool { return n.multi } func (n *PipeNode) SetMulti(b bool) { n.multi = b } // AddCmd add another command to end of the pipeline func (n *PipeNode) AddCmd(c *CommandNode) { n.cmds = append(n.cmds, c) } // Commands returns the list of pipeline commands func (n *PipeNode) Commands() []*CommandNode { return n.cmds } // IsEqual returns if it is equal to the other node. func (n *PipeNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*PipeNode) if !ok { debug("Failed to convert to PipeNode") return false } if len(n.cmds) != len(o.cmds) { debug("Number of pipe commands differ: %d != %d", len(n.cmds), len(o.cmds)) return false } for i := 0; i < len(n.cmds); i++ { if !n.cmds[i].IsEqual(o.cmds[i]) { debug("Command differs. '%s' != '%s'", n.cmds[i], o.cmds[i]) return false } } return true } // NewRedirectNode creates a new redirection node for commands func NewRedirectNode(info token.FileInfo) *RedirectNode { return &RedirectNode{ NodeType: NodeRedirect, FileInfo: info, rmap: RedirMap{ lfd: -1, rfd: -1, }, } } // SetMap sets the redirection map. Eg.: [2=1] func (r *RedirectNode) SetMap(lfd int, rfd int) { r.rmap.lfd = lfd r.rmap.rfd = rfd } // LeftFD return the lhs of the redirection map. func (r *RedirectNode) LeftFD() int { return r.rmap.lfd } // RightFD return the rhs of the redirection map. func (r *RedirectNode) RightFD() int { return r.rmap.rfd } // SetLocation of the output func (r *RedirectNode) SetLocation(s Expr) { r.location = s } // Location return the location of the redirection. func (r *RedirectNode) Location() Expr { return r.location } // IsEqual return if it is equal to the other node. func (r *RedirectNode) IsEqual(other Node) bool { if !r.equal(r, other) { return false } o, ok := other.(*RedirectNode) if !ok { return false } if r.rmap.lfd != o.rmap.lfd || r.rmap.rfd != o.rmap.rfd { return false } if r.location == o.location { return true } else if r.location != nil { return r.location.IsEqual(o.location) } return false } // NewRforkNode creates a new node for rfork func NewRforkNode(info token.FileInfo) *RforkNode { return &RforkNode{ NodeType: NodeRfork, FileInfo: info, } } // Arg return the string argument of the rfork. func (n *RforkNode) Arg() *StringExpr { return n.arg } // SetFlags sets the rfork flags func (n *RforkNode) SetFlags(a *StringExpr) { n.arg = a } // Tree returns the child tree of node func (n *RforkNode) Tree() *Tree { return n.tree } // SetTree set the body of the rfork block. func (n *RforkNode) SetTree(t *Tree) { n.tree = t } func (n *RforkNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*RforkNode) if !ok { return false } if n.arg == o.arg { return true } if n.arg != nil { if !n.arg.IsEqual(o.arg) { return false } } return n.tree.IsEqual(o.tree) } // NewCommentNode creates a new node for comments func NewCommentNode(info token.FileInfo, val string) *CommentNode { return &CommentNode{ NodeType: NodeComment, FileInfo: info, val: val, } } func (n *CommentNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } if n.Type() != other.Type() { return false } o, ok := other.(*CommentNode) if !ok { return false } return n.val == o.val } // NewIfNode creates a new if block statement func NewIfNode(info token.FileInfo) *IfNode { return &IfNode{ NodeType: NodeIf, FileInfo: info, } } // Lvalue returns the lefthand part of condition func (n *IfNode) Lvalue() Expr { return n.lvalue } // Rvalue returns the righthand side of condition func (n *IfNode) Rvalue() Expr { return n.rvalue } // SetLvalue set the lefthand side of condition func (n *IfNode) SetLvalue(arg Expr) { n.lvalue = arg } // SetRvalue set the righthand side of condition func (n *IfNode) SetRvalue(arg Expr) { n.rvalue = arg } // Op returns the condition operation func (n *IfNode) Op() string { return n.op } // SetOp set the condition operation func (n *IfNode) SetOp(op string) { n.op = op } // IsElseIf tells if the if is an else-if statement func (n *IfNode) IsElseIf() bool { return n.elseIf } // SetElseif sets the else-if part func (n *IfNode) SetElseif(b bool) { n.elseIf = b } // SetIfTree sets the block of statements of the if block func (n *IfNode) SetIfTree(t *Tree) { n.ifTree = t } // SetElseTree sets the block of statements of the else block func (n *IfNode) SetElseTree(t *Tree) { n.elseTree = t } // IfTree returns the if block func (n *IfNode) IfTree() *Tree { return n.ifTree } // ElseTree returns the else block func (n *IfNode) ElseTree() *Tree { return n.elseTree } // IsEqual returns if it is equal to the other node. func (n *IfNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*IfNode) if !ok { debug("Failed to convert to ifNode") return false } elvalue := n.Lvalue() ervalue := n.Rvalue() vlvalue := o.Lvalue() vrvalue := o.Rvalue() if !elvalue.IsEqual(vlvalue) { debug("Lvalue differs: '%s' != '%s'", elvalue, vlvalue) return false } if !ervalue.IsEqual(vrvalue) { debug("Rvalue differs: '%s' != '%s'", ervalue, vrvalue) return false } if n.Op() != o.Op() { debug("Operation differs: %s != %s", n.Op(), o.Op()) return false } expectedTree := n.IfTree() valueTree := o.IfTree() if !expectedTree.IsEqual(valueTree) { debug("If tree differs: '%s' != '%s'", expectedTree, valueTree) return false } expectedTree = n.ElseTree() valueTree = o.ElseTree() return expectedTree.IsEqual(valueTree) } func NewFnArgNode(info token.FileInfo, name string, isVariadic bool) *FnArgNode { return &FnArgNode{ NodeType: NodeFnArg, FileInfo: info, Name: name, IsVariadic: isVariadic, } } func (a *FnArgNode) IsEqual(other Node) bool { if !a.equal(a, other) { return false } o, ok := other.(*FnArgNode) if !ok { return false } if a.Name != o.Name || a.IsVariadic != o.IsVariadic { return false } return true } func NewVarAssignDecl(info token.FileInfo, assignNode *AssignNode) *VarAssignDeclNode { return &VarAssignDeclNode{ NodeType: NodeVarAssignDecl, Assign: assignNode, } } func (n *VarAssignDeclNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*VarAssignDeclNode) if !ok { return false } return n.Assign.IsEqual(o.Assign) } func NewVarExecAssignDecl(info token.FileInfo, assignNode *ExecAssignNode) *VarExecAssignDeclNode { return &VarExecAssignDeclNode{ NodeType: NodeVarExecAssignDecl, ExecAssign: assignNode, } } func (n *VarExecAssignDeclNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*VarExecAssignDeclNode) if !ok { return false } return n.ExecAssign.IsEqual(o.ExecAssign) } // NewFnDeclNode creates a new function declaration func NewFnDeclNode(info token.FileInfo, name string) *FnDeclNode { return &FnDeclNode{ NodeType: NodeFnDecl, FileInfo: info, name: name, } } // SetName set the function name func (n *FnDeclNode) SetName(a string) { n.name = a } // Name return the function name func (n *FnDeclNode) Name() string { return n.name } // Args returns function arguments func (n *FnDeclNode) Args() []*FnArgNode { return n.args } // AddArg add a new argument to end of argument list func (n *FnDeclNode) AddArg(arg *FnArgNode) { n.args = append(n.args, arg) } // Tree return the function block func (n *FnDeclNode) Tree() *Tree { return n.tree } // SetTree set the function tree func (n *FnDeclNode) SetTree(t *Tree) { n.tree = t } func (n *FnDeclNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*FnDeclNode) if !ok { return false } if n.name != o.name || len(n.args) != len(o.args) { return false } for i := 0; i < len(n.args); i++ { if !n.args[i].IsEqual(o.args[i]) { return false } } return true } // NewFnInvNode creates a new function invocation func NewFnInvNode(info token.FileInfo, name string) *FnInvNode { return &FnInvNode{ NodeType: NodeFnInv, FileInfo: info, name: name, } } // SetName set the function name func (n *FnInvNode) SetName(a string) { n.name = a } // Name return the function name func (n *FnInvNode) Name() string { return n.name } // AddArg add another argument to end of argument list func (n *FnInvNode) AddArg(arg Expr) { n.args = append(n.args, arg) } // Args return the invocation arguments. func (n *FnInvNode) Args() []Expr { return n.args } // IsEqual returns if it is equal to the other node. func (n *FnInvNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*FnInvNode) if !ok { return false } if len(n.args) != len(o.args) { return false } for i := 0; i < len(n.args); i++ { if !n.args[i].IsEqual(o.args[i]) { return false } } return true } // NewBindFnNode creates a new bindfn statement func NewBindFnNode(info token.FileInfo, name, cmd string) *BindFnNode { return &BindFnNode{ NodeType: NodeBindFn, FileInfo: info, name: name, cmdname: cmd, } } // Name return the function name func (n *BindFnNode) Name() string { return n.name } // CmdName return the command name func (n *BindFnNode) CmdName() string { return n.cmdname } func (n *BindFnNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } o, ok := other.(*BindFnNode) if !ok { return false } return n.name == o.name && n.cmdname == o.cmdname } // NewReturnNode create a return statement func NewReturnNode(info token.FileInfo) *ReturnNode { return &ReturnNode{ FileInfo: info, NodeType: NodeReturn, } } func (n *ReturnNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } if n.Type() != other.Type() { return false } o, ok := other.(*ReturnNode) if !ok { return false } if len(n.Returns) != len(o.Returns) { return false } for i := 0; i < len(n.Returns); i++ { arg := n.Returns[i] oarg := o.Returns[i] if arg != nil && !arg.IsEqual(oarg) { return false } } return true } // NewForNode create a new for statement func NewForNode(info token.FileInfo) *ForNode { return &ForNode{ NodeType: NodeFor, FileInfo: info, } } // SetIdentifier set the for indentifier func (n *ForNode) SetIdentifier(a string) { n.identifier = a } // Identifier return the identifier part func (n *ForNode) Identifier() string { return n.identifier } // InVar return the "in" variable func (n *ForNode) InExpr() Expr { return n.inExpr } // SetInVar set "in" expression func (n *ForNode) SetInExpr(a Expr) { n.inExpr = a } // SetTree set the for block of statements func (n *ForNode) SetTree(a *Tree) { n.tree = a } // Tree return the for block func (n *ForNode) Tree() *Tree { return n.tree } func (n *ForNode) IsEqual(other Node) bool { if !n.equal(n, other) { return false } if n.Type() != other.Type() { return false } o, ok := other.(*ForNode) if !ok { return false } if n.identifier != o.identifier { return false } if n.inExpr == o.inExpr { return true } if n.inExpr != nil { return n.inExpr.IsEqual(o.inExpr) } return false } func cmpInfo(n, other Node) bool { if n.Line() != other.Line() || n.Column() != other.Column() { debug("file info mismatch on %v (%s): (%d, %d) != (%d, %d)", n, n.Type(), n.Line(), n.Column(), other.Line(), other.Column()) return false } return true } ================================================ FILE: ast/node_args.go ================================================ package ast import ( "fmt" "github.com/madlambda/nash/scanner" "github.com/madlambda/nash/token" ) // ArgFromToken is a helper to get an argument based on the lexer token func ExprFromToken(val scanner.Token) (Expr, error) { switch val.Type() { case token.Arg: return NewStringExpr(token.NewFileInfo(val.Line(), val.Column()), val.Value(), false), nil case token.String: return NewStringExpr(token.NewFileInfo(val.Line(), val.Column()), val.Value(), true), nil case token.Variable: return NewVarExpr(token.NewFileInfo(val.Line(), val.Column()), val.Value()), nil } return nil, fmt.Errorf("argFromToken doesn't support type %v", val) } // NewArgString creates a new string argument func NewStringExpr(info token.FileInfo, value string, quoted bool) *StringExpr { return &StringExpr{ NodeType: NodeStringExpr, FileInfo: info, str: value, quoted: quoted, } } // Value returns the argument string value func (s *StringExpr) Value() string { return s.str } func (s *StringExpr) SetValue(a string) { s.str = a } func (s *StringExpr) IsEqual(other Node) bool { if !s.equal(s, other) { return false } value, ok := other.(*StringExpr) if !ok { return false } if s.quoted != value.quoted { return false } return s.str == value.str } func NewIntExpr(info token.FileInfo, val int) *IntExpr { return &IntExpr{ NodeType: NodeIntExpr, FileInfo: info, val: val, } } func (i *IntExpr) Value() int { return i.val } func (i *IntExpr) IsEqual(other Node) bool { if !i.equal(i, other) { return false } o, ok := other.(*IntExpr) if !ok { return false } return i.val == o.val } func NewListExpr(info token.FileInfo, values []Expr) *ListExpr { return NewListVariadicExpr(info, values, false) } func NewListVariadicExpr(info token.FileInfo, values []Expr, variadic bool) *ListExpr { return &ListExpr{ NodeType: NodeListExpr, FileInfo: info, List: values, IsVariadic: variadic, } } // PushExpr push an expression to end of the list func (l *ListExpr) PushExpr(a Expr) { l.List = append(l.List, a) } func (l *ListExpr) IsEqual(other Node) bool { if !l.equal(l, other) { return false } o, ok := other.(*ListExpr) if !ok { return false } if len(l.List) != len(o.List) { return false } for i, val := range l.List { oval := o.List[i] if !val.IsEqual(oval) { debug("%v(%s) != %v(%s)", val, val.Type(), oval, oval.Type()) return false } } return true } func NewConcatExpr(info token.FileInfo, parts []Expr) *ConcatExpr { return &ConcatExpr{ NodeType: NodeConcatExpr, FileInfo: info, concat: parts, } } // PushExpr push an expression to end of the concat list func (c *ConcatExpr) PushExpr(a Expr) { c.concat = append(c.concat, a) } // SetConcatList set the concatenation parts func (c *ConcatExpr) SetConcat(v []Expr) { c.concat = v } func (c *ConcatExpr) List() []Expr { return c.concat } func (c *ConcatExpr) IsEqual(other Node) bool { if !c.equal(c, other) { return false } o, ok := other.(*ConcatExpr) if !ok { return false } if len(c.concat) != len(o.concat) { return false } for i := 0; i < len(c.concat); i++ { if !c.concat[i].IsEqual(o.concat[i]) { return false } } return true } func NewVarExpr(info token.FileInfo, name string) *VarExpr { return NewVarVariadicExpr(info, name, false) } func NewVarVariadicExpr(info token.FileInfo, name string, isVariadic bool) *VarExpr { return &VarExpr{ NodeType: NodeVarExpr, FileInfo: info, Name: name, IsVariadic: isVariadic, } } func (v *VarExpr) IsEqual(other Node) bool { if !v.equal(v, other) { return false } o, ok := other.(*VarExpr) if !ok { return false } return v.Name == o.Name && v.IsVariadic == o.IsVariadic } func NewIndexExpr(info token.FileInfo, va *VarExpr, idx Expr) *IndexExpr { return NewIndexVariadicExpr(info, va, idx, false) } func NewIndexVariadicExpr(info token.FileInfo, va *VarExpr, idx Expr, variadic bool) *IndexExpr { return &IndexExpr{ NodeType: NodeIndexExpr, FileInfo: info, Var: va, Index: idx, IsVariadic: variadic, } } func (i *IndexExpr) IsEqual(other Node) bool { if !i.equal(i, other) { return false } o, ok := other.(*IndexExpr) if !ok { return false } return i.Var.IsEqual(o.Var) && i.Index.IsEqual(o.Index) && i.IsVariadic == o.IsVariadic } ================================================ FILE: ast/node_fmt.go ================================================ package ast import ( "fmt" "strconv" "strings" ) func (s *StringExpr) String() string { if s.quoted { return `"` + stringify(s.str) + `"` } return s.str } func (i *IntExpr) String() string { return strconv.Itoa(i.val) } func (l *ListExpr) string() (string, bool) { elems := make([]string, len(l.List)) columnCount := 0 forceMulti := false for i := 0; i < len(l.List); i++ { if l.List[i].Type() == NodeListExpr { forceMulti = true } elems[i] = l.List[i].String() columnCount += len(elems[i]) } if columnCount+len(elems) > 50 || forceMulti { forceMulti = true return "(\n\t" + strings.Join(elems, "\n\t") + "\n)", forceMulti } return "(" + strings.Join(elems, " ") + ")", false } func (l *ListExpr) String() string { str, _ := l.string() return str } func (c *ConcatExpr) String() string { ret := "" for i := 0; i < len(c.concat); i++ { ret += c.concat[i].String() if i < (len(c.concat) - 1) { ret += "+" } } return ret } func (v *VarExpr) String() string { if v.IsVariadic { return v.Name + "..." } return v.Name } func (i *IndexExpr) String() string { ret := fmt.Sprintf("%s[%s]", i.Var, i.Index) if i.IsVariadic { return ret + "..." } return ret } func (l *BlockNode) adjustGroupAssign(node assignable, nodes []Node) { var ( eqSpace int = node.getEqSpace() i int ) lhs := getlhs(node) eqSpace = len(lhs) + 1 for i = 0; i < len(nodes); i++ { assign, ok := nodes[i].(assignable) if !ok { break } if len(getlhs(assign))+1 > eqSpace { eqSpace = len(getlhs(assign)) + 1 } } for j := 0; j < i; j++ { knode := nodes[j].(assignable) knode.setEqSpace(eqSpace) } node.setEqSpace(eqSpace) } func (l *BlockNode) String() string { nodes := l.Nodes content := make([]string, 0, 8192) last := (len(nodes) - 1) for i := 0; i < len(nodes); i++ { addEOL := false node := nodes[i] nodebytes := node.String() if i == 0 && node.Type() == NodeComment && strings.HasPrefix(node.String(), "#!") { addEOL = true } else if (node.Type() == NodeComment) && i < last { nextNode := nodes[i+1] if nextNode.Line() > node.Line()+1 { addEOL = true } } else if i < last { nextNode := nodes[i+1] if node.Type() != nextNode.Type() { addEOL = true } else if node.Type() == NodeFnDecl { addEOL = true } else if node.Type() == NodeAssign || node.Type() == NodeExecAssign { nodeAssign := node.(assignable) if nodeAssign.getEqSpace() == -1 { // lookahead to decide about best '=' distance l.adjustGroupAssign(nodeAssign, nodes[i+1:]) } nodebytes, addEOL = nodeAssign.string() } } if addEOL { nodebytes += "\n" } content = append(content, nodebytes) } return strings.Join(content, "\n") } // String returns the string representation of the import func (n *ImportNode) String() string { return `import ` + n.Path.String() } // String returns the string representation of assignment func (n *SetenvNode) String() string { if n.assign == nil { return "setenv " + n.Name } return "setenv " + n.assign.String() } func (n *NameNode) String() string { if n.Index != nil { return n.Ident + "[" + n.Index.String() + "]" } return n.Ident } func (n *AssignNode) string() (string, bool) { var ( multi bool ) objs := n.Values lhs := getlhs(n) ret := "" for i := 0; i < len(objs); i++ { var ( objStr string objmulti bool ) obj := objs[i] if obj.Type().IsExpr() { if obj.Type() == NodeListExpr { lobj := obj.(*ListExpr) objStr, objmulti = lobj.string() } else { objStr = obj.String() } } if i == 0 { if n.eqSpace > len(lhs) && !multi { ret = lhs + strings.Repeat(" ", n.eqSpace-len(lhs)) + "= " + objStr } else { ret = lhs + " = " + objStr } } else if i < len(objs)-1 { ret = ret + ", " + objStr + ", " } else { ret = ret + ", " + objStr } if objmulti && !multi { multi = true } } return ret, multi } // String returns the string representation of assignment statement func (n *AssignNode) String() string { str, _ := n.string() return str } func (n *ExecAssignNode) string() (string, bool) { var ( cmdStr string multi bool ) lhs := getlhs(n) if n.cmd.Type() == NodeCommand { cmd := n.cmd.(*CommandNode) cmdStr, multi = cmd.string() } else if n.cmd.Type() == NodePipe { cmd := n.cmd.(*PipeNode) cmdStr, multi = cmd.string() } else { cmd := n.cmd.(*FnInvNode) cmdStr, multi = cmd.string() } if n.eqSpace > len(lhs) { ret := lhs + strings.Repeat(" ", n.eqSpace-len(lhs)) + "<= " + cmdStr return ret, multi } return lhs + " <= " + cmdStr, multi } // String returns the string representation of command assignment statement func (n *ExecAssignNode) String() string { str, _ := n.string() return str } func (n *CommandNode) toStringParts() ([]string, int) { var ( content []string line string last = len(n.args) - 1 totalLen = 0 ) for i := 0; i < len(n.args); i += 2 { var next string arg := n.args[i].String() if i < last { next = n.args[i+1].String() } if i == 0 { arg = n.name + " " + arg } if arg[0] == '-' { if line != "" { content = append(content, line) line = "" } if len(next) > 0 && next[0] != '-' { if line == "" { line += arg + " " + next } else { line += " " + arg + " " + next } } else { content = append(content, arg, next) } } else if next != "" { if line == "" { line += arg + " " + next } else { line += " " + arg + " " + next } } else { if line == "" { line += arg } else { line += " " + arg } } totalLen += len(arg) + len(next) + 1 } if line != "" { content = append(content, line) } if len(content) == 0 { content = append(content, n.name) } for i := 0; i < len(n.redirs); i++ { rstr := n.redirs[i].String() totalLen += len(rstr) + 1 content = append(content, rstr) } return content, totalLen } func (n *CommandNode) multiString() string { content, totalLen := n.toStringParts() if totalLen < 50 { return "(" + strings.Join(content, " ") + ")" } content[0] = "\t" + content[0] gentab := func(n int) string { return strings.Repeat("\t", n) } tabLen := (len(content[0]) + 7) / 8 for i := 1; i < len(content); i++ { content[i] = gentab(tabLen) + content[i] } return "(\n" + strings.Join(content, "\n") + "\n)" } // String returns the string representation of command statement func (n *CommandNode) string() (string, bool) { if n.multi { return n.multiString(), true } var content []string content = append(content, n.name) for i := 0; i < len(n.args); i++ { content = append(content, n.args[i].String()) } for i := 0; i < len(n.redirs); i++ { content = append(content, n.redirs[i].String()) } return strings.Join(content, " "), false } func (n *CommandNode) String() string { str, _ := n.string() return str } func (n *PipeNode) multiString() string { totalLen := 0 type cmdData struct { content []string totalLen int } content := make([]cmdData, len(n.cmds)) for i := 0; i < len(n.cmds); i++ { cmdContent, cmdLen := n.cmds[i].toStringParts() content[i] = cmdData{ cmdContent, cmdLen, } totalLen += cmdLen } if totalLen+3 < 50 { result := "(" for i := 0; i < len(content); i++ { result += strings.Join(content[i].content, " ") if i < len(content)-1 { result += " | " } } return result + ")" } gentab := func(n int) string { return strings.Repeat("\t", n) } result := "(\n" for i := 0; i < len(content); i++ { cmdContent := content[i].content cmdContent[0] = "\t" + cmdContent[0] tabLen := (len(cmdContent[0]) + 7) / 8 for j := 1; j < len(cmdContent); j++ { cmdContent[j] = gentab(tabLen) + cmdContent[j] } result += strings.Join(cmdContent, "\n") if i < len(content)-1 { result += " |\n" } } return result + "\n)" } // String returns the string representation of pipeline statement func (n *PipeNode) string() (string, bool) { if n.multi { return n.multiString(), true } ret := "" for i := 0; i < len(n.cmds); i++ { ret += n.cmds[i].String() if i < (len(n.cmds) - 1) { ret += " | " } } return ret, false } func (n *PipeNode) String() string { str, _ := n.string() return str } // String returns the string representation of redirect func (r *RedirectNode) String() string { var result string if r.rmap.lfd == r.rmap.rfd { if r.location != nil { return "> " + r.location.String() } return "" } if r.rmap.rfd >= 0 { result = ">[" + strconv.Itoa(r.rmap.lfd) + "=" + strconv.Itoa(r.rmap.rfd) + "]" } else if r.rmap.rfd == RedirMapNoValue { result = ">[" + strconv.Itoa(r.rmap.lfd) + "]" } else if r.rmap.rfd == RedirMapSupress { result = ">[" + strconv.Itoa(r.rmap.lfd) + "=]" } if r.location != nil { result = result + " " + r.location.String() } return result } // String returns the string representation of rfork statement func (n *RforkNode) String() string { rforkstr := "rfork " + n.arg.String() tree := n.Tree() if tree != nil { rforkstr += " {\n" block := tree.String() stmts := strings.Split(block, "\n") for i := 0; i < len(stmts); i++ { stmts[i] = "\t" + stmts[i] } rforkstr += strings.Join(stmts, "\n") + "\n}" } return rforkstr } // String returns the string representation of comment func (n *CommentNode) String() string { return n.val } // String returns the string representation of if statement func (n *IfNode) String() string { var lstr, rstr string lstr = n.lvalue.String() rstr = n.rvalue.String() ifStr := "if " + lstr + " " + n.op + " " + rstr + " {\n" ifTree := n.IfTree() block := ifTree.String() stmts := strings.Split(block, "\n") if strings.TrimSpace(block) != "" { for i := 0; i < len(stmts); i++ { stmts[i] = "\t" + stmts[i] } } ifStr += strings.Join(stmts, "\n") + "\n}" elseTree := n.ElseTree() if elseTree != nil { ifStr += " else " elseBlock := elseTree.String() elsestmts := strings.Split(elseBlock, "\n") for i := 0; i < len(elsestmts); i++ { if !n.IsElseIf() { elsestmts[i] = "\t" + elsestmts[i] } } if !n.IsElseIf() { ifStr += "{\n" } ifStr += strings.Join(elsestmts, "\n") if !n.IsElseIf() { ifStr += "\n}" } } return ifStr } func (n *VarAssignDeclNode) String() string { return "var " + n.Assign.String() } func (n *VarExecAssignDeclNode) String() string { return "var " + n.ExecAssign.String() } // String returns the string representation of function declaration func (n *FnDeclNode) String() string { fnStr := "fn" if n.name != "" { fnStr += " " + n.name + "(" } for i := 0; i < len(n.args); i++ { fnStr += n.args[i].String() if i < (len(n.args) - 1) { fnStr += ", " } } fnStr += ") {\n" tree := n.Tree() stmts := strings.Split(tree.String(), "\n") for i := 0; i < len(stmts); i++ { if len(stmts[i]) > 0 { fnStr += "\t" + stmts[i] + "\n" } else { fnStr += "\n" } } fnStr += "}" return fnStr } func (arg *FnArgNode) String() string { ret := arg.Name if arg.IsVariadic { ret += "..." } return ret } // String returns the string representation of function invocation func (n *FnInvNode) string() (string, bool) { fnInvStr := n.name + "(" for i := 0; i < len(n.args); i++ { fnInvStr += n.args[i].String() if i < (len(n.args) - 1) { fnInvStr += ", " } } fnInvStr += ")" return fnInvStr, false } func (n *FnInvNode) String() string { str, _ := n.string() return str } // String returns the string representation of bindfn func (n *BindFnNode) String() string { return "bindfn " + n.name + " " + n.cmdname } // String returns the string representation of return statement func (n *ReturnNode) String() string { var returns []string ret := "return" returnExprs := n.Returns for i := 0; i < len(returnExprs); i++ { returns = append(returns, returnExprs[i].String()) } if len(returns) > 0 { return ret + " " + strings.Join(returns, ", ") } return ret } // String returns the string representation of for statement func (n *ForNode) String() string { ret := "for" if n.identifier != "" { ret += " " + n.identifier + " in " + n.inExpr.String() } ret += " {\n" tree := n.Tree() stmts := strings.Split(tree.String(), "\n") for i := 0; i < len(stmts); i++ { if len(stmts[i]) > 0 { ret += "\t" + stmts[i] + "\n" } else { ret += "\n" } } ret += "}" return ret } func stringify(s string) string { buf := make([]byte, 0, len(s)) for i := 0; i < len(s); i++ { switch s[i] { case '"': buf = append(buf, '\\', '"') case '\t': buf = append(buf, '\\', 't') case '\n': buf = append(buf, '\\', 'n') case '\r': buf = append(buf, '\\', 'r') case '\\': buf = append(buf, '\\', '\\') default: buf = append(buf, s[i]) } } return string(buf) } func getlhs(node assignable) string { var nameStrs []string nodeNames := node.names() for i := 0; i < len(nodeNames); i++ { nameStrs = append(nameStrs, nodeNames[i].String()) } return strings.Join(nameStrs, ", ") } ================================================ FILE: ast/node_fmt_test.go ================================================ package ast import ( "testing" "github.com/madlambda/nash/token" ) func testPrinter(t *testing.T, node Node, expected string) { if node.String() != expected { t.Errorf("Values differ: '%s' != '%s'", node, expected) } } func TestAstPrinterStringExpr(t *testing.T) { for _, testcase := range []struct { expected string node Node }{ // quote { expected: `"\""`, node: NewStringExpr(token.NewFileInfo(1, 0), "\"", true), }, // escape { expected: `"\\this is a test\n"`, node: NewStringExpr(token.NewFileInfo(1, 0), "\\this is a test\n", true), }, // tab { expected: `"this is a test\t"`, node: NewStringExpr(token.NewFileInfo(1, 0), "this is a test\t", true), }, // linefeed { expected: `"this is a test\n"`, node: NewStringExpr(token.NewFileInfo(1, 0), "this is a test\n", true), }, { expected: `"\nthis is a test"`, node: NewStringExpr(token.NewFileInfo(1, 0), "\nthis is a test", true), }, { expected: `"\n\n\n"`, node: NewStringExpr(token.NewFileInfo(1, 0), "\n\n\n", true), }, // carriege return { expected: `"this is a test\r"`, node: NewStringExpr(token.NewFileInfo(1, 0), "this is a test\r", true), }, { expected: `"\rthis is a test"`, node: NewStringExpr(token.NewFileInfo(1, 0), "\rthis is a test", true), }, { expected: `"\r\r\r"`, node: NewStringExpr(token.NewFileInfo(1, 0), "\r\r\r", true), }, } { testPrinter(t, testcase.node, testcase.expected) } } func TestASTPrinterAssignment(t *testing.T) { zeroFileInfo := token.NewFileInfo(1, 0) for _, testcase := range []struct { expected string node Node }{ { expected: `a = "1"`, node: NewAssignNode(zeroFileInfo, []*NameNode{ NewNameNode(zeroFileInfo, "a", nil), }, []Expr{NewStringExpr(zeroFileInfo, "1", true)}), }, { expected: `a = ()`, node: NewAssignNode(zeroFileInfo, []*NameNode{ NewNameNode(zeroFileInfo, "a", nil), }, []Expr{NewListExpr(zeroFileInfo, []Expr{})}), }, { expected: `a, b = (), ()`, node: NewAssignNode(zeroFileInfo, []*NameNode{ NewNameNode(zeroFileInfo, "a", nil), NewNameNode(zeroFileInfo, "b", nil), }, []Expr{NewListExpr(zeroFileInfo, []Expr{}), NewListExpr(zeroFileInfo, []Expr{})}), }, { expected: `a, b = "1", "2"`, node: NewAssignNode(zeroFileInfo, []*NameNode{ NewNameNode(zeroFileInfo, "a", nil), NewNameNode(zeroFileInfo, "b", nil), }, []Expr{NewStringExpr(zeroFileInfo, "1", true), NewStringExpr(zeroFileInfo, "2", true)}), }, } { testPrinter(t, testcase.node, testcase.expected) } } ================================================ FILE: ast/nodetype_string.go ================================================ // Code generated by "stringer -type=NodeType"; DO NOT EDIT package ast import "fmt" const _NodeType_name = "NodeSetenvNodeBlockNodeNameNodeAssignNodeExecAssignNodeImportexecBeginNodeCommandNodePipeNodeRedirectNodeFnInvexecEndexpressionBeginNodeStringExprNodeIntExprNodeVarExprNodeListExprNodeIndexExprNodeConcatExprexpressionEndNodeStringNodeRforkNodeRforkFlagsNodeIfNodeCommentNodeFnArgNodeVarAssignDeclNodeVarExecAssignDeclNodeFnDeclNodeReturnNodeBindFnNodeDumpNodeFor" var _NodeType_index = [...]uint16{0, 10, 19, 27, 37, 51, 61, 70, 81, 89, 101, 110, 117, 132, 146, 157, 168, 180, 193, 207, 220, 230, 239, 253, 259, 270, 279, 296, 317, 327, 337, 347, 355, 362} func (i NodeType) String() string { i -= 1 if i < 0 || i >= NodeType(len(_NodeType_index)-1) { return fmt.Sprintf("NodeType(%d)", i+1) } return _NodeType_name[_NodeType_index[i]:_NodeType_index[i+1]] } ================================================ FILE: ast/tree.go ================================================ package ast type ( // Tree is the AST Tree struct { Name string Root *BlockNode // top-level root of the tree. } ) // NewTree creates a new AST tree func NewTree(name string) *Tree { return &Tree{ Name: name, } } func (t *Tree) IsEqual(other *Tree) bool { if t == other { return true } return t.Root.IsEqual(other.Root) } func (tree *Tree) String() string { if tree.Root == nil { return "" } if len(tree.Root.Nodes) == 0 { return "" } return tree.Root.String() } ================================================ FILE: ast/tree_test.go ================================================ package ast import ( "testing" "github.com/madlambda/nash/token" ) // Test API func TestTreeCreation(t *testing.T) { tr := NewTree("example") if tr.Name != "example" { t.Errorf("Invalid name") return } } func TestTreeRawCreation(t *testing.T) { tr := NewTree("creating a tree by hand") ln := NewBlockNode(token.NewFileInfo(1, 0)) rfarg := NewStringExpr(token.NewFileInfo(1, 0), "unp", false) r := NewRforkNode(token.NewFileInfo(1, 0)) r.SetFlags(rfarg) ln.Push(r) tr.Root = ln if tr.String() != "rfork unp" { t.Error("Failed to build AST by hand") } } ================================================ FILE: cmd/nash/cli.go ================================================ package main import ( "bytes" "fmt" "io" "os" "strings" "github.com/madlambda/nash" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/parser" "github.com/madlambda/nash/sh" "github.com/chzyer/readline" ) type ( Interrupted interface { Interrupted() bool } Ignored interface { Ignore() bool } BlockNotFinished interface { Unfinished() bool } ) var completers = []readline.PrefixCompleterInterface{} func execFn(shell *nash.Shell, fnDef sh.FnDef, args []sh.Obj) { fn := fnDef.Build() err := fn.SetArgs(args) if err != nil { fmt.Fprintf(os.Stderr, "%s failed: %s\n", fnDef.Name(), err.Error()) } fn.SetStdin(shell.Stdin()) fn.SetStdout(shell.Stdout()) fn.SetStderr(shell.Stderr()) if err := fn.Start(); err != nil { fmt.Fprintf(os.Stderr, "%s failed: %s\n", fnDef.Name(), err.Error()) return } if err := fn.Wait(); err != nil { fmt.Fprintf(os.Stderr, "%s failed: %s\n", fnDef.Name(), err.Error()) return } } func importInitFile(shell *nash.Shell, initFile string) (bool, error) { if d, err := os.Stat(initFile); err == nil { if m := d.Mode(); !m.IsDir() { err := shell.ExecuteString("init", fmt.Sprintf("import %q", initFile)) if err != nil { return false, fmt.Errorf("Failed to evaluate '%s': %s", initFile, err.Error()) } return true, nil } } return false, nil } func loadInit(shell *nash.Shell) error { if noInit { return nil } initFiles := []string{ shell.NashPath() + "/init", shell.NashPath() + "/init.sh", } for _, init := range initFiles { imported, err := importInitFile(shell, init) if err != nil { return err } if imported { break } } return nil } func cli(shell *nash.Shell) error { shell.SetInteractive(true) if err := loadInit(shell); err != nil { fmt.Fprintf(os.Stderr, "error loading init file:\n%s\n", err) } historyFile := shell.NashPath() + "/history" cfg := readline.Config{ Prompt: shell.Prompt(), HistoryFile: historyFile, InterruptPrompt: "^C", EOFPrompt: "exit", } term, err := readline.NewTerminal(&cfg) if err != nil { return err } op := term.Readline() rline := &readline.Instance{ Config: &cfg, Terminal: term, Operation: op, } defer rline.Close() completer := NewCompleter(op, term, shell) cfg.AutoComplete = completer if lineMode, ok := shell.Getvar("LINEMODE"); ok { if lineStr, ok := lineMode.(*sh.StrObj); ok && lineStr.Str() == "vim" { rline.SetVimMode(true) } else { rline.SetVimMode(false) } } return docli(shell, rline) } func docli(shell *nash.Shell, rline *readline.Instance) error { var ( content bytes.Buffer lineidx int line string parse *parser.Parser tr *ast.Tree err error unfinished bool prompt string ) for { if fnDef, err := shell.GetFn("nash_repl_before"); err == nil && !unfinished { execFn(shell, fnDef, nil) } if !unfinished { prompt = shell.Prompt() } rline.SetPrompt(prompt) line, err = rline.Readline() if err == readline.ErrInterrupt { goto cont } else if err == io.EOF { err = nil break } lineidx++ line = strings.TrimSpace(line) // handle special cli commands switch { case strings.HasPrefix(line, "set mode "): switch line[9:] { case "vi": rline.SetVimMode(true) case "emacs": rline.SetVimMode(false) default: fmt.Printf("invalid mode: %s\n", line[9:]) } goto cont case line == "mode": if rline.IsVimMode() { fmt.Printf("Current mode: vim\n") } else { fmt.Printf("Current mode: emacs\n") } goto cont case line == "exit": break } content.Write([]byte(line + "\n")) parse = parser.NewParser(fmt.Sprintf("", lineidx), string(content.Bytes())) line = string(content.Bytes()) tr, err = parse.Parse() if err != nil { if interrupted, ok := err.(Interrupted); ok && interrupted.Interrupted() { content.Reset() goto cont } else if errBlock, ok := err.(BlockNotFinished); ok && errBlock.Unfinished() { prompt = ">>> " unfinished = true goto cont } fmt.Printf("ERROR: %s\n", err.Error()) content.Reset() goto cont } unfinished = false content.Reset() _, err = shell.ExecuteTree(tr) if err != nil { fmt.Printf("ERROR: %s\n", err.Error()) } cont: if fnDef, err := shell.GetFn("nash_repl_after"); err == nil && !unfinished { var status sh.Obj var ok bool if status, ok = shell.Getvar("status"); !ok { status = sh.NewStrObj("") } execFn(shell, fnDef, []sh.Obj{sh.NewStrObj(line), status}) } rline.SetPrompt(prompt) } return nil } ================================================ FILE: cmd/nash/completer.go ================================================ package main import ( "fmt" "os" "strconv" "github.com/madlambda/nash" "github.com/madlambda/nash/sh" "github.com/chzyer/readline" ) var runes = readline.Runes{} type Completer struct { op *readline.Operation term *readline.Terminal sh *nash.Shell } func NewCompleter(op *readline.Operation, term *readline.Terminal, sh *nash.Shell) *Completer { return &Completer{op, term, sh} } func (c *Completer) Do(line []rune, pos int) ([][]rune, int) { var ( newLine [][]rune offset int lineArg = sh.NewStrObj(string(line)) posArg = sh.NewStrObj(strconv.Itoa(pos)) ) defer c.op.Refresh() defer c.term.PauseRead(false) fnDef, err := c.sh.GetFn("nash_complete") if err != nil { // no complete available return [][]rune{[]rune{'\t'}}, offset } nashFunc := fnDef.Build() err = nashFunc.SetArgs([]sh.Obj{lineArg, posArg}) if err != nil { fmt.Fprintf(os.Stderr, "Failed to autocomplete: %s\n", err.Error()) return newLine, offset } nashFunc.SetStdin(c.sh.Stdin()) nashFunc.SetStdout(c.sh.Stdout()) nashFunc.SetStderr(c.sh.Stderr()) if err = nashFunc.Start(); err != nil { fmt.Fprintf(os.Stderr, "Failed to autocomplete: %s\n", err.Error()) return newLine, offset } if err = nashFunc.Wait(); err != nil { fmt.Fprintf(os.Stderr, "Failed to autocomplete: %s\n", err.Error()) return newLine, offset } ret := nashFunc.Results() if len(ret) != 1 || ret[0].Type() != sh.ListType { fmt.Fprintf(os.Stderr, "ignoring autocomplete value: %v\n", ret) return newLine, offset } retval := ret[0] retlist := retval.(*sh.ListObj) if len(retlist.List()) != 2 { return newLine, pos } newline := retlist.List()[0] newpos := retlist.List()[1] if newline.Type() != sh.StringType || newpos.Type() != sh.StringType { fmt.Fprintf(os.Stderr, "ignoring autocomplete value: (%s) (%s)\n", newline, newpos) return newLine, offset } objline := newline.(*sh.StrObj) objpos := newpos.(*sh.StrObj) newoffset, err := strconv.Atoi(objpos.Str()) if err != nil { fmt.Fprintf(os.Stderr, "Failed to autocomplete: %s\n", err.Error()) return newLine, offset } newLine = append(newLine, []rune(objline.Str())) return newLine, newoffset } ================================================ FILE: cmd/nash/env.go ================================================ package main import ( "errors" "os" "path/filepath" ) func NashPath() (string, error) { nashpath := os.Getenv("NASHPATH") if nashpath != "" { return nashpath, nil } h, err := home() return filepath.Join(h, "nash"), err } func NashRoot() (string, error) { nashroot, ok := os.LookupEnv("NASHROOT") if ok { return nashroot, nil } h, err := home() return filepath.Join(h, "nashroot"), err } func home() (string, error) { homedir, err := os.UserHomeDir() if err != nil { return "", err } if homedir == "" { return "", errors.New("invalid empty home dir") } return homedir, nil } ================================================ FILE: cmd/nash/env_test.go ================================================ package main_test import ( "os" "path/filepath" "strings" "testing" main "github.com/madlambda/nash/cmd/nash" ) // TODO: No idea on how to inject failures like empty HOME folders for now func TestLoadNASHPATH(t *testing.T) { defaultNashPath := filepath.Join(home(t), "nash") runTests(t, main.NashPath, []EnvTest{ { name: "Exported", env: map[string]string{ "NASHPATH": filepath.Join("etc", "nash"), }, want: filepath.Join("etc", "nash"), }, { name: "IgnoresNASHROOT", env: map[string]string{ "NASHROOT": "/etc/nashroot/tests", "HOME": home(t), }, want: defaultNashPath, }, { name: "UseUserHomeWhenUnset", env: map[string]string{ "NASHROOT": "/etc/nashroot/tests", "HOME": home(t), }, want: defaultNashPath, }, }) } func TestLoadNASHROOT(t *testing.T) { defaultNashRoot := filepath.Join(home(t), "nashroot") runTests(t, main.NashRoot, []EnvTest{ { name: "Exported", env: map[string]string{ "NASHROOT": filepath.Join("etc", "nashroot"), }, want: filepath.Join("etc", "nashroot"), }, { name: "IgnoresGOPATHIfSet", env: map[string]string{ "GOPATH": filepath.Join("go", "natel", "review"), "NASHROOT": filepath.Join("nashroot", "ignoredgopath"), }, want: filepath.Join("nashroot", "ignoredgopath"), }, { name: "UsesHOMEevenWhenGOPATHIsSet", env: map[string]string{ "HOME": home(t), "GOPATH": filepath.Join("go", "path"), }, want: defaultNashRoot, }, { name: "UsesUserHomeWhenNASHROOTAndGOPATHAreUnset", env: map[string]string{ "HOME": home(t), }, want: filepath.Join(home(t), "nashroot"), }, }) } func runTests(t *testing.T, testfunc func() (string, error), cases []EnvTest) { t.Helper() for _, c := range cases { t.Run(c.name, func(t *testing.T) { restore := clearenv(t) defer restore() export(t, c.env) got, err := testfunc() if err != nil { t.Fatal(err) } if got != c.want { t.Fatalf("got[%s] != want[%s]", got, c.want) } }) } } type EnvTest struct { name string env map[string]string want string } func clearenv(t *testing.T) func() { env := os.Environ() os.Clearenv() return func() { for _, envvar := range env { parsed := strings.Split(envvar, "=") name := parsed[0] val := strings.Join(parsed[1:], "=") err := os.Setenv(name, val) if err != nil { t.Fatalf("error[%s] restoring env var[%s]", err, envvar) } } } } func export(t *testing.T, env map[string]string) { t.Helper() for name, val := range env { err := os.Setenv(name, val) if err != nil { t.Fatal(err) } } } func home(t *testing.T) string { t.Helper() homedir, err := os.UserHomeDir() if err != nil { t.Fatal(err) } return homedir } ================================================ FILE: cmd/nash/example.sh ================================================ #!/usr/bin/env nash -rm -rf rootfs rfork upmis { mount -t proc proc /proc mkdir -p rootfs mount -t tmpfs -o size=1G tmpfs rootfs cd rootfs wget "https://busybox.net/downloads/binaries/latest/busybox-x86_64" -O busybox chmod +x busybox mkdir bin ./busybox --install ./bin mkdir -p proc mkdir -p dev mount -t proc proc proc mount -t tmpfs tmpfs dev cp ../nash . chroot . /bin/sh } ================================================ FILE: cmd/nash/install.go ================================================ package main import ( "fmt" "io" "io/ioutil" "os" "path/filepath" ) func NashLibDir(nashpath string) string { //FIXME: This is sadly duplicated from the shell implementation =( return filepath.Join(nashpath, "lib") } func InstallLib(nashpath string, sourcepath string) error { nashlibdir := NashLibDir(nashpath) sourcepathAbs, err := filepath.Abs(sourcepath) if err != nil { return fmt.Errorf("error[%s] getting absolute path of [%s]", err, sourcepath) } if filepath.HasPrefix(sourcepathAbs, nashlibdir) { return fmt.Errorf( "lib source path[%s] can't be inside nash lib dir[%s]", sourcepath, nashlibdir) } return installLib(nashlibdir, sourcepathAbs) } func installLib(targetdir string, sourcepath string) error { f, err := os.Stat(sourcepath) if err != nil { return fmt.Errorf("error[%s] checking if path[%s] is dir", err, sourcepath) } if !f.IsDir() { return copyfile(targetdir, sourcepath) } basedir := filepath.Base(sourcepath) targetdir = filepath.Join(targetdir, basedir) files, err := ioutil.ReadDir(sourcepath) if err != nil { return fmt.Errorf("error[%s] reading dir[%s]", err, sourcepath) } for _, file := range files { err := installLib(targetdir, filepath.Join(sourcepath, file.Name())) if err != nil { return err } } return nil } func copyfile(targetdir string, sourcefilepath string) error { fail := func(err error) error { return fmt.Errorf( "error[%s] trying to copy file[%s] to [%s]", err, sourcefilepath, targetdir) } err := os.MkdirAll(targetdir, os.ModePerm) if err != nil { return fail(err) } sourcefile, err := os.Open(sourcefilepath) if err != nil { return fail(err) } defer sourcefile.Close() targetfilepath := filepath.Join(targetdir, filepath.Base(sourcefilepath)) targetfile, err := os.Create(targetfilepath) if err != nil { return fail(err) } defer targetfile.Close() _, err = io.Copy(targetfile, sourcefile) return err } ================================================ FILE: cmd/nash/install_test.go ================================================ package main_test import ( "io/ioutil" "os" "path/filepath" "testing" main "github.com/madlambda/nash/cmd/nash" "github.com/madlambda/nash/internal/testing/fixture" ) // TODO: test when nashpath lib already exists and has libraries inside func TestInstallLib(t *testing.T) { type testcase struct { name string libfiles []string installpath string // want will map the wanted files to the original files copied from the lib // the wanted files paths are relative to inside the nashpath lib dir. // the files need to be mapped to the original files because of content validation // when multiple files are installed. want map[string]string } cases := []testcase{ { name: "SingleFile", libfiles: []string{ "/testfile/file.sh", }, installpath: "/testfile/file.sh", want: map[string]string{ "file.sh": "/testfile/file.sh", }, }, { name: "SingleDir", libfiles: []string{ "/testfile/file.sh", }, installpath: "/testfile", want: map[string]string{ "/testfile/file.sh": "/testfile/file.sh", }, }, { name: "SingleDirWithMultipleFiles", libfiles: []string{ "/testfile/file.sh", "/testfile/fileagain.sh", }, installpath: "/testfile", want: map[string]string{ "/testfile/file.sh": "/testfile/file.sh", "/testfile/fileagain.sh": "/testfile/fileagain.sh", }, }, { name: "MultipleDirsWithMultipleFiles", libfiles: []string{ "/testfile/file.sh", "/testfile/dir1/file.sh", "/testfile/dir1/fileagain.sh", "/testfile/dir2/file.sh", "/testfile/dir2/fileagain.sh", "/testfile/dir2/dir3/file.sh", }, installpath: "/testfile", want: map[string]string{ "/testfile/file.sh": "/testfile/file.sh", "/testfile/dir1/file.sh": "/testfile/dir1/file.sh", "/testfile/dir1/fileagain.sh": "/testfile/dir1/fileagain.sh", "/testfile/dir2/file.sh": "/testfile/dir2/file.sh", "/testfile/dir2/fileagain.sh": "/testfile/dir2/fileagain.sh", "/testfile/dir2/dir3/file.sh": "/testfile/dir2/dir3/file.sh", }, }, { name: "InstallOnlyFilesIndicatedByInstallDir", libfiles: []string{ "/testfile/file.sh", "/testfile/dir1/file.sh", "/testfile/dir1/fileagain.sh", "/testfile/dir2/file.sh", "/testfile/dir2/fileagain.sh", "/testfile/dir2/dir3/file.sh", }, installpath: "/testfile/dir2", want: map[string]string{ "/dir2/file.sh": "/testfile/dir2/file.sh", "/dir2/fileagain.sh": "/testfile/dir2/fileagain.sh", "/dir2/dir3/file.sh": "/testfile/dir2/dir3/file.sh", }, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() libfilesDir, rmlibfilesDir := fixture.Tmpdir(t) defer rmlibfilesDir() nashlibdir := main.NashLibDir(nashpath) libfiles := []string{} libfileFullPath := func(libfilepath string) string { return filepath.Join(libfilesDir, libfilepath) } for _, f := range c.libfiles { libfiles = append(libfiles, libfileFullPath(f)) } createdLibFiles := fixture.CreateFiles(t, libfiles) installpath := filepath.Join(libfilesDir, c.installpath) err := main.InstallLib(nashpath, installpath) if err != nil { t.Fatal(err) } listNashPathFiles := func() []string { files := []string{} filepath.Walk(nashpath, func(path string, stats os.FileInfo, err error) error { if stats.IsDir() { return nil } files = append(files, path) return nil }) return files } gotFiles := listNashPathFiles() fatal := func() { t.Errorf("nashpath: [%s]", nashpath) t.Errorf("nashpath contents:") for _, path := range gotFiles { t.Errorf("[%s]", path) } t.Fatal("") } if len(gotFiles) != len(c.want) { t.Errorf("wanted[%d] files but got[%d]", len(c.want), len(gotFiles)) fatal() } for wantFilepath, libfilepath := range c.want { completeLibFilepath := libfileFullPath(libfilepath) wantContents, ok := createdLibFiles[completeLibFilepath] if !ok { t.Errorf("unable to find libfilepath[%s] contents on created lib files map[%+v]", completeLibFilepath, createdLibFiles) t.Fatal("this probably means a wrongly specified test case with wanted files that are not present on the libfiles") } fullWantFilepath := filepath.Join(nashlibdir, wantFilepath) wantFile, err := os.Open(fullWantFilepath) if err != nil { t.Errorf("error[%s] checking wanted file[%s]", err, wantFilepath) fatal() } gotContentsRaw, err := ioutil.ReadAll(wantFile) wantFile.Close() if err != nil { t.Errorf("error[%s] checking existence of wanted file[%s]", err, wantFilepath) fatal() } gotContents := string(gotContentsRaw) if gotContents != wantContents { t.Errorf("for file [%s] wanted contents [%s] but got [%s]", wantFilepath, wantContents, gotContents) fatal() } } }) } } func TestSourcePathCantBeEqualToNashLibDir(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() nashlibdir := main.NashLibDir(nashpath) fixture.CreateFile(t, filepath.Join(nashlibdir, "whatever.sh")) assertInstallLibFails(t, nashpath, nashlibdir) } func TestSourcePathCantBeInsideNashLibDir(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() nashlibdir := main.NashLibDir(nashpath) sourcelibdir := filepath.Join(nashlibdir, "somedir") fixture.CreateFile(t, filepath.Join(sourcelibdir, "whatever.sh")) assertInstallLibFails(t, nashpath, sourcelibdir) } func TestRelativeSourcePathCantBeInsideNashLibDir(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() nashlibdir := main.NashLibDir(nashpath) fixture.CreateFile(t, filepath.Join(nashlibdir, "somedir", "whatever.sh")) oldwd := fixture.WorkingDir(t) defer fixture.ChangeDir(t, oldwd) fixture.ChangeDir(t, nashlibdir) assertInstallLibFails(t, nashpath, "./somedir") } func TestFailsOnUnexistentSourcePath(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() assertInstallLibFails(t, nashpath, "/nonexistent/nash/crap") } func TestFailsOnUnreadableSourcePath(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() sourcedir, rmsourcedir := fixture.Tmpdir(t) defer rmsourcedir() fixture.Chmod(t, sourcedir, writeOnly) assertInstallLibFails(t, nashpath, sourcedir) } func TestFailsOnUnreadableFileInsideSourcePath(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() sourcedir, rmsourcedir := fixture.Tmpdir(t) defer rmsourcedir() readableFile := filepath.Join(sourcedir, "file1.sh") unreadableFile := filepath.Join(sourcedir, "file2.sh") fixture.CreateFiles(t, []string{readableFile, unreadableFile}) fixture.Chmod(t, unreadableFile, writeOnly) assertInstallLibFails(t, nashpath, sourcedir) } func TestFailsOnUnwriteableNashPath(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() sourcedir, rmsourcedir := fixture.Tmpdir(t) defer rmsourcedir() fixture.Chmod(t, nashpath, readOnly) fixture.CreateFile(t, filepath.Join(sourcedir, "file.sh")) assertInstallLibFails(t, nashpath, sourcedir) } func TestFailsOnUnwriteableFileInsideNashLibdir(t *testing.T) { nashpath, rmnashpath := fixture.Tmpdir(t) defer rmnashpath() sourcedir, rmsourcedir := fixture.Tmpdir(t) defer rmsourcedir() filename := "test.sh" sourcefile := filepath.Join(sourcedir, filename) expectedInstalledFile := filepath.Join( main.NashLibDir(nashpath), filepath.Base(sourcedir), filename, ) fixture.CreateFiles(t, []string{sourcefile, expectedInstalledFile}) fixture.Chmod(t, expectedInstalledFile, readOnly) assertInstallLibFails(t, nashpath, sourcedir) } func assertInstallLibFails(t *testing.T, nashpath string, sourcepath string) { t.Helper() err := main.InstallLib(nashpath, sourcepath) if err == nil { t.Fatal("expected error, got nil") } } const writeOnly = 0333 const readOnly = 0555 ================================================ FILE: cmd/nash/main.go ================================================ // Package main has two sides: // - User mode: shell // - tool mode: unix socket server for handling namespace operations // When started, the program choses their side based on the argv[0]. // The name "nash" indicates a user shell and the name -nashd- indicates // the namespace server tool. package main import ( "flag" "fmt" "os" "github.com/madlambda/nash" ) var ( // version is set at build time VersionString = "No version provided" version bool debug bool file string command string addr string noInit bool interactive bool install string ) func init() { flag.BoolVar(&version, "version", false, "Show version") flag.BoolVar(&debug, "debug", false, "enable debug") flag.BoolVar(&noInit, "noinit", false, "do not load init/init.sh file") flag.StringVar(&command, "c", "", "command to execute") flag.StringVar(&install, "install", "", "path of the library that you want to install (can be a single file)") flag.BoolVar(&interactive, "i", false, "Interactive mode (default if no args)") if os.Args[0] == "-nashd-" || (len(os.Args) > 1 && os.Args[1] == "-daemon") { flag.Bool("daemon", false, "force enable nashd mode") flag.StringVar(&addr, "addr", "", "rcd unix file") } } func main() { var args []string var shell *nash.Shell var err error flag.Parse() if version { fmt.Printf("build tag: %s\n", VersionString) return } if install != "" { fmt.Printf("installing library located at [%s]\n", install) np, err := NashPath() if err != nil { fmt.Printf("error[%s] getting NASHPATH, cant install library\n", err) os.Exit(1) } err = InstallLib(np, install) if err != nil { fmt.Printf("error[%s] installing library\n", err) os.Exit(1) } fmt.Println("installed with success") return } if len(flag.Args()) > 0 { args = flag.Args() file = args[0] } if shell, err = initShell(); err != nil { goto Error } shell.SetDebug(debug) if addr != "" { startNashd(shell, addr) return } if (file == "" && command == "") || interactive { if err = cli(shell); err != nil { goto Error } return } if file != "" { if err = shell.ExecFile(file, args...); err != nil { goto Error } } if command != "" { err = shell.ExecuteString("", command) if err != nil { goto Error } } Error: if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err.Error()) os.Exit(1) } } func initShell() (*nash.Shell, error) { nashpath, err := NashPath() if err != nil { return nil, err } nashroot, err := NashRoot() if err != nil { return nil, err } os.Mkdir(nashpath, 0755) return nash.New(nashpath, nashroot) } ================================================ FILE: cmd/nash/rpc.go ================================================ package main import ( "fmt" "io" "net" "os" "github.com/madlambda/nash" ) func serveConn(sh *nash.Shell, conn net.Conn) { var data [1024]byte for { n, err := conn.Read(data[:]) if err != nil { if err == io.EOF { return } fmt.Printf("Failed to read data: %s", err.Error()) return } if string(data[0:n]) == "quit" { return } err = sh.ExecuteString("-nashd-", string(data[0:n])) if err != nil { fmt.Printf("nashd: %s\n", err.Error()) _, err = conn.Write([]byte("1")) if err != nil { fmt.Printf("Failed to send command status.\n") return } } else { _, err = conn.Write([]byte("0")) if err != nil { fmt.Printf("Failed to send command status.\n") return } } } } func startNashd(sh *nash.Shell, socketPath string) { os.Remove(socketPath) addr := &net.UnixAddr{ Net: "unix", Name: socketPath, } listener, err := net.ListenUnix("unix", addr) if err != nil { fmt.Printf("ERROR: %s\n", err.Error()) return } // Accept only one connection conn, err := listener.AcceptUnix() if err != nil { fmt.Printf("ERROR: %v", err.Error()) } serveConn(sh, conn) listener.Close() } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/.travis.yml ================================================ language: go go: - 1.5 - 1.7 script: - GOOS=windows go install github.com/chzyer/readline/example/... - GOOS=linux go install github.com/chzyer/readline/example/... - GOOS=darwin go install github.com/chzyer/readline/example/... - go test -race -v ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/CHANGELOG.md ================================================ # ChangeLog ### 1.4 - 2016-07-25 * [#60][60] Support dynamic autocompletion * Fix ANSI parser on Windows * Fix wrong column width in complete mode on Windows * Remove dependent package "golang.org/x/crypto/ssh/terminal" ### 1.3 - 2016-05-09 * [#38][38] add SetChildren for prefix completer interface * [#42][42] improve multiple lines compatibility * [#43][43] remove sub-package(runes) for gopkg compatiblity * [#46][46] Auto complete with space prefixed line * [#48][48] support suspend process (ctrl+Z) * [#49][49] fix bug that check equals with previous command * [#53][53] Fix bug which causes integer divide by zero panicking when input buffer is empty ### 1.2 - 2016-03-05 * Add a demo for checking password strength [example/readline-pass-strength](https://github.com/chzyer/readline/blob/master/example/readline-pass-strength/readline-pass-strength.go), , written by [@sahib](https://github.com/sahib) * [#23][23], support stdin remapping * [#27][27], add a `UniqueEditLine` to `Config`, which will erase the editing line after user submited it, usually use in IM. * Add a demo for multiline [example/readline-multiline](https://github.com/chzyer/readline/blob/master/example/readline-multiline/readline-multiline.go) which can submit one SQL by multiple lines. * Supports performs even stdin/stdout is not a tty. * Add a new simple apis for single instance, check by [here](https://github.com/chzyer/readline/blob/master/std.go). It need to save history manually if using this api. * [#28][28], fixes the history is not working as expected. * [#33][33], vim mode now support `c`, `d`, `x (delete character)`, `r (replace character)` ### 1.1 - 2015-11-20 * [#12][12] Add support for key ``/``/`` * Only enter raw mode as needed (calling `Readline()`), program will receive signal(e.g. Ctrl+C) if not interact with `readline`. * Bugs fixed for `PrefixCompleter` * Press `Ctrl+D` in empty line will cause `io.EOF` in error, Press `Ctrl+C` in anytime will cause `ErrInterrupt` instead of `io.EOF`, this will privodes a shell-like user experience. * Customable Interrupt/EOF prompt in `Config` * [#17][17] Change atomic package to use 32bit function to let it runnable on arm 32bit devices * Provides a new password user experience(`readline.ReadPasswordEx()`). ### 1.0 - 2015-10-14 * Initial public release. [12]: https://github.com/chzyer/readline/pull/12 [17]: https://github.com/chzyer/readline/pull/17 [23]: https://github.com/chzyer/readline/pull/23 [27]: https://github.com/chzyer/readline/pull/27 [28]: https://github.com/chzyer/readline/pull/28 [33]: https://github.com/chzyer/readline/pull/33 [38]: https://github.com/chzyer/readline/pull/38 [42]: https://github.com/chzyer/readline/pull/42 [43]: https://github.com/chzyer/readline/pull/43 [46]: https://github.com/chzyer/readline/pull/46 [48]: https://github.com/chzyer/readline/pull/48 [49]: https://github.com/chzyer/readline/pull/49 [53]: https://github.com/chzyer/readline/pull/53 [60]: https://github.com/chzyer/readline/pull/60 ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/LICENSE ================================================ The MIT License (MIT) Copyright (c) 2015 Chzyer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/README.md ================================================ [![Build Status](https://travis-ci.org/chzyer/readline.svg?branch=master)](https://travis-ci.org/chzyer/readline) [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE.md) [![Version](https://img.shields.io/github/tag/chzyer/readline.svg)](https://github.com/chzyer/readline/releases) [![GoDoc](https://godoc.org/github.com/chzyer/readline?status.svg)](https://godoc.org/github.com/chzyer/readline) [![OpenCollective](https://opencollective.com/readline/badge/backers.svg)](#backers) [![OpenCollective](https://opencollective.com/readline/badge/sponsors.svg)](#sponsors)

A powerful readline library in `Linux` `macOS` `Windows` ## Guide * [Demo](example/readline-demo/readline-demo.go) * [Shortcut](doc/shortcut.md) ## Repos using readline [![cockroachdb](https://img.shields.io/github/stars/cockroachdb/cockroach.svg?label=cockroachdb/cockroach)](https://github.com/cockroachdb/cockroach) [![empire](https://img.shields.io/github/stars/remind101/empire.svg?label=remind101/empire)](https://github.com/remind101/empire) [![youtube/doorman](https://img.shields.io/github/stars/youtube/doorman.svg?label=youtube/doorman)](https://github.com/youtube/doorman) [![bom-d-van/harp](https://img.shields.io/github/stars/bom-d-van/harp.svg?label=bom-d-van/harp)](https://github.com/bom-d-van/harp) [![abiosoft/ishell](https://img.shields.io/github/stars/abiosoft/ishell.svg?label=abiosoft/ishell)](https://github.com/abiosoft/ishell) [![robertkrimen/otto](https://img.shields.io/github/stars/robertkrimen/otto.svg?label=robertkrimen/otto)](https://github.com/robertkrimen/otto) [![Netflix/hal-9001](https://img.shields.io/github/stars/Netflix/hal-9001.svg?label=Netflix/hal-9001)](https://github.com/Netflix/hal-9001) [![docker/go-p9p](https://img.shields.io/github/stars/docker/go-p9p.svg?label=docker/go-p9p)](https://github.com/docker/go-p9p) [![mehrdadrad/mylg](https://img.shields.io/github/stars/mehrdadrad/mylg.svg?label=mehrdadrad/mylg)](https://github.com/mehrdadrad/mylg) ## Feedback If you have any questions, please submit a github issue and any pull requests is welcomed :) * [https://twitter.com/chzyer](https://twitter.com/chzyer) * [http://weibo.com/2145262190](http://weibo.com/2145262190) ## Backers Love Readline? Help me keep it alive by donating funds to cover project expenses!
[[Become a backer](https://opencollective.com/readline#backer)] ## Sponsors Become a sponsor and get your logo here on our Github page. [[Become a sponsor](https://opencollective.com/readline#sponsor)] ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/ansi_windows.go ================================================ // +build windows package readline import ( "bufio" "io" "strconv" "strings" "sync" "unicode/utf8" "unsafe" ) const ( _ = uint16(0) COLOR_FBLUE = 0x0001 COLOR_FGREEN = 0x0002 COLOR_FRED = 0x0004 COLOR_FINTENSITY = 0x0008 COLOR_BBLUE = 0x0010 COLOR_BGREEN = 0x0020 COLOR_BRED = 0x0040 COLOR_BINTENSITY = 0x0080 COMMON_LVB_UNDERSCORE = 0x8000 ) var ColorTableFg = []word{ 0, // 30: Black COLOR_FRED, // 31: Red COLOR_FGREEN, // 32: Green COLOR_FRED | COLOR_FGREEN, // 33: Yellow COLOR_FBLUE, // 34: Blue COLOR_FRED | COLOR_FBLUE, // 35: Magenta COLOR_FGREEN | COLOR_FBLUE, // 36: Cyan COLOR_FRED | COLOR_FBLUE | COLOR_FGREEN, // 37: White } var ColorTableBg = []word{ 0, // 40: Black COLOR_BRED, // 41: Red COLOR_BGREEN, // 42: Green COLOR_BRED | COLOR_BGREEN, // 43: Yellow COLOR_BBLUE, // 44: Blue COLOR_BRED | COLOR_BBLUE, // 45: Magenta COLOR_BGREEN | COLOR_BBLUE, // 46: Cyan COLOR_BRED | COLOR_BBLUE | COLOR_BGREEN, // 47: White } type ANSIWriter struct { target io.Writer wg sync.WaitGroup ctx *ANSIWriterCtx sync.Mutex } func NewANSIWriter(w io.Writer) *ANSIWriter { a := &ANSIWriter{ target: w, ctx: NewANSIWriterCtx(w), } return a } func (a *ANSIWriter) Close() error { a.wg.Wait() return nil } type ANSIWriterCtx struct { isEsc bool isEscSeq bool arg []string target *bufio.Writer wantFlush bool } func NewANSIWriterCtx(target io.Writer) *ANSIWriterCtx { return &ANSIWriterCtx{ target: bufio.NewWriter(target), } } func (a *ANSIWriterCtx) Flush() { a.target.Flush() } func (a *ANSIWriterCtx) process(r rune) bool { if a.wantFlush { if r == 0 || r == CharEsc { a.wantFlush = false a.target.Flush() } } if a.isEscSeq { a.isEscSeq = a.ioloopEscSeq(a.target, r, &a.arg) return true } switch r { case CharEsc: a.isEsc = true case '[': if a.isEsc { a.arg = nil a.isEscSeq = true a.isEsc = false break } fallthrough default: a.target.WriteRune(r) a.wantFlush = true } return true } func (a *ANSIWriterCtx) ioloopEscSeq(w *bufio.Writer, r rune, argptr *[]string) bool { arg := *argptr var err error if r >= 'A' && r <= 'D' { count := short(GetInt(arg, 1)) info, err := GetConsoleScreenBufferInfo() if err != nil { return false } switch r { case 'A': // up info.dwCursorPosition.y -= count case 'B': // down info.dwCursorPosition.y += count case 'C': // right info.dwCursorPosition.x += count case 'D': // left info.dwCursorPosition.x -= count } SetConsoleCursorPosition(&info.dwCursorPosition) return false } switch r { case 'J': killLines() case 'K': eraseLine() case 'm': color := word(0) for _, item := range arg { var c int c, err = strconv.Atoi(item) if err != nil { w.WriteString("[" + strings.Join(arg, ";") + "m") break } if c >= 30 && c < 40 { color ^= COLOR_FINTENSITY color |= ColorTableFg[c-30] } else if c >= 40 && c < 50 { color ^= COLOR_BINTENSITY color |= ColorTableBg[c-40] } else if c == 4 { color |= COMMON_LVB_UNDERSCORE | ColorTableFg[7] } else { // unknown code treat as reset color = ColorTableFg[7] } } if err != nil { break } kernel.SetConsoleTextAttribute(stdout, uintptr(color)) case '\007': // set title case ';': if len(arg) == 0 || arg[len(arg)-1] != "" { arg = append(arg, "") *argptr = arg } return true default: if len(arg) == 0 { arg = append(arg, "") } arg[len(arg)-1] += string(r) *argptr = arg return true } *argptr = nil return false } func (a *ANSIWriter) Write(b []byte) (int, error) { a.Lock() defer a.Unlock() off := 0 for len(b) > off { r, size := utf8.DecodeRune(b[off:]) if size == 0 { return off, io.ErrShortWrite } off += size a.ctx.process(r) } a.ctx.Flush() return off, nil } func killLines() error { sbi, err := GetConsoleScreenBufferInfo() if err != nil { return err } size := (sbi.dwCursorPosition.y - sbi.dwSize.y) * sbi.dwSize.x size += sbi.dwCursorPosition.x var written int kernel.FillConsoleOutputAttribute(stdout, uintptr(ColorTableFg[7]), uintptr(size), sbi.dwCursorPosition.ptr(), uintptr(unsafe.Pointer(&written)), ) return kernel.FillConsoleOutputCharacterW(stdout, uintptr(' '), uintptr(size), sbi.dwCursorPosition.ptr(), uintptr(unsafe.Pointer(&written)), ) } func eraseLine() error { sbi, err := GetConsoleScreenBufferInfo() if err != nil { return err } size := sbi.dwSize.x sbi.dwCursorPosition.x = 0 var written int return kernel.FillConsoleOutputCharacterW(stdout, uintptr(' '), uintptr(size), sbi.dwCursorPosition.ptr(), uintptr(unsafe.Pointer(&written)), ) } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/complete.go ================================================ package readline import ( "bufio" "bytes" "fmt" "io" ) type AutoCompleter interface { // Readline will pass the whole line and current offset to it // Completer need to pass all the candidates, and how long they shared the same characters in line // Example: // [go, git, git-shell, grep] // Do("g", 1) => ["o", "it", "it-shell", "rep"], 1 // Do("gi", 2) => ["t", "t-shell"], 2 // Do("git", 3) => ["", "-shell"], 3 Do(line []rune, pos int) (newLine [][]rune, length int) } type TabCompleter struct{} func (t *TabCompleter) Do([]rune, int) ([][]rune, int) { return [][]rune{[]rune("\t")}, 0 } type opCompleter struct { w io.Writer op *Operation width int inCompleteMode bool inSelectMode bool candidate [][]rune candidateSource []rune candidateOff int candidateChoise int candidateColNum int } func newOpCompleter(w io.Writer, op *Operation, width int) *opCompleter { return &opCompleter{ w: w, op: op, width: width, } } func (o *opCompleter) doSelect() { if len(o.candidate) == 1 { o.op.buf.WriteRunes(o.candidate[0]) o.ExitCompleteMode(false) return } o.nextCandidate(1) o.CompleteRefresh() } func (o *opCompleter) nextCandidate(i int) { o.candidateChoise += i o.candidateChoise = o.candidateChoise % len(o.candidate) if o.candidateChoise < 0 { o.candidateChoise = len(o.candidate) + o.candidateChoise } } func (o *opCompleter) OnComplete() bool { if o.width == 0 { return false } if o.IsInCompleteSelectMode() { o.doSelect() return true } buf := o.op.buf rs := buf.Runes() if o.IsInCompleteMode() && o.candidateSource != nil && runes.Equal(rs, o.candidateSource) { o.EnterCompleteSelectMode() o.doSelect() return true } o.ExitCompleteSelectMode() o.candidateSource = rs newLines, offset := o.op.cfg.AutoComplete.Do(rs, buf.idx) if len(newLines) == 0 { o.ExitCompleteMode(false) return true } // only Aggregate candidates in non-complete mode if !o.IsInCompleteMode() { if len(newLines) == 1 { if offset > 0 { buf.Set([]rune(string(buf.Runes())[0:offset] + string(newLines[0]))) } else { buf.WriteRunes(newLines[0]) } o.ExitCompleteMode(false) return true } same, size := runes.Aggregate(newLines) if size > 0 { buf.WriteRunes(same) o.ExitCompleteMode(false) return true } } o.EnterCompleteMode(offset, newLines) return true } func (o *opCompleter) IsInCompleteSelectMode() bool { return o.inSelectMode } func (o *opCompleter) IsInCompleteMode() bool { return o.inCompleteMode } func (o *opCompleter) HandleCompleteSelect(r rune) bool { next := true switch r { case CharEnter, CharCtrlJ: next = false o.op.buf.WriteRunes(o.op.candidate[o.op.candidateChoise]) o.ExitCompleteMode(false) case CharLineStart: num := o.candidateChoise % o.candidateColNum o.nextCandidate(-num) case CharLineEnd: num := o.candidateColNum - o.candidateChoise%o.candidateColNum - 1 o.candidateChoise += num if o.candidateChoise >= len(o.candidate) { o.candidateChoise = len(o.candidate) - 1 } case CharBackspace: o.ExitCompleteSelectMode() next = false case CharTab, CharForward: o.doSelect() case CharBell, CharInterrupt: o.ExitCompleteMode(true) next = false case CharNext: tmpChoise := o.candidateChoise + o.candidateColNum if tmpChoise >= o.getMatrixSize() { tmpChoise -= o.getMatrixSize() } else if tmpChoise >= len(o.candidate) { tmpChoise += o.candidateColNum tmpChoise -= o.getMatrixSize() } o.candidateChoise = tmpChoise case CharBackward: o.nextCandidate(-1) case CharPrev: tmpChoise := o.candidateChoise - o.candidateColNum if tmpChoise < 0 { tmpChoise += o.getMatrixSize() if tmpChoise >= len(o.candidate) { tmpChoise -= o.candidateColNum } } o.candidateChoise = tmpChoise default: next = false o.ExitCompleteSelectMode() } if next { o.CompleteRefresh() return true } return false } func (o *opCompleter) getMatrixSize() int { line := len(o.candidate) / o.candidateColNum if len(o.candidate)%o.candidateColNum != 0 { line++ } return line * o.candidateColNum } func (o *opCompleter) OnWidthChange(newWidth int) { o.width = newWidth } func (o *opCompleter) CompleteRefresh() { if !o.inCompleteMode { return } lineCnt := o.op.buf.CursorLineCount() colWidth := 0 for _, c := range o.candidate { w := runes.WidthAll(c) if w > colWidth { colWidth = w } } colWidth += o.candidateOff + 1 same := o.op.buf.RuneSlice(-o.candidateOff) // -1 to avoid reach the end of line width := o.width - 1 colNum := width / colWidth colWidth += (width - (colWidth * colNum)) / colNum o.candidateColNum = colNum buf := bufio.NewWriter(o.w) buf.Write(bytes.Repeat([]byte("\n"), lineCnt)) colIdx := 0 lines := 1 buf.WriteString("\033[J") for idx, c := range o.candidate { inSelect := idx == o.candidateChoise && o.IsInCompleteSelectMode() if inSelect { buf.WriteString("\033[30;47m") } buf.WriteString(string(same)) buf.WriteString(string(c)) buf.Write(bytes.Repeat([]byte(" "), colWidth-len(c)-len(same))) if inSelect { buf.WriteString("\033[0m") } colIdx++ if colIdx == colNum { buf.WriteString("\n") lines++ colIdx = 0 } } // move back fmt.Fprintf(buf, "\033[%dA\r", lineCnt-1+lines) fmt.Fprintf(buf, "\033[%dC", o.op.buf.idx+o.op.buf.PromptLen()) buf.Flush() } func (o *opCompleter) aggCandidate(candidate [][]rune) int { offset := 0 for i := 0; i < len(candidate[0]); i++ { for j := 0; j < len(candidate)-1; j++ { if i > len(candidate[j]) { goto aggregate } if candidate[j][i] != candidate[j+1][i] { goto aggregate } } offset = i } aggregate: return offset } func (o *opCompleter) EnterCompleteSelectMode() { o.inSelectMode = true o.candidateChoise = -1 o.CompleteRefresh() } func (o *opCompleter) EnterCompleteMode(offset int, candidate [][]rune) { o.inCompleteMode = true o.candidate = candidate o.candidateOff = offset o.CompleteRefresh() } func (o *opCompleter) ExitCompleteSelectMode() { o.inSelectMode = false o.candidate = nil o.candidateChoise = -1 o.candidateOff = -1 o.candidateSource = nil } func (o *opCompleter) ExitCompleteMode(revent bool) { o.inCompleteMode = false o.ExitCompleteSelectMode() } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/complete_helper.go ================================================ package readline import ( "bytes" "strings" ) // Caller type for dynamic completion type DynamicCompleteFunc func(string) []string type PrefixCompleterInterface interface { Print(prefix string, level int, buf *bytes.Buffer) Do(line []rune, pos int) (newLine [][]rune, length int) GetName() []rune GetChildren() []PrefixCompleterInterface SetChildren(children []PrefixCompleterInterface) } type DynamicPrefixCompleterInterface interface { PrefixCompleterInterface IsDynamic() bool GetDynamicNames(line []rune) [][]rune } type PrefixCompleter struct { Name []rune Dynamic bool Callback DynamicCompleteFunc Children []PrefixCompleterInterface } func (p *PrefixCompleter) Tree(prefix string) string { buf := bytes.NewBuffer(nil) p.Print(prefix, 0, buf) return buf.String() } func Print(p PrefixCompleterInterface, prefix string, level int, buf *bytes.Buffer) { if strings.TrimSpace(string(p.GetName())) != "" { buf.WriteString(prefix) if level > 0 { buf.WriteString("├") buf.WriteString(strings.Repeat("─", (level*4)-2)) buf.WriteString(" ") } buf.WriteString(string(p.GetName()) + "\n") level++ } for _, ch := range p.GetChildren() { ch.Print(prefix, level, buf) } } func (p *PrefixCompleter) Print(prefix string, level int, buf *bytes.Buffer) { Print(p, prefix, level, buf) } func (p *PrefixCompleter) IsDynamic() bool { return p.Dynamic } func (p *PrefixCompleter) GetName() []rune { return p.Name } func (p *PrefixCompleter) GetDynamicNames(line []rune) [][]rune { var names = [][]rune{} for _, name := range p.Callback(string(line)) { names = append(names, []rune(name+" ")) } return names } func (p *PrefixCompleter) GetChildren() []PrefixCompleterInterface { return p.Children } func (p *PrefixCompleter) SetChildren(children []PrefixCompleterInterface) { p.Children = children } func NewPrefixCompleter(pc ...PrefixCompleterInterface) *PrefixCompleter { return PcItem("", pc...) } func PcItem(name string, pc ...PrefixCompleterInterface) *PrefixCompleter { name += " " return &PrefixCompleter{ Name: []rune(name), Dynamic: false, Children: pc, } } func PcItemDynamic(callback DynamicCompleteFunc, pc ...PrefixCompleterInterface) *PrefixCompleter { return &PrefixCompleter{ Callback: callback, Dynamic: true, Children: pc, } } func (p *PrefixCompleter) Do(line []rune, pos int) (newLine [][]rune, offset int) { return doInternal(p, line, pos, line) } func Do(p PrefixCompleterInterface, line []rune, pos int) (newLine [][]rune, offset int) { return doInternal(p, line, pos, line) } func doInternal(p PrefixCompleterInterface, line []rune, pos int, origLine []rune) (newLine [][]rune, offset int) { line = runes.TrimSpaceLeft(line[:pos]) goNext := false var lineCompleter PrefixCompleterInterface for _, child := range p.GetChildren() { childNames := make([][]rune, 1) childDynamic, ok := child.(DynamicPrefixCompleterInterface) if ok && childDynamic.IsDynamic() { childNames = childDynamic.GetDynamicNames(origLine) } else { childNames[0] = child.GetName() } for _, childName := range childNames { if len(line) >= len(childName) { if runes.HasPrefix(line, childName) { if len(line) == len(childName) { newLine = append(newLine, []rune{' '}) } else { newLine = append(newLine, childName) } offset = len(childName) lineCompleter = child goNext = true } } else { if runes.HasPrefix(childName, line) { newLine = append(newLine, childName[len(line):]) offset = len(line) lineCompleter = child } } } } if len(newLine) != 1 { return } tmpLine := make([]rune, 0, len(line)) for i := offset; i < len(line); i++ { if line[i] == ' ' { continue } tmpLine = append(tmpLine, line[i:]...) return doInternal(lineCompleter, tmpLine, len(tmpLine), origLine) } if goNext { return doInternal(lineCompleter, nil, 0, origLine) } return } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/complete_segment.go ================================================ package readline type SegmentCompleter interface { // a // |- a1 // |--- a11 // |- a2 // b // input: // DoTree([], 0) [a, b] // DoTree([a], 1) [a] // DoTree([a, ], 0) [a1, a2] // DoTree([a, a], 1) [a1, a2] // DoTree([a, a1], 2) [a1] // DoTree([a, a1, ], 0) [a11] // DoTree([a, a1, a], 1) [a11] DoSegment([][]rune, int) [][]rune } type dumpSegmentCompleter struct { f func([][]rune, int) [][]rune } func (d *dumpSegmentCompleter) DoSegment(segment [][]rune, n int) [][]rune { return d.f(segment, n) } func SegmentFunc(f func([][]rune, int) [][]rune) AutoCompleter { return &SegmentComplete{&dumpSegmentCompleter{f}} } func SegmentAutoComplete(completer SegmentCompleter) *SegmentComplete { return &SegmentComplete{ SegmentCompleter: completer, } } type SegmentComplete struct { SegmentCompleter } func RetSegment(segments [][]rune, cands [][]rune, idx int) ([][]rune, int) { ret := make([][]rune, 0, len(cands)) lastSegment := segments[len(segments)-1] for _, cand := range cands { if !runes.HasPrefix(cand, lastSegment) { continue } ret = append(ret, cand[len(lastSegment):]) } return ret, idx } func SplitSegment(line []rune, pos int) ([][]rune, int) { segs := [][]rune{} lastIdx := -1 line = line[:pos] pos = 0 for idx, l := range line { if l == ' ' { pos = 0 segs = append(segs, line[lastIdx+1:idx]) lastIdx = idx } else { pos++ } } segs = append(segs, line[lastIdx+1:]) return segs, pos } func (c *SegmentComplete) Do(line []rune, pos int) (newLine [][]rune, offset int) { segment, idx := SplitSegment(line, pos) cands := c.DoSegment(segment, idx) newLine, offset = RetSegment(segment, cands, idx) for idx := range newLine { newLine[idx] = append(newLine[idx], ' ') } return newLine, offset } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/complete_segment_test.go ================================================ package readline import ( "fmt" "testing" "github.com/chzyer/test" ) func rs(s [][]rune) []string { ret := make([]string, len(s)) for idx, ss := range s { ret[idx] = string(ss) } return ret } func sr(s ...string) [][]rune { ret := make([][]rune, len(s)) for idx, ss := range s { ret[idx] = []rune(ss) } return ret } func TestRetSegment(t *testing.T) { defer test.New(t) // a // |- a1 // |--- a11 // |--- a12 // |- a2 // |--- a21 // b // add // adddomain ret := []struct { Segments [][]rune Cands [][]rune idx int Ret [][]rune pos int }{ {sr(""), sr("a", "b", "add", "adddomain"), 0, sr("a", "b", "add", "adddomain"), 0}, {sr("a"), sr("a", "add", "adddomain"), 1, sr("", "dd", "dddomain"), 1}, {sr("a", ""), sr("a1", "a2"), 0, sr("a1", "a2"), 0}, {sr("a", "a"), sr("a1", "a2"), 1, sr("1", "2"), 1}, {sr("a", "a1"), sr("a1"), 2, sr(""), 2}, {sr("add"), sr("add", "adddomain"), 2, sr("", "domain"), 2}, } for idx, r := range ret { ret, pos := RetSegment(r.Segments, r.Cands, r.idx) test.Equal(ret, r.Ret, fmt.Errorf("%v", idx)) test.Equal(pos, r.pos, fmt.Errorf("%v", idx)) } } func TestSplitSegment(t *testing.T) { defer test.New(t) // a // |- a1 // |--- a11 // |--- a12 // |- a2 // |--- a21 // b ret := []struct { Line string Pos int Segments [][]rune Idx int }{ {"", 0, sr(""), 0}, {"a", 1, sr("a"), 1}, {"a ", 2, sr("a", ""), 0}, {"a a", 3, sr("a", "a"), 1}, {"a a1", 4, sr("a", "a1"), 2}, {"a a1 ", 5, sr("a", "a1", ""), 0}, } for i, r := range ret { ret, idx := SplitSegment([]rune(r.Line), r.Pos) test.Equal(rs(ret), rs(r.Segments), fmt.Errorf("%v", i)) test.Equal(idx, r.Idx, fmt.Errorf("%v", i)) } } type Tree struct { Name string Children []Tree } func TestSegmentCompleter(t *testing.T) { defer test.New(t) tree := Tree{"", []Tree{ {"a", []Tree{ {"a1", []Tree{ {"a11", nil}, {"a12", nil}, }}, {"a2", []Tree{ {"a21", nil}, }}, }}, {"b", nil}, {"route", []Tree{ {"add", nil}, {"adddomain", nil}, }}, }} s := SegmentFunc(func(ret [][]rune, n int) [][]rune { tree := tree main: for level := 0; level < len(ret)-1; { name := string(ret[level]) for _, t := range tree.Children { if t.Name == name { tree = t level++ continue main } } } ret = make([][]rune, len(tree.Children)) for idx, r := range tree.Children { ret[idx] = []rune(r.Name) } return ret }) // a // |- a1 // |--- a11 // |--- a12 // |- a2 // |--- a21 // b ret := []struct { Line string Pos int Ret [][]rune Share int }{ {"", 0, sr("a", "b", "route"), 0}, {"a", 1, sr(""), 1}, {"a ", 2, sr("a1", "a2"), 0}, {"a a", 3, sr("1", "2"), 1}, {"a a1", 4, sr(""), 2}, {"a a1 ", 5, sr("a11", "a12"), 0}, {"a a1 a", 6, sr("11", "12"), 1}, {"a a1 a1", 7, sr("1", "2"), 2}, {"a a1 a11", 8, sr(""), 3}, {"route add", 9, sr("", "domain"), 3}, } for _, r := range ret { for idx, rr := range r.Ret { r.Ret[idx] = append(rr, ' ') } } for i, r := range ret { newLine, length := s.Do([]rune(r.Line), r.Pos) test.Equal(rs(newLine), rs(r.Ret), fmt.Errorf("%v", i)) test.Equal(length, r.Share, fmt.Errorf("%v", i)) } } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/doc/shortcut.md ================================================ ## Readline Shortcut `Meta`+`B` means press `Esc` and `n` separately. Users can change that in terminal simulator(i.e. iTerm2) to `Alt`+`B` Notice: `Meta`+`B` is equals with `Alt`+`B` in windows. * Shortcut in normal mode | Shortcut | Comment | | ------------------ | --------------------------------- | | `Ctrl`+`A` | Beginning of line | | `Ctrl`+`B` / `←` | Backward one character | | `Meta`+`B` | Backward one word | | `Ctrl`+`C` | Send io.EOF | | `Ctrl`+`D` | Delete one character | | `Meta`+`D` | Delete one word | | `Ctrl`+`E` | End of line | | `Ctrl`+`F` / `→` | Forward one character | | `Meta`+`F` | Forward one word | | `Ctrl`+`G` | Cancel | | `Ctrl`+`H` | Delete previous character | | `Ctrl`+`I` / `Tab` | Command line completion | | `Ctrl`+`J` | Line feed | | `Ctrl`+`K` | Cut text to the end of line | | `Ctrl`+`L` | Clear screen | | `Ctrl`+`M` | Same as Enter key | | `Ctrl`+`N` / `↓` | Next line (in history) | | `Ctrl`+`P` / `↑` | Prev line (in history) | | `Ctrl`+`R` | Search backwards in history | | `Ctrl`+`S` | Search forwards in history | | `Ctrl`+`T` | Transpose characters | | `Meta`+`T` | Transpose words (TODO) | | `Ctrl`+`U` | Cut text to the beginning of line | | `Ctrl`+`W` | Cut previous word | | `Backspace` | Delete previous character | | `Meta`+`Backspace` | Cut previous word | | `Enter` | Line feed | * Shortcut in Search Mode (`Ctrl`+`S` or `Ctrl`+`r` to enter this mode) | Shortcut | Comment | | ----------------------- | --------------------------------------- | | `Ctrl`+`S` | Search forwards in history | | `Ctrl`+`R` | Search backwards in history | | `Ctrl`+`C` / `Ctrl`+`G` | Exit Search Mode and revert the history | | `Backspace` | Delete previous character | | Other | Exit Search Mode | * Shortcut in Complete Select Mode (double `Tab` to enter this mode) | Shortcut | Comment | | ----------------------- | ---------------------------------------- | | `Ctrl`+`F` | Move Forward | | `Ctrl`+`B` | Move Backward | | `Ctrl`+`N` | Move to next line | | `Ctrl`+`P` | Move to previous line | | `Ctrl`+`A` | Move to the first candicate in current line | | `Ctrl`+`E` | Move to the last candicate in current line | | `Tab` / `Enter` | Use the word on cursor to complete | | `Ctrl`+`C` / `Ctrl`+`G` | Exit Complete Select Mode | | Other | Exit Complete Select Mode | ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/example/readline-demo/readline-demo.go ================================================ package main import ( "fmt" "io" "io/ioutil" "log" "strconv" "strings" "time" "github.com/chzyer/readline" ) func usage(w io.Writer) { io.WriteString(w, "commands:\n") io.WriteString(w, completer.Tree(" ")) } // Function constructor - constructs new function for listing given directory func listFiles(path string) func(string) []string { return func(line string) []string { names := make([]string, 0) files, _ := ioutil.ReadDir(path) for _, f := range files { names = append(names, f.Name()) } return names } } var completer = readline.NewPrefixCompleter( readline.PcItem("mode", readline.PcItem("vi"), readline.PcItem("emacs"), ), readline.PcItem("login"), readline.PcItem("say", readline.PcItemDynamic(listFiles("./"), readline.PcItem("with", readline.PcItem("following"), readline.PcItem("items"), ), ), readline.PcItem("hello"), readline.PcItem("bye"), ), readline.PcItem("setprompt"), readline.PcItem("setpassword"), readline.PcItem("bye"), readline.PcItem("help"), readline.PcItem("go", readline.PcItem("build", readline.PcItem("-o"), readline.PcItem("-v")), readline.PcItem("install", readline.PcItem("-v"), readline.PcItem("-vv"), readline.PcItem("-vvv"), ), readline.PcItem("test"), ), readline.PcItem("sleep"), ) func main() { l, err := readline.NewEx(&readline.Config{ Prompt: "\033[31m»\033[0m ", HistoryFile: "/tmp/readline.tmp", AutoComplete: completer, InterruptPrompt: "^C", EOFPrompt: "exit", HistorySearchFold: true, }) if err != nil { panic(err) } defer l.Close() setPasswordCfg := l.GenPasswordConfig() setPasswordCfg.SetListener(func(line []rune, pos int, key rune) (newLine []rune, newPos int, ok bool) { l.SetPrompt(fmt.Sprintf("Enter password(%v): ", len(line))) l.Refresh() return nil, 0, false }) log.SetOutput(l.Stderr()) for { line, err := l.Readline() if err == readline.ErrInterrupt { if len(line) == 0 { break } else { continue } } else if err == io.EOF { break } line = strings.TrimSpace(line) switch { case strings.HasPrefix(line, "mode "): switch line[5:] { case "vi": l.SetVimMode(true) case "emacs": l.SetVimMode(false) default: println("invalid mode:", line[5:]) } case line == "mode": if l.IsVimMode() { println("current mode: vim") } else { println("current mode: emacs") } case line == "login": pswd, err := l.ReadPassword("please enter your password: ") if err != nil { break } println("you enter:", strconv.Quote(string(pswd))) case line == "help": usage(l.Stderr()) case line == "setpassword": pswd, err := l.ReadPasswordWithConfig(setPasswordCfg) if err == nil { println("you set:", strconv.Quote(string(pswd))) } case strings.HasPrefix(line, "setprompt"): prompt := line[10:] if prompt == "" { log.Println("setprompt ") break } l.SetPrompt(prompt) case strings.HasPrefix(line, "say"): line := strings.TrimSpace(line[3:]) if len(line) == 0 { log.Println("say what?") break } go func() { for range time.Tick(time.Second) { log.Println(line) } }() case line == "bye": goto exit case line == "sleep": log.Println("sleep 4 second") time.Sleep(4 * time.Second) case line == "": default: log.Println("you said:", strconv.Quote(line)) } } exit: } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/example/readline-im/README.md ================================================ # readline-im ![readline-im](https://dl.dropboxusercontent.com/s/52hc7bo92g3pgi5/03F93B8D-9B4B-4D35-BBAA-22FBDAC7F299-26173-000164AA33980001.gif?dl=0) ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/example/readline-im/readline-im.go ================================================ package main import ( "fmt" "math/rand" "time" "github.com/chzyer/readline" ) import "log" func main() { rl, err := readline.NewEx(&readline.Config{ UniqueEditLine: true, }) if err != nil { panic(err) } defer rl.Close() rl.SetPrompt("username: ") username, err := rl.Readline() if err != nil { return } rl.ResetHistory() log.SetOutput(rl.Stderr()) fmt.Fprintln(rl, "Hi,", username+"! My name is Dave.") rl.SetPrompt(username + "> ") done := make(chan struct{}) go func() { rand.Seed(time.Now().Unix()) loop: for { select { case <-time.After(time.Duration(rand.Intn(20)) * 100 * time.Millisecond): case <-done: break loop } log.Println("Dave:", "hello") } log.Println("Dave:", "bye") done <- struct{}{} }() for { ln := rl.Line() if ln.CanContinue() { continue } else if ln.CanBreak() { break } log.Println(username+":", ln.Line) } rl.Clean() done <- struct{}{} <-done } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/example/readline-multiline/readline-multiline.go ================================================ package main import ( "strings" "github.com/chzyer/readline" ) func main() { rl, err := readline.NewEx(&readline.Config{ Prompt: "> ", HistoryFile: "/tmp/readline-multiline", DisableAutoSaveHistory: true, }) if err != nil { panic(err) } defer rl.Close() var cmds []string for { line, err := rl.Readline() if err != nil { break } line = strings.TrimSpace(line) if len(line) == 0 { continue } cmds = append(cmds, line) if !strings.HasSuffix(line, ";") { rl.SetPrompt(">>> ") continue } cmd := strings.Join(cmds, " ") cmds = cmds[:0] rl.SetPrompt("> ") rl.SaveHistory(cmd) println(cmd) } } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/example/readline-pass-strength/readline-pass-strength.go ================================================ // This is a small example using readline to read a password // and check it's strength while typing using the zxcvbn library. // Depending on the strength the prompt is colored nicely to indicate strength. // // This file is licensed under the WTFPL: // // DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE // Version 2, December 2004 // // Copyright (C) 2004 Sam Hocevar // // Everyone is permitted to copy and distribute verbatim or modified // copies of this license document, and changing it is allowed as long // as the name is changed. // // DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE // TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION // // 0. You just DO WHAT THE FUCK YOU WANT TO. package main import ( "fmt" "github.com/chzyer/readline" zxcvbn "github.com/nbutton23/zxcvbn-go" ) const ( Cyan = 36 Green = 32 Magenta = 35 Red = 31 Yellow = 33 BackgroundRed = 41 ) // Reset sequence var ColorResetEscape = "\033[0m" // ColorResetEscape translates a ANSI color number to a color escape. func ColorEscape(color int) string { return fmt.Sprintf("\033[0;%dm", color) } // Colorize the msg using ANSI color escapes func Colorize(msg string, color int) string { return ColorEscape(color) + msg + ColorResetEscape } func createStrengthPrompt(password []rune) string { symbol, color := "", Red strength := zxcvbn.PasswordStrength(string(password), nil) switch { case strength.Score <= 1: symbol = "✗" color = Red case strength.Score <= 2: symbol = "⚡" color = Magenta case strength.Score <= 3: symbol = "⚠" color = Yellow case strength.Score <= 4: symbol = "✔" color = Green } prompt := Colorize(symbol, color) if strength.Entropy > 0 { entropy := fmt.Sprintf(" %3.0f", strength.Entropy) prompt += Colorize(entropy, Cyan) } else { prompt += Colorize(" ENT", Cyan) } prompt += Colorize(" New Password: ", color) return prompt } func main() { rl, err := readline.New("") if err != nil { return } defer rl.Close() setPasswordCfg := rl.GenPasswordConfig() setPasswordCfg.SetListener(func(line []rune, pos int, key rune) (newLine []rune, newPos int, ok bool) { rl.SetPrompt(createStrengthPrompt(line)) rl.Refresh() return nil, 0, false }) pswd, err := rl.ReadPasswordWithConfig(setPasswordCfg) if err != nil { return } fmt.Println("Your password was:", string(pswd)) } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/example/readline-remote/readline-remote-client/client.go ================================================ package main import "github.com/chzyer/readline" func main() { if err := readline.DialRemote("tcp", ":12344"); err != nil { println(err.Error()) } } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/example/readline-remote/readline-remote-server/server.go ================================================ package main import ( "fmt" "github.com/chzyer/readline" ) func main() { cfg := &readline.Config{ Prompt: "readline-remote: ", } handleFunc := func(rl *readline.Instance) { for { line, err := rl.Readline() if err != nil { break } fmt.Fprintln(rl.Stdout(), "receive:"+line) } } err := readline.ListenRemote("tcp", ":12344", cfg, handleFunc) if err != nil { println(err.Error()) } } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/history.go ================================================ package readline import ( "bufio" "container/list" "fmt" "os" "strings" "sync" ) type hisItem struct { Source []rune Version int64 Tmp []rune } func (h *hisItem) Clean() { h.Source = nil h.Tmp = nil } type opHistory struct { cfg *Config history *list.List historyVer int64 current *list.Element fd *os.File fdLock sync.Mutex } func newOpHistory(cfg *Config) (o *opHistory) { o = &opHistory{ cfg: cfg, history: list.New(), } return o } func (o *opHistory) Reset() { o.history = list.New() o.current = nil } func (o *opHistory) IsHistoryClosed() bool { o.fdLock.Lock() defer o.fdLock.Unlock() return o.fd.Fd() == ^(uintptr(0)) } func (o *opHistory) Init() { if o.IsHistoryClosed() { o.initHistory() } } func (o *opHistory) initHistory() { if o.cfg.HistoryFile != "" { o.historyUpdatePath(o.cfg.HistoryFile) } } // only called by newOpHistory func (o *opHistory) historyUpdatePath(path string) { o.fdLock.Lock() defer o.fdLock.Unlock() f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666) if err != nil { return } o.fd = f r := bufio.NewReader(o.fd) total := 0 for ; ; total++ { line, err := r.ReadString('\n') if err != nil { break } // ignore the empty line line = strings.TrimSpace(line) if len(line) == 0 { continue } o.Push([]rune(line)) o.Compact() } if total > o.cfg.HistoryLimit { o.rewriteLocked() } o.historyVer++ o.Push(nil) return } func (o *opHistory) Compact() { for o.history.Len() > o.cfg.HistoryLimit && o.history.Len() > 0 { o.history.Remove(o.history.Front()) } } func (o *opHistory) Rewrite() { o.fdLock.Lock() defer o.fdLock.Unlock() o.rewriteLocked() } func (o *opHistory) rewriteLocked() { if o.cfg.HistoryFile == "" { return } tmpFile := o.cfg.HistoryFile + ".tmp" fd, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0666) if err != nil { return } buf := bufio.NewWriter(fd) for elem := o.history.Front(); elem != nil; elem = elem.Next() { buf.WriteString(string(elem.Value.(*hisItem).Source)) } buf.Flush() // replace history file if err = os.Rename(tmpFile, o.cfg.HistoryFile); err != nil { fd.Close() return } if o.fd != nil { o.fd.Close() } // fd is write only, just satisfy what we need. o.fd = fd } func (o *opHistory) Close() { o.fdLock.Lock() defer o.fdLock.Unlock() if o.fd != nil { o.fd.Close() } } func (o *opHistory) FindBck(isNewSearch bool, rs []rune, start int) (int, *list.Element) { for elem := o.current; elem != nil; elem = elem.Prev() { item := o.showItem(elem.Value) if isNewSearch { start += len(rs) } if elem == o.current { if len(item) >= start { item = item[:start] } } idx := runes.IndexAllBckEx(item, rs, o.cfg.HistorySearchFold) if idx < 0 { continue } return idx, elem } return -1, nil } func (o *opHistory) FindFwd(isNewSearch bool, rs []rune, start int) (int, *list.Element) { for elem := o.current; elem != nil; elem = elem.Next() { item := o.showItem(elem.Value) if isNewSearch { start -= len(rs) if start < 0 { start = 0 } } if elem == o.current { if len(item)-1 >= start { item = item[start:] } else { continue } } idx := runes.IndexAllEx(item, rs, o.cfg.HistorySearchFold) if idx < 0 { continue } if elem == o.current { idx += start } return idx, elem } return -1, nil } func (o *opHistory) showItem(obj interface{}) []rune { item := obj.(*hisItem) if item.Version == o.historyVer { return item.Tmp } return item.Source } func (o *opHistory) Prev() []rune { if o.current == nil { return nil } current := o.current.Prev() if current == nil { return nil } o.current = current return runes.Copy(o.showItem(current.Value)) } func (o *opHistory) Next() ([]rune, bool) { if o.current == nil { return nil, false } current := o.current.Next() if current == nil { return nil, false } o.current = current return runes.Copy(o.showItem(current.Value)), true } func (o *opHistory) debug() { Debug("-------") for item := o.history.Front(); item != nil; item = item.Next() { Debug(fmt.Sprintf("%+v", item.Value)) } } // save history func (o *opHistory) New(current []rune) (err error) { current = runes.Copy(current) // if just use last command without modify // just clean lastest history if back := o.history.Back(); back != nil { prev := back.Prev() if prev != nil { if runes.Equal(current, prev.Value.(*hisItem).Source) { o.current = o.history.Back() o.current.Value.(*hisItem).Clean() o.historyVer++ return nil } } } if len(current) == 0 { o.current = o.history.Back() if o.current != nil { o.current.Value.(*hisItem).Clean() o.historyVer++ return nil } } if o.current != o.history.Back() { // move history item to current command currentItem := o.current.Value.(*hisItem) // set current to last item o.current = o.history.Back() current = runes.Copy(currentItem.Tmp) } // err only can be a IO error, just report err = o.Update(current, true) // push a new one to commit current command o.historyVer++ o.Push(nil) return } func (o *opHistory) Revert() { o.historyVer++ o.current = o.history.Back() } func (o *opHistory) Update(s []rune, commit bool) (err error) { o.fdLock.Lock() defer o.fdLock.Unlock() s = runes.Copy(s) if o.current == nil { o.Push(s) o.Compact() return } r := o.current.Value.(*hisItem) r.Version = o.historyVer if commit { r.Source = s if o.fd != nil { // just report the error _, err = o.fd.Write([]byte(string(r.Source) + "\n")) } } else { r.Tmp = append(r.Tmp[:0], s...) } o.current.Value = r o.Compact() return } func (o *opHistory) Push(s []rune) { s = runes.Copy(s) elem := o.history.PushBack(&hisItem{Source: s}) o.current = elem } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/operation.go ================================================ package readline import ( "errors" "io" ) var ( ErrInterrupt = errors.New("Interrupt") ) type InterruptError struct { Line []rune } func (*InterruptError) Error() string { return "Interrupted" } type Operation struct { cfg *Config t *Terminal buf *RuneBuffer outchan chan []rune errchan chan error w io.Writer history *opHistory *opCompleter *opPassword *opVim } type wrapWriter struct { r *Operation t *Terminal target io.Writer } func (w *wrapWriter) Write(b []byte) (int, error) { if !w.t.IsReading() { return w.target.Write(b) } var ( n int err error ) w.r.buf.Refresh(func() { n, err = w.target.Write(b) }) if w.r.IsInCompleteMode() { w.r.CompleteRefresh() } return n, err } func NewOperation(t *Terminal, cfg *Config) *Operation { width := cfg.FuncGetWidth() op := &Operation{ t: t, buf: NewRuneBuffer(t, cfg.Prompt, cfg, width), outchan: make(chan []rune), errchan: make(chan error), } op.w = op.buf.w op.SetConfig(cfg) op.opVim = newVimMode(op) op.opCompleter = newOpCompleter(op.buf.w, op, width) op.opPassword = newOpPassword(op) op.cfg.FuncOnWidthChanged(func() { newWidth := cfg.FuncGetWidth() op.opCompleter.OnWidthChange(newWidth) op.buf.OnWidthChange(newWidth) }) go op.ioloop() return op } func (o *Operation) SetPrompt(s string) { o.buf.SetPrompt(s) } func (o *Operation) SetMaskRune(r rune) { o.buf.SetMask(r) } func (o *Operation) ioloop() { for { keepInCompleteMode := false r := o.t.ReadRune() if r == 0 { // io.EOF if o.buf.Len() == 0 { o.buf.Clean() select { case o.errchan <- io.EOF: } break } else { // if stdin got io.EOF and there is something left in buffer, // let's flush them by sending CharEnter. // And we will got io.EOF int next loop. r = CharEnter } } isUpdateHistory := true if o.IsInCompleteSelectMode() { keepInCompleteMode = o.HandleCompleteSelect(r) if keepInCompleteMode { continue } o.buf.Refresh(nil) switch r { case CharEnter, CharCtrlJ: o.history.Update(o.buf.Runes(), false) fallthrough case CharInterrupt: o.t.KickRead() fallthrough case CharBell: continue } } if o.IsEnableVimMode() { r = o.HandleVim(r, o.t.ReadRune) if r == 0 { continue } } switch r { case CharBell: if o.IsInCompleteMode() { o.ExitCompleteMode(true) o.buf.Refresh(nil) } case CharTab: if o.cfg.AutoComplete == nil { o.t.Bell() break } if o.OnComplete() { keepInCompleteMode = true } else { o.t.Bell() break } case CharCtrlU: o.buf.KillFront() case CharKill: o.buf.Kill() keepInCompleteMode = true case MetaForward: o.buf.MoveToNextWord() case CharTranspose: o.buf.Transpose() case MetaBackward: o.buf.MoveToPrevWord() case MetaDelete: o.buf.DeleteWord() case CharLineStart: o.buf.MoveToLineStart() case CharLineEnd: o.buf.MoveToLineEnd() case CharBackspace, CharCtrlH: if o.buf.Len() == 0 { o.t.Bell() break } o.buf.Backspace() if o.IsInCompleteMode() { o.OnComplete() } case CharCtrlZ: o.buf.Clean() o.t.SleepToResume() o.Refresh() case CharCtrlL: ClearScreen(o.w) o.Refresh() case MetaBackspace, CharCtrlW: o.buf.BackEscapeWord() case CharEnter, CharCtrlJ: o.buf.MoveToLineEnd() var data []rune if !o.cfg.UniqueEditLine { o.buf.WriteRune('\n') data = o.buf.Reset() data = data[:len(data)-1] // trim \n } else { o.buf.Clean() data = o.buf.Reset() } o.outchan <- data if !o.cfg.DisableAutoSaveHistory { // ignore IO error _ = o.history.New(data) } else { isUpdateHistory = false } case CharBackward: o.buf.MoveBackward() case CharForward: o.buf.MoveForward() case CharPrev: buf := o.history.Prev() if buf != nil { o.buf.Set(buf) } else { o.t.Bell() } case CharNext: buf, ok := o.history.Next() if ok { o.buf.Set(buf) } else { o.t.Bell() } case CharDelete: if o.buf.Len() > 0 || !o.IsNormalMode() { o.t.KickRead() if !o.buf.Delete() { o.t.Bell() } break } // treat as EOF if !o.cfg.UniqueEditLine { o.buf.WriteString(o.cfg.EOFPrompt + "\n") } o.buf.Reset() isUpdateHistory = false o.history.Revert() o.errchan <- io.EOF if o.cfg.UniqueEditLine { o.buf.Clean() } case CharInterrupt: if o.IsInCompleteMode() { o.t.KickRead() o.ExitCompleteMode(true) o.buf.Refresh(nil) break } o.buf.MoveToLineEnd() o.buf.Refresh(nil) hint := o.cfg.InterruptPrompt + "\n" if !o.cfg.UniqueEditLine { o.buf.WriteString(hint) } remain := o.buf.Reset() if !o.cfg.UniqueEditLine { remain = remain[:len(remain)-len([]rune(hint))] } isUpdateHistory = false o.history.Revert() o.errchan <- &InterruptError{remain} default: o.buf.WriteRune(r) if o.IsInCompleteMode() { o.OnComplete() keepInCompleteMode = true } } if o.cfg.Listener != nil { newLine, newPos, ok := o.cfg.Listener.OnChange(o.buf.Runes(), o.buf.Pos(), r) if ok { o.buf.SetWithIdx(newPos, newLine) } } if o.IsInCompleteMode() { if !keepInCompleteMode { o.ExitCompleteMode(false) o.Refresh() } else { o.buf.Refresh(nil) o.CompleteRefresh() } } if isUpdateHistory { // it will cause null history o.history.Update(o.buf.Runes(), false) } } } func (o *Operation) Stderr() io.Writer { return &wrapWriter{target: o.cfg.Stderr, r: o, t: o.t} } func (o *Operation) Stdout() io.Writer { return &wrapWriter{target: o.cfg.Stdout, r: o, t: o.t} } func (o *Operation) String() (string, error) { r, err := o.Runes() return string(r), err } func (o *Operation) Runes() ([]rune, error) { o.t.EnterRawMode() defer o.t.ExitRawMode() if o.cfg.Listener != nil { o.cfg.Listener.OnChange(nil, 0, 0) } o.buf.Refresh(nil) // print prompt o.t.KickRead() select { case r := <-o.outchan: return r, nil case err := <-o.errchan: if e, ok := err.(*InterruptError); ok { return e.Line, ErrInterrupt } return nil, err } } func (o *Operation) PasswordEx(prompt string, l Listener) ([]byte, error) { cfg := o.GenPasswordConfig() cfg.Prompt = prompt cfg.Listener = l return o.PasswordWithConfig(cfg) } func (o *Operation) GenPasswordConfig() *Config { return o.opPassword.PasswordConfig() } func (o *Operation) PasswordWithConfig(cfg *Config) ([]byte, error) { if err := o.opPassword.EnterPasswordMode(cfg); err != nil { return nil, err } defer o.opPassword.ExitPasswordMode() return o.Slice() } func (o *Operation) Password(prompt string) ([]byte, error) { return o.PasswordEx(prompt, nil) } func (o *Operation) SetTitle(t string) { o.w.Write([]byte("\033[2;" + t + "\007")) } func (o *Operation) Slice() ([]byte, error) { r, err := o.Runes() if err != nil { return nil, err } return []byte(string(r)), nil } func (o *Operation) Close() { o.history.Close() } func (o *Operation) SetHistoryPath(path string) { if o.history != nil { o.history.Close() } o.cfg.HistoryFile = path o.history = newOpHistory(o.cfg) } func (o *Operation) IsNormalMode() bool { return !o.IsInCompleteMode() } func (op *Operation) SetConfig(cfg *Config) (*Config, error) { if op.cfg == cfg { return op.cfg, nil } if err := cfg.Init(); err != nil { return op.cfg, err } old := op.cfg op.cfg = cfg op.SetPrompt(cfg.Prompt) op.SetMaskRune(cfg.MaskRune) op.buf.SetConfig(cfg) width := op.cfg.FuncGetWidth() if cfg.opHistory == nil { op.SetHistoryPath(cfg.HistoryFile) cfg.opHistory = op.history } op.history = cfg.opHistory // SetHistoryPath will close opHistory which already exists // so if we use it next time, we need to reopen it by `InitHistory()` op.history.Init() if op.cfg.AutoComplete != nil { op.opCompleter = newOpCompleter(op.buf.w, op, width) } return old, nil } func (o *Operation) ResetHistory() { o.history.Reset() } // if err is not nil, it just mean it fail to write to file // other things goes fine. func (o *Operation) SaveHistory(content string) error { return o.history.New([]rune(content)) } func (o *Operation) Refresh() { if o.t.IsReading() { o.buf.Refresh(nil) } } func (o *Operation) Clean() { o.buf.Clean() } func FuncListener(f func(line []rune, pos int, key rune) (newLine []rune, newPos int, ok bool)) Listener { return &DumpListener{f: f} } type DumpListener struct { f func(line []rune, pos int, key rune) (newLine []rune, newPos int, ok bool) } func (d *DumpListener) OnChange(line []rune, pos int, key rune) (newLine []rune, newPos int, ok bool) { return d.f(line, pos, key) } type Listener interface { OnChange(line []rune, pos int, key rune) (newLine []rune, newPos int, ok bool) } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/password.go ================================================ package readline type opPassword struct { o *Operation backupCfg *Config } func newOpPassword(o *Operation) *opPassword { return &opPassword{o: o} } func (o *opPassword) ExitPasswordMode() { o.o.SetConfig(o.backupCfg) o.backupCfg = nil } func (o *opPassword) EnterPasswordMode(cfg *Config) (err error) { o.backupCfg, err = o.o.SetConfig(cfg) return } func (o *opPassword) PasswordConfig() *Config { return &Config{ EnableMask: true, InterruptPrompt: "\n", EOFPrompt: "\n", HistoryLimit: -1, Stdout: o.o.cfg.Stdout, Stderr: o.o.cfg.Stderr, } } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/rawreader_windows.go ================================================ // +build windows package readline import "unsafe" const ( VK_CANCEL = 0x03 VK_BACK = 0x08 VK_TAB = 0x09 VK_RETURN = 0x0D VK_SHIFT = 0x10 VK_CONTROL = 0x11 VK_MENU = 0x12 VK_ESCAPE = 0x1B VK_LEFT = 0x25 VK_UP = 0x26 VK_RIGHT = 0x27 VK_DOWN = 0x28 VK_DELETE = 0x2E VK_LSHIFT = 0xA0 VK_RSHIFT = 0xA1 VK_LCONTROL = 0xA2 VK_RCONTROL = 0xA3 ) // RawReader translate input record to ANSI escape sequence. // To provides same behavior as unix terminal. type RawReader struct { ctrlKey bool altKey bool } func NewRawReader() *RawReader { r := new(RawReader) return r } // only process one action in one read func (r *RawReader) Read(buf []byte) (int, error) { ir := new(_INPUT_RECORD) var read int var err error next: err = kernel.ReadConsoleInputW(stdin, uintptr(unsafe.Pointer(ir)), 1, uintptr(unsafe.Pointer(&read)), ) if err != nil { return 0, err } if ir.EventType != EVENT_KEY { goto next } ker := (*_KEY_EVENT_RECORD)(unsafe.Pointer(&ir.Event[0])) if ker.bKeyDown == 0 { // keyup if r.ctrlKey || r.altKey { switch ker.wVirtualKeyCode { case VK_RCONTROL, VK_LCONTROL: r.ctrlKey = false case VK_MENU: //alt r.altKey = false } } goto next } if ker.unicodeChar == 0 { var target rune switch ker.wVirtualKeyCode { case VK_RCONTROL, VK_LCONTROL: r.ctrlKey = true case VK_MENU: //alt r.altKey = true case VK_LEFT: target = CharBackward case VK_RIGHT: target = CharForward case VK_UP: target = CharPrev case VK_DOWN: target = CharNext } if target != 0 { return r.write(buf, target) } goto next } char := rune(ker.unicodeChar) if r.ctrlKey { switch char { case 'A': char = CharLineStart case 'E': char = CharLineEnd case 'R': char = CharBckSearch case 'S': char = CharFwdSearch } } else if r.altKey { switch char { case VK_BACK: char = CharBackspace } return r.writeEsc(buf, char) } return r.write(buf, char) } func (r *RawReader) writeEsc(b []byte, char rune) (int, error) { b[0] = '\033' n := copy(b[1:], []byte(string(char))) return n + 1, nil } func (r *RawReader) write(b []byte, char rune) (int, error) { n := copy(b, []byte(string(char))) return n, nil } func (r *RawReader) Close() error { return nil } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/readline.go ================================================ // Readline is a pure go implementation for GNU-Readline kind library. // // example: // rl, err := readline.New("> ") // if err != nil { // panic(err) // } // defer rl.Close() // // for { // line, err := rl.Readline() // if err != nil { // io.EOF // break // } // println(line) // } // package readline import "io" type Instance struct { Config *Config Terminal *Terminal Operation *Operation } type Config struct { // prompt supports ANSI escape sequence, so we can color some characters even in windows Prompt string // readline will persist historys to file where HistoryFile specified HistoryFile string // specify the max length of historys, it's 500 by default, set it to -1 to disable history HistoryLimit int DisableAutoSaveHistory bool // enable case-insensitive history searching HistorySearchFold bool // AutoCompleter will called once user press TAB AutoComplete AutoCompleter // Any key press will pass to Listener // NOTE: Listener will be triggered by (nil, 0, 0) immediately Listener Listener // If VimMode is true, readline will in vim.insert mode by default VimMode bool InterruptPrompt string EOFPrompt string FuncGetWidth func() int Stdin io.Reader Stdout io.Writer Stderr io.Writer EnableMask bool MaskRune rune // erase the editing line after user submited it // it use in IM usually. UniqueEditLine bool // force use interactive even stdout is not a tty FuncIsTerminal func() bool FuncMakeRaw func() error FuncExitRaw func() error FuncOnWidthChanged func(func()) ForceUseInteractive bool // private fields inited bool opHistory *opHistory } func (c *Config) useInteractive() bool { if c.ForceUseInteractive { return true } return c.FuncIsTerminal() } func (c *Config) Init() error { if c.inited { return nil } c.inited = true if c.Stdin == nil { c.Stdin = NewCancelableStdin(Stdin) } if c.Stdout == nil { c.Stdout = Stdout } if c.Stderr == nil { c.Stderr = Stderr } if c.HistoryLimit == 0 { c.HistoryLimit = 500 } if c.InterruptPrompt == "" { c.InterruptPrompt = "^C" } else if c.InterruptPrompt == "\n" { c.InterruptPrompt = "" } if c.EOFPrompt == "" { c.EOFPrompt = "^D" } else if c.EOFPrompt == "\n" { c.EOFPrompt = "" } if c.AutoComplete == nil { c.AutoComplete = &TabCompleter{} } if c.FuncGetWidth == nil { c.FuncGetWidth = GetScreenWidth } if c.FuncIsTerminal == nil { c.FuncIsTerminal = DefaultIsTerminal } rm := new(RawMode) if c.FuncMakeRaw == nil { c.FuncMakeRaw = rm.Enter } if c.FuncExitRaw == nil { c.FuncExitRaw = rm.Exit } if c.FuncOnWidthChanged == nil { c.FuncOnWidthChanged = DefaultOnWidthChanged } return nil } func (c Config) Clone() *Config { c.opHistory = nil return &c } func (c *Config) SetListener(f func(line []rune, pos int, key rune) (newLine []rune, newPos int, ok bool)) { c.Listener = FuncListener(f) } func NewEx(cfg *Config) (*Instance, error) { t, err := NewTerminal(cfg) if err != nil { return nil, err } rl := t.Readline() return &Instance{ Config: cfg, Terminal: t, Operation: rl, }, nil } func New(prompt string) (*Instance, error) { return NewEx(&Config{Prompt: prompt}) } func (i *Instance) ResetHistory() { i.Operation.ResetHistory() } func (i *Instance) SetPrompt(s string) { i.Operation.SetPrompt(s) } func (i *Instance) SetMaskRune(r rune) { i.Operation.SetMaskRune(r) } // change history persistence in runtime func (i *Instance) SetHistoryPath(p string) { i.Operation.SetHistoryPath(p) } // readline will refresh automatic when write through Stdout() func (i *Instance) Stdout() io.Writer { return i.Operation.Stdout() } // readline will refresh automatic when write through Stdout() func (i *Instance) Stderr() io.Writer { return i.Operation.Stderr() } // switch VimMode in runtime func (i *Instance) SetVimMode(on bool) { i.Operation.SetVimMode(on) } func (i *Instance) IsVimMode() bool { return i.Operation.IsEnableVimMode() } func (i *Instance) GenPasswordConfig() *Config { return i.Operation.GenPasswordConfig() } // we can generate a config by `i.GenPasswordConfig()` func (i *Instance) ReadPasswordWithConfig(cfg *Config) ([]byte, error) { return i.Operation.PasswordWithConfig(cfg) } func (i *Instance) ReadPasswordEx(prompt string, l Listener) ([]byte, error) { return i.Operation.PasswordEx(prompt, l) } func (i *Instance) ReadPassword(prompt string) ([]byte, error) { return i.Operation.Password(prompt) } type Result struct { Line string Error error } func (l *Result) CanContinue() bool { return len(l.Line) != 0 && l.Error == ErrInterrupt } func (l *Result) CanBreak() bool { return !l.CanContinue() && l.Error != nil } func (i *Instance) Line() *Result { ret, err := i.Readline() return &Result{ret, err} } // err is one of (nil, io.EOF, readline.ErrInterrupt) func (i *Instance) Readline() (string, error) { return i.Operation.String() } func (i *Instance) SaveHistory(content string) error { return i.Operation.SaveHistory(content) } // same as readline func (i *Instance) ReadSlice() ([]byte, error) { return i.Operation.Slice() } // we must make sure that call Close() before process exit. func (i *Instance) Close() error { if err := i.Terminal.Close(); err != nil { return err } i.Operation.Close() return nil } func (i *Instance) Clean() { i.Operation.Clean() } func (i *Instance) Write(b []byte) (int, error) { return i.Stdout().Write(b) } func (i *Instance) SetConfig(cfg *Config) *Config { if i.Config == cfg { return cfg } old := i.Config i.Config = cfg i.Operation.SetConfig(cfg) i.Terminal.SetConfig(cfg) return old } func (i *Instance) Refresh() { i.Operation.Refresh() } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/readline_test.go ================================================ package readline import ( "testing" "time" ) func TestRace(t *testing.T) { rl, err := NewEx(&Config{}) if err != nil { t.Fatal(err) return } go func() { for range time.Tick(time.Millisecond) { rl.SetPrompt("hello") } }() go func() { time.Sleep(100 * time.Millisecond) rl.Close() }() rl.Readline() } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/remote.go ================================================ package readline import ( "bufio" "bytes" "encoding/binary" "fmt" "io" "net" "os" "sync" "sync/atomic" ) type MsgType int16 const ( T_DATA = MsgType(iota) T_WIDTH T_WIDTH_REPORT T_ISTTY_REPORT T_RAW T_ERAW // exit raw T_EOF ) type RemoteSvr struct { eof int32 closed int32 width int32 reciveChan chan struct{} writeChan chan *writeCtx conn net.Conn isTerminal bool funcWidthChan func() stopChan chan struct{} dataBufM sync.Mutex dataBuf bytes.Buffer } type writeReply struct { n int err error } type writeCtx struct { msg *Message reply chan *writeReply } func newWriteCtx(msg *Message) *writeCtx { return &writeCtx{ msg: msg, reply: make(chan *writeReply), } } func NewRemoteSvr(conn net.Conn) (*RemoteSvr, error) { rs := &RemoteSvr{ width: -1, conn: conn, writeChan: make(chan *writeCtx), reciveChan: make(chan struct{}), stopChan: make(chan struct{}), } buf := bufio.NewReader(rs.conn) if err := rs.init(buf); err != nil { return nil, err } go rs.readLoop(buf) go rs.writeLoop() return rs, nil } func (r *RemoteSvr) init(buf *bufio.Reader) error { m, err := ReadMessage(buf) if err != nil { return err } // receive isTerminal if m.Type != T_ISTTY_REPORT { return fmt.Errorf("unexpected init message") } r.GotIsTerminal(m.Data) // receive width m, err = ReadMessage(buf) if err != nil { return err } if m.Type != T_WIDTH_REPORT { return fmt.Errorf("unexpected init message") } r.GotReportWidth(m.Data) return nil } func (r *RemoteSvr) HandleConfig(cfg *Config) { cfg.Stderr = r cfg.Stdout = r cfg.Stdin = r cfg.FuncExitRaw = r.ExitRawMode cfg.FuncIsTerminal = r.IsTerminal cfg.FuncMakeRaw = r.EnterRawMode cfg.FuncExitRaw = r.ExitRawMode cfg.FuncGetWidth = r.GetWidth cfg.FuncOnWidthChanged = func(f func()) { r.funcWidthChan = f } } func (r *RemoteSvr) IsTerminal() bool { return r.isTerminal } func (r *RemoteSvr) checkEOF() error { if atomic.LoadInt32(&r.eof) == 1 { return io.EOF } return nil } func (r *RemoteSvr) Read(b []byte) (int, error) { r.dataBufM.Lock() n, err := r.dataBuf.Read(b) r.dataBufM.Unlock() if n == 0 { if err := r.checkEOF(); err != nil { return 0, err } } if n == 0 && err == io.EOF { <-r.reciveChan r.dataBufM.Lock() n, err = r.dataBuf.Read(b) r.dataBufM.Unlock() } if n == 0 { if err := r.checkEOF(); err != nil { return 0, err } } return n, err } func (r *RemoteSvr) writeMsg(m *Message) error { ctx := newWriteCtx(m) r.writeChan <- ctx reply := <-ctx.reply return reply.err } func (r *RemoteSvr) Write(b []byte) (int, error) { ctx := newWriteCtx(NewMessage(T_DATA, b)) r.writeChan <- ctx reply := <-ctx.reply return reply.n, reply.err } func (r *RemoteSvr) EnterRawMode() error { return r.writeMsg(NewMessage(T_RAW, nil)) } func (r *RemoteSvr) ExitRawMode() error { return r.writeMsg(NewMessage(T_ERAW, nil)) } func (r *RemoteSvr) writeLoop() { defer r.Close() loop: for { select { case ctx, ok := <-r.writeChan: if !ok { break } n, err := ctx.msg.WriteTo(r.conn) ctx.reply <- &writeReply{n, err} case <-r.stopChan: break loop } } } func (r *RemoteSvr) Close() { if atomic.CompareAndSwapInt32(&r.closed, 0, 1) { close(r.stopChan) r.conn.Close() } } func (r *RemoteSvr) readLoop(buf *bufio.Reader) { defer r.Close() for { m, err := ReadMessage(buf) if err != nil { break } switch m.Type { case T_EOF: atomic.StoreInt32(&r.eof, 1) select { case r.reciveChan <- struct{}{}: default: } case T_DATA: r.dataBufM.Lock() r.dataBuf.Write(m.Data) r.dataBufM.Unlock() select { case r.reciveChan <- struct{}{}: default: } case T_WIDTH_REPORT: r.GotReportWidth(m.Data) case T_ISTTY_REPORT: r.GotIsTerminal(m.Data) } } } func (r *RemoteSvr) GotIsTerminal(data []byte) { if binary.BigEndian.Uint16(data) == 0 { r.isTerminal = false } else { r.isTerminal = true } } func (r *RemoteSvr) GotReportWidth(data []byte) { atomic.StoreInt32(&r.width, int32(binary.BigEndian.Uint16(data))) if r.funcWidthChan != nil { r.funcWidthChan() } } func (r *RemoteSvr) GetWidth() int { return int(atomic.LoadInt32(&r.width)) } // ----------------------------------------------------------------------------- type Message struct { Type MsgType Data []byte } func ReadMessage(r io.Reader) (*Message, error) { m := new(Message) var length int32 if err := binary.Read(r, binary.BigEndian, &length); err != nil { return nil, err } if err := binary.Read(r, binary.BigEndian, &m.Type); err != nil { return nil, err } m.Data = make([]byte, int(length)-2) if _, err := io.ReadFull(r, m.Data); err != nil { return nil, err } return m, nil } func NewMessage(t MsgType, data []byte) *Message { return &Message{t, data} } func (m *Message) WriteTo(w io.Writer) (int, error) { buf := bytes.NewBuffer(make([]byte, 0, len(m.Data)+2+4)) binary.Write(buf, binary.BigEndian, int32(len(m.Data)+2)) binary.Write(buf, binary.BigEndian, m.Type) buf.Write(m.Data) n, err := buf.WriteTo(w) return int(n), err } // ----------------------------------------------------------------------------- type RemoteCli struct { conn net.Conn raw RawMode receiveChan chan struct{} inited int32 isTerminal *bool data bytes.Buffer dataM sync.Mutex } func NewRemoteCli(conn net.Conn) (*RemoteCli, error) { r := &RemoteCli{ conn: conn, receiveChan: make(chan struct{}), } return r, nil } func (r *RemoteCli) MarkIsTerminal(is bool) { r.isTerminal = &is } func (r *RemoteCli) init() error { if !atomic.CompareAndSwapInt32(&r.inited, 0, 1) { return nil } if err := r.reportIsTerminal(); err != nil { return err } if err := r.reportWidth(); err != nil { return err } // register sig for width changed DefaultOnWidthChanged(func() { r.reportWidth() }) return nil } func (r *RemoteCli) writeMsg(m *Message) error { r.dataM.Lock() _, err := m.WriteTo(r.conn) r.dataM.Unlock() return err } func (r *RemoteCli) Write(b []byte) (int, error) { m := NewMessage(T_DATA, b) r.dataM.Lock() _, err := m.WriteTo(r.conn) r.dataM.Unlock() return len(b), err } func (r *RemoteCli) reportWidth() error { screenWidth := GetScreenWidth() data := make([]byte, 2) binary.BigEndian.PutUint16(data, uint16(screenWidth)) msg := NewMessage(T_WIDTH_REPORT, data) if err := r.writeMsg(msg); err != nil { return err } return nil } func (r *RemoteCli) reportIsTerminal() error { var isTerminal bool if r.isTerminal != nil { isTerminal = *r.isTerminal } else { isTerminal = DefaultIsTerminal() } data := make([]byte, 2) if isTerminal { binary.BigEndian.PutUint16(data, 1) } else { binary.BigEndian.PutUint16(data, 0) } msg := NewMessage(T_ISTTY_REPORT, data) if err := r.writeMsg(msg); err != nil { return err } return nil } func (r *RemoteCli) readLoop() { buf := bufio.NewReader(r.conn) for { msg, err := ReadMessage(buf) if err != nil { break } switch msg.Type { case T_ERAW: r.raw.Exit() case T_RAW: r.raw.Enter() case T_DATA: os.Stdout.Write(msg.Data) } } } func (r *RemoteCli) ServeBy(source io.Reader) error { if err := r.init(); err != nil { return err } go func() { defer r.Close() for { n, _ := io.Copy(r, source) if n == 0 { break } } }() defer r.raw.Exit() r.readLoop() return nil } func (r *RemoteCli) Close() { r.writeMsg(NewMessage(T_EOF, nil)) } func (r *RemoteCli) Serve() error { return r.ServeBy(os.Stdin) } func ListenRemote(n, addr string, cfg *Config, h func(*Instance), onListen ...func(net.Listener) error) error { ln, err := net.Listen(n, addr) if err != nil { return err } if len(onListen) > 0 { if err := onListen[0](ln); err != nil { return err } } for { conn, err := ln.Accept() if err != nil { break } go func() { defer conn.Close() rl, err := HandleConn(*cfg, conn) if err != nil { return } h(rl) }() } return nil } func HandleConn(cfg Config, conn net.Conn) (*Instance, error) { r, err := NewRemoteSvr(conn) if err != nil { return nil, err } r.HandleConfig(&cfg) rl, err := NewEx(&cfg) if err != nil { return nil, err } return rl, nil } func DialRemote(n, addr string) error { conn, err := net.Dial(n, addr) if err != nil { return err } defer conn.Close() cli, err := NewRemoteCli(conn) if err != nil { return err } return cli.Serve() } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/runebuf.go ================================================ package readline import ( "bufio" "bytes" "io" "strings" "sync" ) type runeBufferBck struct { buf []rune idx int } type RuneBuffer struct { buf []rune idx int prompt []rune w io.Writer hadClean bool interactive bool cfg *Config width int bck *runeBufferBck offset string sync.Mutex } func (r *RuneBuffer) OnWidthChange(newWidth int) { r.Lock() r.width = newWidth r.Unlock() } func (r *RuneBuffer) Backup() { r.Lock() r.bck = &runeBufferBck{r.buf, r.idx} r.Unlock() } func (r *RuneBuffer) Restore() { r.Refresh(func() { if r.bck == nil { return } r.buf = r.bck.buf r.idx = r.bck.idx }) } func NewRuneBuffer(w io.Writer, prompt string, cfg *Config, width int) *RuneBuffer { rb := &RuneBuffer{ w: w, interactive: cfg.useInteractive(), cfg: cfg, width: width, } rb.SetPrompt(prompt) return rb } func (r *RuneBuffer) SetConfig(cfg *Config) { r.Lock() r.cfg = cfg r.interactive = cfg.useInteractive() r.Unlock() } func (r *RuneBuffer) SetMask(m rune) { r.Lock() r.cfg.MaskRune = m r.Unlock() } func (r *RuneBuffer) CurrentWidth(x int) int { r.Lock() defer r.Unlock() return runes.WidthAll(r.buf[:x]) } func (r *RuneBuffer) PromptLen() int { r.Lock() width := r.promptLen() r.Unlock() return width } func (r *RuneBuffer) promptLen() int { return runes.WidthAll(runes.ColorFilter(r.prompt)) } func (r *RuneBuffer) RuneSlice(i int) []rune { r.Lock() defer r.Unlock() if i > 0 { rs := make([]rune, i) copy(rs, r.buf[r.idx:r.idx+i]) return rs } rs := make([]rune, -i) copy(rs, r.buf[r.idx+i:r.idx]) return rs } func (r *RuneBuffer) Runes() []rune { r.Lock() newr := make([]rune, len(r.buf)) copy(newr, r.buf) r.Unlock() return newr } func (r *RuneBuffer) Pos() int { r.Lock() defer r.Unlock() return r.idx } func (r *RuneBuffer) Len() int { r.Lock() defer r.Unlock() return len(r.buf) } func (r *RuneBuffer) MoveToLineStart() { r.Refresh(func() { if r.idx == 0 { return } r.idx = 0 }) } func (r *RuneBuffer) MoveBackward() { r.Refresh(func() { if r.idx == 0 { return } r.idx-- }) } func (r *RuneBuffer) WriteString(s string) { r.WriteRunes([]rune(s)) } func (r *RuneBuffer) WriteRune(s rune) { r.WriteRunes([]rune{s}) } func (r *RuneBuffer) WriteRunes(s []rune) { r.Refresh(func() { tail := append(s, r.buf[r.idx:]...) r.buf = append(r.buf[:r.idx], tail...) r.idx += len(s) }) } func (r *RuneBuffer) MoveForward() { r.Refresh(func() { if r.idx == len(r.buf) { return } r.idx++ }) } func (r *RuneBuffer) IsCursorInEnd() bool { r.Lock() defer r.Unlock() return r.idx == len(r.buf) } func (r *RuneBuffer) Replace(ch rune) { r.Refresh(func() { r.buf[r.idx] = ch }) } func (r *RuneBuffer) Erase() { r.Refresh(func() { r.idx = 0 r.buf = r.buf[:0] }) } func (r *RuneBuffer) Delete() (success bool) { r.Refresh(func() { if r.idx == len(r.buf) { return } r.buf = append(r.buf[:r.idx], r.buf[r.idx+1:]...) success = true }) return } func (r *RuneBuffer) DeleteWord() { if r.idx == len(r.buf) { return } init := r.idx for init < len(r.buf) && IsWordBreak(r.buf[init]) { init++ } for i := init + 1; i < len(r.buf); i++ { if !IsWordBreak(r.buf[i]) && IsWordBreak(r.buf[i-1]) { r.Refresh(func() { r.buf = append(r.buf[:r.idx], r.buf[i-1:]...) }) return } } r.Kill() } func (r *RuneBuffer) MoveToPrevWord() (success bool) { r.Refresh(func() { if r.idx == 0 { return } for i := r.idx - 1; i > 0; i-- { if !IsWordBreak(r.buf[i]) && IsWordBreak(r.buf[i-1]) { r.idx = i success = true return } } r.idx = 0 success = true }) return } func (r *RuneBuffer) KillFront() { r.Refresh(func() { if r.idx == 0 { return } length := len(r.buf) - r.idx copy(r.buf[:length], r.buf[r.idx:]) r.idx = 0 r.buf = r.buf[:length] }) } func (r *RuneBuffer) Kill() { r.Refresh(func() { r.buf = r.buf[:r.idx] }) } func (r *RuneBuffer) Transpose() { r.Refresh(func() { if len(r.buf) == 1 { r.idx++ } if len(r.buf) < 2 { return } if r.idx == 0 { r.idx = 1 } else if r.idx >= len(r.buf) { r.idx = len(r.buf) - 1 } r.buf[r.idx], r.buf[r.idx-1] = r.buf[r.idx-1], r.buf[r.idx] r.idx++ }) } func (r *RuneBuffer) MoveToNextWord() { r.Refresh(func() { for i := r.idx + 1; i < len(r.buf); i++ { if !IsWordBreak(r.buf[i]) && IsWordBreak(r.buf[i-1]) { r.idx = i return } } r.idx = len(r.buf) }) } func (r *RuneBuffer) MoveToEndWord() { r.Refresh(func() { // already at the end, so do nothing if r.idx == len(r.buf) { return } // if we are at the end of a word already, go to next if !IsWordBreak(r.buf[r.idx]) && IsWordBreak(r.buf[r.idx+1]) { r.idx++ } // keep going until at the end of a word for i := r.idx + 1; i < len(r.buf); i++ { if IsWordBreak(r.buf[i]) && !IsWordBreak(r.buf[i-1]) { r.idx = i - 1 return } } r.idx = len(r.buf) }) } func (r *RuneBuffer) BackEscapeWord() { r.Refresh(func() { if r.idx == 0 { return } for i := r.idx - 1; i > 0; i-- { if !IsWordBreak(r.buf[i]) && IsWordBreak(r.buf[i-1]) { r.buf = append(r.buf[:i], r.buf[r.idx:]...) r.idx = i return } } r.buf = r.buf[:0] r.idx = 0 }) } func (r *RuneBuffer) Backspace() { r.Refresh(func() { if r.idx == 0 { return } r.idx-- r.buf = append(r.buf[:r.idx], r.buf[r.idx+1:]...) }) } func (r *RuneBuffer) MoveToLineEnd() { r.Refresh(func() { if r.idx == len(r.buf) { return } r.idx = len(r.buf) }) } func (r *RuneBuffer) LineCount(width int) int { if width == -1 { width = r.width } return LineCount(width, runes.WidthAll(r.buf)+r.PromptLen()) } func (r *RuneBuffer) MoveTo(ch rune, prevChar, reverse bool) (success bool) { r.Refresh(func() { if reverse { for i := r.idx - 1; i >= 0; i-- { if r.buf[i] == ch { r.idx = i if prevChar { r.idx++ } success = true return } } return } for i := r.idx + 1; i < len(r.buf); i++ { if r.buf[i] == ch { r.idx = i if prevChar { r.idx-- } success = true return } } }) return } func (r *RuneBuffer) isInLineEdge() bool { if isWindows { return false } sp := r.getSplitByLine(r.buf) return len(sp[len(sp)-1]) == 0 } func (r *RuneBuffer) getSplitByLine(rs []rune) []string { return SplitByLine(r.promptLen(), r.width, rs) } func (r *RuneBuffer) IdxLine(width int) int { r.Lock() defer r.Unlock() return r.idxLine(width) } func (r *RuneBuffer) idxLine(width int) int { if width == 0 { return 0 } sp := r.getSplitByLine(r.buf[:r.idx]) return len(sp) - 1 } func (r *RuneBuffer) CursorLineCount() int { return r.LineCount(r.width) - r.IdxLine(r.width) } func (r *RuneBuffer) Refresh(f func()) { r.Lock() defer r.Unlock() if !r.interactive { if f != nil { f() } return } r.clean() if f != nil { f() } r.print() } func (r *RuneBuffer) SetOffset(offset string) { r.Lock() r.offset = offset r.Unlock() } func (r *RuneBuffer) print() { r.w.Write(r.output()) r.hadClean = false } func (r *RuneBuffer) output() []byte { buf := bytes.NewBuffer(nil) buf.WriteString(string(r.prompt)) if r.cfg.EnableMask && len(r.buf) > 0 { buf.Write([]byte(strings.Repeat(string(r.cfg.MaskRune), len(r.buf)-1))) if r.buf[len(r.buf)-1] == '\n' { buf.Write([]byte{'\n'}) } else { buf.Write([]byte(string(r.cfg.MaskRune))) } if len(r.buf) > r.idx { buf.Write(runes.Backspace(r.buf[r.idx:])) } } else { for idx := range r.buf { if r.buf[idx] == '\t' { buf.WriteString(strings.Repeat(" ", TabWidth)) } else { buf.WriteRune(r.buf[idx]) } } if r.isInLineEdge() { buf.Write([]byte(" \b")) } } if len(r.buf) > r.idx { buf.Write(runes.Backspace(r.buf[r.idx:])) } return buf.Bytes() } func (r *RuneBuffer) Reset() []rune { ret := runes.Copy(r.buf) r.buf = r.buf[:0] r.idx = 0 return ret } func (r *RuneBuffer) calWidth(m int) int { if m > 0 { return runes.WidthAll(r.buf[r.idx : r.idx+m]) } return runes.WidthAll(r.buf[r.idx+m : r.idx]) } func (r *RuneBuffer) SetStyle(start, end int, style string) { if end < start { panic("end < start") } // goto start move := start - r.idx if move > 0 { r.w.Write([]byte(string(r.buf[r.idx : r.idx+move]))) } else { r.w.Write(bytes.Repeat([]byte("\b"), r.calWidth(move))) } r.w.Write([]byte("\033[" + style + "m")) r.w.Write([]byte(string(r.buf[start:end]))) r.w.Write([]byte("\033[0m")) // TODO: move back } func (r *RuneBuffer) SetWithIdx(idx int, buf []rune) { r.Refresh(func() { r.buf = buf r.idx = idx }) } func (r *RuneBuffer) Set(buf []rune) { r.SetWithIdx(len(buf), buf) } func (r *RuneBuffer) SetPrompt(prompt string) { r.Lock() r.prompt = []rune(prompt) r.Unlock() } func (r *RuneBuffer) cleanOutput(w io.Writer, idxLine int) { buf := bufio.NewWriter(w) if r.width == 0 { buf.WriteString(strings.Repeat("\r\b", len(r.buf)+r.promptLen())) buf.Write([]byte("\033[J")) } else { buf.Write([]byte("\033[J")) // just like ^k :) if idxLine == 0 { buf.WriteString("\033[2K") buf.WriteString("\r") } else { for i := 0; i < idxLine; i++ { io.WriteString(buf, "\033[2K\r\033[A") } io.WriteString(buf, "\033[2K\r") } } buf.Flush() return } func (r *RuneBuffer) Clean() { r.Lock() r.clean() r.Unlock() } func (r *RuneBuffer) clean() { r.cleanWithIdxLine(r.idxLine(r.width)) } func (r *RuneBuffer) cleanWithIdxLine(idxLine int) { if r.hadClean || !r.interactive { return } r.hadClean = true r.cleanOutput(r.w, idxLine) } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/runes/runes.go ================================================ // deprecated. // see https://github.com/chzyer/readline/issues/43 // use github.com/chzyer/readline/runes.go package runes import ( "bytes" "unicode" ) func Equal(a, b []rune) bool { if len(a) != len(b) { return false } for i := 0; i < len(a); i++ { if a[i] != b[i] { return false } } return true } // Search in runes from end to front func IndexAllBck(r, sub []rune) int { for i := len(r) - len(sub); i >= 0; i-- { found := true for j := 0; j < len(sub); j++ { if r[i+j] != sub[j] { found = false break } } if found { return i } } return -1 } // Search in runes from front to end func IndexAll(r, sub []rune) int { for i := 0; i < len(r); i++ { found := true if len(r[i:]) < len(sub) { return -1 } for j := 0; j < len(sub); j++ { if r[i+j] != sub[j] { found = false break } } if found { return i } } return -1 } func Index(r rune, rs []rune) int { for i := 0; i < len(rs); i++ { if rs[i] == r { return i } } return -1 } func ColorFilter(r []rune) []rune { newr := make([]rune, 0, len(r)) for pos := 0; pos < len(r); pos++ { if r[pos] == '\033' && r[pos+1] == '[' { idx := Index('m', r[pos+2:]) if idx == -1 { continue } pos += idx + 2 continue } newr = append(newr, r[pos]) } return newr } var zeroWidth = []*unicode.RangeTable{ unicode.Mn, unicode.Me, unicode.Cc, unicode.Cf, } var doubleWidth = []*unicode.RangeTable{ unicode.Han, unicode.Hangul, unicode.Hiragana, unicode.Katakana, } func Width(r rune) int { if unicode.IsOneOf(zeroWidth, r) { return 0 } if unicode.IsOneOf(doubleWidth, r) { return 2 } return 1 } func WidthAll(r []rune) (length int) { for i := 0; i < len(r); i++ { length += Width(r[i]) } return } func Backspace(r []rune) []byte { return bytes.Repeat([]byte{'\b'}, WidthAll(r)) } func Copy(r []rune) []rune { n := make([]rune, len(r)) copy(n, r) return n } func HasPrefix(r, prefix []rune) bool { if len(r) < len(prefix) { return false } return Equal(r[:len(prefix)], prefix) } func Aggregate(candicate [][]rune) (same []rune, size int) { for i := 0; i < len(candicate[0]); i++ { for j := 0; j < len(candicate)-1; j++ { if i >= len(candicate[j]) || i >= len(candicate[j+1]) { goto aggregate } if candicate[j][i] != candicate[j+1][i] { goto aggregate } } size = i + 1 } aggregate: if size > 0 { same = Copy(candicate[0][:size]) for i := 0; i < len(candicate); i++ { n := Copy(candicate[i]) copy(n, n[size:]) candicate[i] = n[:len(n)-size] } } return } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/runes/runes_test.go ================================================ package runes import ( "reflect" "testing" ) type twidth struct { r []rune length int } func TestRuneWidth(t *testing.T) { runes := []twidth{ {[]rune("☭"), 1}, {[]rune("a"), 1}, {[]rune("你"), 2}, {ColorFilter([]rune("☭\033[13;1m你")), 3}, } for _, r := range runes { if w := WidthAll(r.r); w != r.length { t.Fatal("result not expect", r.r, r.length, w) } } } type tagg struct { r [][]rune e [][]rune length int } func TestAggRunes(t *testing.T) { runes := []tagg{ { [][]rune{[]rune("ab"), []rune("a"), []rune("abc")}, [][]rune{[]rune("b"), []rune(""), []rune("bc")}, 1, }, { [][]rune{[]rune("addb"), []rune("ajkajsdf"), []rune("aasdfkc")}, [][]rune{[]rune("ddb"), []rune("jkajsdf"), []rune("asdfkc")}, 1, }, { [][]rune{[]rune("ddb"), []rune("ajksdf"), []rune("aasdfkc")}, [][]rune{[]rune("ddb"), []rune("ajksdf"), []rune("aasdfkc")}, 0, }, { [][]rune{[]rune("ddb"), []rune("ddajksdf"), []rune("ddaasdfkc")}, [][]rune{[]rune("b"), []rune("ajksdf"), []rune("aasdfkc")}, 2, }, } for _, r := range runes { same, off := Aggregate(r.r) if off != r.length { t.Fatal("result not expect", off) } if len(same) != off { t.Fatal("result not expect", same) } if !reflect.DeepEqual(r.r, r.e) { t.Fatal("result not expect") } } } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/runes.go ================================================ package readline import ( "bytes" "unicode" "unicode/utf8" ) var runes = Runes{} var TabWidth = 4 type Runes struct{} func (Runes) EqualRune(a, b rune, fold bool) bool { if a == b { return true } if !fold { return false } if a > b { a, b = b, a } if b < utf8.RuneSelf && 'A' <= a && a <= 'Z' { if b == a+'a'-'A' { return true } } return false } func (r Runes) EqualRuneFold(a, b rune) bool { return r.EqualRune(a, b, true) } func (r Runes) EqualFold(a, b []rune) bool { if len(a) != len(b) { return false } for i := 0; i < len(a); i++ { if r.EqualRuneFold(a[i], b[i]) { continue } return false } return true } func (Runes) Equal(a, b []rune) bool { if len(a) != len(b) { return false } for i := 0; i < len(a); i++ { if a[i] != b[i] { return false } } return true } func (rs Runes) IndexAllBckEx(r, sub []rune, fold bool) int { for i := len(r) - len(sub); i >= 0; i-- { found := true for j := 0; j < len(sub); j++ { if !rs.EqualRune(r[i+j], sub[j], fold) { found = false break } } if found { return i } } return -1 } // Search in runes from end to front func (rs Runes) IndexAllBck(r, sub []rune) int { return rs.IndexAllBckEx(r, sub, false) } // Search in runes from front to end func (rs Runes) IndexAll(r, sub []rune) int { return rs.IndexAllEx(r, sub, false) } func (rs Runes) IndexAllEx(r, sub []rune, fold bool) int { for i := 0; i < len(r); i++ { found := true if len(r[i:]) < len(sub) { return -1 } for j := 0; j < len(sub); j++ { if !rs.EqualRune(r[i+j], sub[j], fold) { found = false break } } if found { return i } } return -1 } func (Runes) Index(r rune, rs []rune) int { for i := 0; i < len(rs); i++ { if rs[i] == r { return i } } return -1 } func (Runes) ColorFilter(r []rune) []rune { newr := make([]rune, 0, len(r)) for pos := 0; pos < len(r); pos++ { if r[pos] == '\033' && r[pos+1] == '[' { idx := runes.Index('m', r[pos+2:]) if idx == -1 { continue } pos += idx + 2 continue } newr = append(newr, r[pos]) } return newr } var zeroWidth = []*unicode.RangeTable{ unicode.Mn, unicode.Me, unicode.Cc, unicode.Cf, } var doubleWidth = []*unicode.RangeTable{ unicode.Han, unicode.Hangul, unicode.Hiragana, unicode.Katakana, } func (Runes) Width(r rune) int { if r == '\t' { return TabWidth } if unicode.IsOneOf(zeroWidth, r) { return 0 } if unicode.IsOneOf(doubleWidth, r) { return 2 } return 1 } func (Runes) WidthAll(r []rune) (length int) { for i := 0; i < len(r); i++ { length += runes.Width(r[i]) } return } func (Runes) Backspace(r []rune) []byte { return bytes.Repeat([]byte{'\b'}, runes.WidthAll(r)) } func (Runes) Copy(r []rune) []rune { n := make([]rune, len(r)) copy(n, r) return n } func (Runes) HasPrefixFold(r, prefix []rune) bool { if len(r) < len(prefix) { return false } return runes.EqualFold(r[:len(prefix)], prefix) } func (Runes) HasPrefix(r, prefix []rune) bool { if len(r) < len(prefix) { return false } return runes.Equal(r[:len(prefix)], prefix) } func (Runes) Aggregate(candicate [][]rune) (same []rune, size int) { for i := 0; i < len(candicate[0]); i++ { for j := 0; j < len(candicate)-1; j++ { if i >= len(candicate[j]) || i >= len(candicate[j+1]) { goto aggregate } if candicate[j][i] != candicate[j+1][i] { goto aggregate } } size = i + 1 } aggregate: if size > 0 { same = runes.Copy(candicate[0][:size]) for i := 0; i < len(candicate); i++ { n := runes.Copy(candicate[i]) copy(n, n[size:]) candicate[i] = n[:len(n)-size] } } return } func (Runes) TrimSpaceLeft(in []rune) []rune { firstIndex := len(in) for i, r := range in { if unicode.IsSpace(r) == false { firstIndex = i break } } return in[firstIndex:] } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/runes_test.go ================================================ package readline import ( "reflect" "testing" ) type twidth struct { r []rune length int } func TestRuneWidth(t *testing.T) { rs := []twidth{ {[]rune("☭"), 1}, {[]rune("a"), 1}, {[]rune("你"), 2}, {runes.ColorFilter([]rune("☭\033[13;1m你")), 3}, } for _, r := range rs { if w := runes.WidthAll(r.r); w != r.length { t.Fatal("result not expect", r.r, r.length, w) } } } type tagg struct { r [][]rune e [][]rune length int } func TestAggRunes(t *testing.T) { rs := []tagg{ { [][]rune{[]rune("ab"), []rune("a"), []rune("abc")}, [][]rune{[]rune("b"), []rune(""), []rune("bc")}, 1, }, { [][]rune{[]rune("addb"), []rune("ajkajsdf"), []rune("aasdfkc")}, [][]rune{[]rune("ddb"), []rune("jkajsdf"), []rune("asdfkc")}, 1, }, { [][]rune{[]rune("ddb"), []rune("ajksdf"), []rune("aasdfkc")}, [][]rune{[]rune("ddb"), []rune("ajksdf"), []rune("aasdfkc")}, 0, }, { [][]rune{[]rune("ddb"), []rune("ddajksdf"), []rune("ddaasdfkc")}, [][]rune{[]rune("b"), []rune("ajksdf"), []rune("aasdfkc")}, 2, }, } for _, r := range rs { same, off := runes.Aggregate(r.r) if off != r.length { t.Fatal("result not expect", off) } if len(same) != off { t.Fatal("result not expect", same) } if !reflect.DeepEqual(r.r, r.e) { t.Fatal("result not expect") } } } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/std.go ================================================ package readline import ( "io" "os" "sync" ) var ( Stdin io.ReadCloser = os.Stdin Stdout io.WriteCloser = os.Stdout Stderr io.WriteCloser = os.Stderr ) var ( std *Instance stdOnce sync.Once ) // global instance will not submit history automatic func getInstance() *Instance { stdOnce.Do(func() { std, _ = NewEx(&Config{ DisableAutoSaveHistory: true, }) }) return std } // let readline load history from filepath // and try to persist history into disk // set fp to "" to prevent readline persisting history to disk // so the `AddHistory` will return nil error forever. func SetHistoryPath(fp string) { ins := getInstance() cfg := ins.Config.Clone() cfg.HistoryFile = fp ins.SetConfig(cfg) } // set auto completer to global instance func SetAutoComplete(completer AutoCompleter) { ins := getInstance() cfg := ins.Config.Clone() cfg.AutoComplete = completer ins.SetConfig(cfg) } // add history to global instance manually // raise error only if `SetHistoryPath` is set with a non-empty path func AddHistory(content string) error { ins := getInstance() return ins.SaveHistory(content) } func Password(prompt string) ([]byte, error) { ins := getInstance() return ins.ReadPassword(prompt) } // readline with global configs func Line(prompt string) (string, error) { ins := getInstance() ins.SetPrompt(prompt) return ins.Readline() } type CancelableStdin struct { r io.Reader mutex sync.Mutex stop chan struct{} notify chan struct{} data []byte read int err error } func NewCancelableStdin(r io.Reader) *CancelableStdin { c := &CancelableStdin{ r: r, notify: make(chan struct{}), stop: make(chan struct{}), } go c.ioloop() return c } func (c *CancelableStdin) ioloop() { loop: for { select { case <-c.notify: c.read, c.err = c.r.Read(c.data) c.notify <- struct{}{} case <-c.stop: break loop } } } func (c *CancelableStdin) Read(b []byte) (n int, err error) { c.mutex.Lock() defer c.mutex.Unlock() c.data = b c.notify <- struct{}{} select { case <-c.notify: return c.read, c.err case <-c.stop: return 0, io.EOF } } func (c *CancelableStdin) Close() error { close(c.stop) return nil } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/std_windows.go ================================================ // +build windows package readline func init() { Stdin = NewRawReader() Stdout = NewANSIWriter(Stdout) Stderr = NewANSIWriter(Stderr) } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/term.go ================================================ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build darwin dragonfly freebsd linux,!appengine netbsd openbsd // Package terminal provides support functions for dealing with terminals, as // commonly found on UNIX systems. // // Putting a terminal into raw mode is the most common requirement: // // oldState, err := terminal.MakeRaw(0) // if err != nil { // panic(err) // } // defer terminal.Restore(0, oldState) package readline import ( "io" "syscall" "unsafe" ) // State contains the state of a terminal. type State struct { termios syscall.Termios } // IsTerminal returns true if the given file descriptor is a terminal. func IsTerminal(fd int) bool { var termios syscall.Termios _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) return err == 0 } // MakeRaw put the terminal connected to the given file descriptor into raw // mode and returns the previous state of the terminal so that it can be // restored. func MakeRaw(fd int) (*State, error) { var oldState State if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 { return nil, err } newState := oldState.termios // This attempts to replicate the behaviour documented for cfmakeraw in // the termios(3) manpage. newState.Iflag &^= syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON // newState.Oflag &^= syscall.OPOST newState.Lflag &^= syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN newState.Cflag &^= syscall.CSIZE | syscall.PARENB newState.Cflag |= syscall.CS8 if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 { return nil, err } return &oldState, nil } // GetState returns the current state of a terminal which may be useful to // restore the terminal after a signal. func GetState(fd int) (*State, error) { var oldState State if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 { return nil, err } return &oldState, nil } // Restore restores the terminal connected to the given file descriptor to a // previous state. func restoreTerm(fd int, state *State) error { _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0) return err } // GetSize returns the dimensions of the given terminal. func GetSize(fd int) (width, height int, err error) { var dimensions [4]uint16 if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 { return -1, -1, err } return int(dimensions[1]), int(dimensions[0]), nil } // ReadPassword reads a line of input from a terminal without local echo. This // is commonly used for inputting passwords and other sensitive data. The slice // returned does not include the \n. func ReadPassword(fd int) ([]byte, error) { var oldState syscall.Termios if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0); err != 0 { return nil, err } newState := oldState newState.Lflag &^= syscall.ECHO newState.Lflag |= syscall.ICANON | syscall.ISIG newState.Iflag |= syscall.ICRNL if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 { return nil, err } defer func() { syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlWriteTermios, uintptr(unsafe.Pointer(&oldState)), 0, 0, 0) }() var buf [16]byte var ret []byte for { n, err := syscall.Read(fd, buf[:]) if err != nil { return nil, err } if n == 0 { if len(ret) == 0 { return nil, io.EOF } break } if buf[n-1] == '\n' { n-- } ret = append(ret, buf[:n]...) if n < len(buf) { break } } return ret, nil } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/term_bsd.go ================================================ // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build darwin dragonfly freebsd netbsd openbsd package readline import "syscall" const ioctlReadTermios = syscall.TIOCGETA const ioctlWriteTermios = syscall.TIOCSETA ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/term_linux.go ================================================ // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package readline // These constants are declared here, rather than importing // them from the syscall package as some syscall packages, even // on linux, for example gccgo, do not declare them. const ioctlReadTermios = 0x5401 // syscall.TCGETS const ioctlWriteTermios = 0x5402 // syscall.TCSETS ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/term_windows.go ================================================ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build windows // Package terminal provides support functions for dealing with terminals, as // commonly found on UNIX systems. // // Putting a terminal into raw mode is the most common requirement: // // oldState, err := terminal.MakeRaw(0) // if err != nil { // panic(err) // } // defer terminal.Restore(0, oldState) package readline import ( "io" "syscall" "unsafe" ) const ( enableLineInput = 2 enableEchoInput = 4 enableProcessedInput = 1 enableWindowInput = 8 enableMouseInput = 16 enableInsertMode = 32 enableQuickEditMode = 64 enableExtendedFlags = 128 enableAutoPosition = 256 enableProcessedOutput = 1 enableWrapAtEolOutput = 2 ) var kernel32 = syscall.NewLazyDLL("kernel32.dll") var ( procGetConsoleMode = kernel32.NewProc("GetConsoleMode") procSetConsoleMode = kernel32.NewProc("SetConsoleMode") procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo") ) type ( coord struct { x short y short } smallRect struct { left short top short right short bottom short } consoleScreenBufferInfo struct { size coord cursorPosition coord attributes word window smallRect maximumWindowSize coord } ) type State struct { mode uint32 } // IsTerminal returns true if the given file descriptor is a terminal. func IsTerminal(fd int) bool { var st uint32 r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0) return r != 0 && e == 0 } // MakeRaw put the terminal connected to the given file descriptor into raw // mode and returns the previous state of the terminal so that it can be // restored. func MakeRaw(fd int) (*State, error) { var st uint32 _, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0) if e != 0 { return nil, error(e) } raw := st &^ (enableEchoInput | enableProcessedInput | enableLineInput | enableProcessedOutput) _, _, e = syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(raw), 0) if e != 0 { return nil, error(e) } return &State{st}, nil } // GetState returns the current state of a terminal which may be useful to // restore the terminal after a signal. func GetState(fd int) (*State, error) { var st uint32 _, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0) if e != 0 { return nil, error(e) } return &State{st}, nil } // Restore restores the terminal connected to the given file descriptor to a // previous state. func restoreTerm(fd int, state *State) error { _, _, err := syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(state.mode), 0) return err } // GetSize returns the dimensions of the given terminal. func GetSize(fd int) (width, height int, err error) { var info consoleScreenBufferInfo _, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&info)), 0) if e != 0 { return 0, 0, error(e) } return int(info.size.x), int(info.size.y), nil } // ReadPassword reads a line of input from a terminal without local echo. This // is commonly used for inputting passwords and other sensitive data. The slice // returned does not include the \n. func ReadPassword(fd int) ([]byte, error) { var st uint32 _, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0) if e != 0 { return nil, error(e) } old := st st &^= (enableEchoInput) st |= (enableProcessedInput | enableLineInput | enableProcessedOutput) _, _, e = syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(st), 0) if e != 0 { return nil, error(e) } defer func() { syscall.Syscall(procSetConsoleMode.Addr(), 2, uintptr(fd), uintptr(old), 0) }() var buf [16]byte var ret []byte for { n, err := syscall.Read(syscall.Handle(fd), buf[:]) if err != nil { return nil, err } if n == 0 { if len(ret) == 0 { return nil, io.EOF } break } if buf[n-1] == '\n' { n-- } if n > 0 && buf[n-1] == '\r' { n-- } ret = append(ret, buf[:n]...) if n < len(buf) { break } } return ret, nil } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/terminal.go ================================================ package readline import ( "bufio" "fmt" "io" "strings" "sync" "sync/atomic" "time" ) type Terminal struct { cfg *Config outchan chan rune closed int32 stopChan chan struct{} kickChan chan struct{} wg sync.WaitGroup isReading int32 sleeping int32 pause int32 sizeChan chan string } func NewTerminal(cfg *Config) (*Terminal, error) { if err := cfg.Init(); err != nil { return nil, err } t := &Terminal{ cfg: cfg, kickChan: make(chan struct{}, 1), outchan: make(chan rune), stopChan: make(chan struct{}, 1), sizeChan: make(chan string, 1), } go t.ioloop() return t, nil } // SleepToResume will sleep myself, and return only if I'm resumed. func (t *Terminal) SleepToResume() { if !atomic.CompareAndSwapInt32(&t.sleeping, 0, 1) { return } defer atomic.StoreInt32(&t.sleeping, 0) t.ExitRawMode() ch := WaitForResume() SuspendMe() <-ch t.EnterRawMode() } func (t *Terminal) EnterRawMode() (err error) { return t.cfg.FuncMakeRaw() } func (t *Terminal) ExitRawMode() (err error) { return t.cfg.FuncExitRaw() } func (t *Terminal) Write(b []byte) (int, error) { return t.cfg.Stdout.Write(b) } type termSize struct { left int top int } func (t *Terminal) GetOffset(f func(offset string)) { go func() { f(<-t.sizeChan) }() t.Write([]byte("\033[6n")) } func (t *Terminal) Print(s string) { fmt.Fprintf(t.cfg.Stdout, "%s", s) } func (t *Terminal) PrintRune(r rune) { fmt.Fprintf(t.cfg.Stdout, "%c", r) } func (t *Terminal) Readline() *Operation { return NewOperation(t, t.cfg) } // return rune(0) if meet EOF func (t *Terminal) ReadRune() rune { ch, ok := <-t.outchan if !ok { return rune(0) } return ch } func (t *Terminal) IsReading() bool { return atomic.LoadInt32(&t.isReading) == 1 } func (t *Terminal) KickRead() { select { case t.kickChan <- struct{}{}: default: } } func (t *Terminal) PauseRead(b bool) { if b { atomic.StoreInt32(&t.pause, 1) } else { atomic.StoreInt32(&t.pause, 0) } } func (t *Terminal) ioloop() { t.wg.Add(1) defer func() { t.wg.Done() close(t.outchan) }() var ( isEscape bool isEscapeEx bool expectNextChar bool ) buf := bufio.NewReader(t.cfg.Stdin) for { pause := atomic.LoadInt32(&t.pause) if pause == 1 { time.Sleep(100 * time.Millisecond) continue } if !expectNextChar { atomic.StoreInt32(&t.isReading, 0) select { case <-t.kickChan: atomic.StoreInt32(&t.isReading, 1) case <-t.stopChan: return } } expectNextChar = false r, _, err := buf.ReadRune() if err != nil { if strings.Contains(err.Error(), "interrupted system call") { expectNextChar = true continue } break } if isEscape { isEscape = false if r == CharEscapeEx { expectNextChar = true isEscapeEx = true continue } r = escapeKey(r, buf) } else if isEscapeEx { isEscapeEx = false if key := readEscKey(r, buf); key != nil { r = escapeExKey(key) // offset if key.typ == 'R' { if _, _, ok := key.Get2(); ok { select { case t.sizeChan <- key.attr: default: } } expectNextChar = true continue } } if r == 0 { expectNextChar = true continue } } expectNextChar = true switch r { case CharEsc: if t.cfg.VimMode { t.outchan <- r break } isEscape = true case CharInterrupt, CharEnter, CharCtrlJ, CharDelete: expectNextChar = false fallthrough default: if r == CharTab { t.PauseRead(true) } t.outchan <- r } } } func (t *Terminal) Bell() { fmt.Fprintf(t, "%c", CharBell) } func (t *Terminal) Close() error { if atomic.SwapInt32(&t.closed, 1) != 0 { return nil } if closer, ok := t.cfg.Stdin.(io.Closer); ok { closer.Close() } close(t.stopChan) t.wg.Wait() return t.ExitRawMode() } func (t *Terminal) SetConfig(c *Config) error { if err := c.Init(); err != nil { return err } t.cfg = c return nil } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/utils.go ================================================ package readline import ( "bufio" "bytes" "container/list" "fmt" "os" "strconv" "strings" "sync" "time" "unicode" ) var ( isWindows = false ) const ( CharLineStart = 1 CharBackward = 2 CharInterrupt = 3 CharDelete = 4 CharLineEnd = 5 CharForward = 6 CharBell = 7 CharCtrlH = 8 CharTab = 9 CharCtrlJ = 10 CharKill = 11 CharCtrlL = 12 CharEnter = 13 CharNext = 14 CharPrev = 16 CharBckSearch = 18 CharFwdSearch = 19 CharTranspose = 20 CharCtrlU = 21 CharCtrlW = 23 CharCtrlZ = 26 CharEsc = 27 CharEscapeEx = 91 CharBackspace = 127 ) const ( MetaBackward rune = -iota - 1 MetaForward MetaDelete MetaBackspace MetaTranspose ) // WaitForResume need to call before current process got suspend. // It will run a ticker until a long duration is occurs, // which means this process is resumed. func WaitForResume() chan struct{} { ch := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) go func() { ticker := time.NewTicker(10 * time.Millisecond) t := time.Now() wg.Done() for { now := <-ticker.C if now.Sub(t) > 100*time.Millisecond { break } t = now } ticker.Stop() ch <- struct{}{} }() wg.Wait() return ch } func Restore(fd int, state *State) error { err := restoreTerm(fd, state) if err != nil { // errno 0 means everything is ok :) if err.Error() == "errno 0" { err = nil } } return nil } func IsPrintable(key rune) bool { isInSurrogateArea := key >= 0xd800 && key <= 0xdbff return key >= 32 && !isInSurrogateArea } // translate Esc[X func escapeExKey(key *escapeKeyPair) rune { var r rune switch key.typ { case 'D': r = CharBackward case 'C': r = CharForward case 'A': r = CharPrev case 'B': r = CharNext case 'H': r = CharLineStart case 'F': r = CharLineEnd case '~': if key.attr == "3" { r = CharDelete } default: } return r } type escapeKeyPair struct { attr string typ rune } func (e *escapeKeyPair) Get2() (int, int, bool) { sp := strings.Split(e.attr, ";") if len(sp) < 2 { return -1, -1, false } s1, err := strconv.Atoi(sp[0]) if err != nil { return -1, -1, false } s2, err := strconv.Atoi(sp[1]) if err != nil { return -1, -1, false } return s1, s2, true } func readEscKey(r rune, reader *bufio.Reader) *escapeKeyPair { p := escapeKeyPair{} buf := bytes.NewBuffer(nil) for { if r == ';' { } else if unicode.IsNumber(r) { } else { p.typ = r break } buf.WriteRune(r) r, _, _ = reader.ReadRune() } p.attr = buf.String() return &p } // translate EscX to Meta+X func escapeKey(r rune, reader *bufio.Reader) rune { switch r { case 'b': r = MetaBackward case 'f': r = MetaForward case 'd': r = MetaDelete case CharTranspose: r = MetaTranspose case CharBackspace: r = MetaBackspace case 'O': d, _, _ := reader.ReadRune() switch d { case 'H': r = CharLineStart case 'F': r = CharLineEnd default: reader.UnreadRune() } case CharEsc: } return r } func SplitByLine(start, screenWidth int, rs []rune) []string { var ret []string buf := bytes.NewBuffer(nil) currentWidth := start for _, r := range rs { w := runes.Width(r) currentWidth += w buf.WriteRune(r) if currentWidth >= screenWidth { ret = append(ret, buf.String()) buf.Reset() currentWidth = 0 } } ret = append(ret, buf.String()) return ret } // calculate how many lines for N character func LineCount(screenWidth, w int) int { r := w / screenWidth if w%screenWidth != 0 { r++ } return r } func IsWordBreak(i rune) bool { switch { case i >= 'a' && i <= 'z': case i >= 'A' && i <= 'Z': case i >= '0' && i <= '9': default: return true } return false } func GetInt(s []string, def int) int { if len(s) == 0 { return def } c, err := strconv.Atoi(s[0]) if err != nil { return def } return c } type RawMode struct { state *State } func (r *RawMode) Enter() (err error) { r.state, err = MakeRaw(GetStdin()) return err } func (r *RawMode) Exit() error { if r.state == nil { return nil } return Restore(GetStdin(), r.state) } // ----------------------------------------------------------------------------- func sleep(n int) { Debug(n) time.Sleep(2000 * time.Millisecond) } // print a linked list to Debug() func debugList(l *list.List) { idx := 0 for e := l.Front(); e != nil; e = e.Next() { Debug(idx, fmt.Sprintf("%+v", e.Value)) idx++ } } // append log info to another file func Debug(o ...interface{}) { f, _ := os.OpenFile("debug.tmp", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) fmt.Fprintln(f, o...) f.Close() } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/utils_test.go ================================================ package readline ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/utils_unix.go ================================================ // +build darwin dragonfly freebsd linux,!appengine netbsd openbsd package readline import ( "io" "os" "os/signal" "sync" "syscall" "unsafe" ) type winsize struct { Row uint16 Col uint16 Xpixel uint16 Ypixel uint16 } // SuspendMe use to send suspend signal to myself, when we in the raw mode. // For OSX it need to send to parent's pid // For Linux it need to send to myself func SuspendMe() { p, _ := os.FindProcess(os.Getppid()) p.Signal(syscall.SIGTSTP) p, _ = os.FindProcess(os.Getpid()) p.Signal(syscall.SIGTSTP) } // get width of the terminal func getWidth(stdoutFd int) int { ws := &winsize{} retCode, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(stdoutFd), uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(ws))) if int(retCode) == -1 { _ = errno return -1 } return int(ws.Col) } func GetScreenWidth() int { w := getWidth(syscall.Stdout) if w < 0 { w = getWidth(syscall.Stderr) } return w } // ClearScreen clears the console screen func ClearScreen(w io.Writer) (int, error) { return w.Write([]byte("\033[H")) } func DefaultIsTerminal() bool { return IsTerminal(syscall.Stdin) && (IsTerminal(syscall.Stdout) || IsTerminal(syscall.Stderr)) } func GetStdin() int { return syscall.Stdin } // ----------------------------------------------------------------------------- var ( widthChange sync.Once widthChangeCallback func() ) func DefaultOnWidthChanged(f func()) { widthChangeCallback = f widthChange.Do(func() { ch := make(chan os.Signal, 1) signal.Notify(ch, syscall.SIGWINCH) go func() { for { _, ok := <-ch if !ok { break } widthChangeCallback() } }() }) } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/utils_windows.go ================================================ // +build windows package readline import ( "io" "syscall" ) func SuspendMe() { } func GetStdin() int { return int(syscall.Stdin) } func init() { isWindows = true } // get width of the terminal func GetScreenWidth() int { info, _ := GetConsoleScreenBufferInfo() if info == nil { return -1 } return int(info.dwSize.x) } // ClearScreen clears the console screen func ClearScreen(_ io.Writer) error { return SetConsoleCursorPosition(&_COORD{0, 0}) } func DefaultIsTerminal() bool { return true } func DefaultOnWidthChanged(func()) { } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/vim.go ================================================ package readline const ( VIM_NORMAL = iota VIM_INSERT VIM_VISUAL ) type opVim struct { cfg *Config op *Operation vimMode int } func newVimMode(op *Operation) *opVim { ov := &opVim{ cfg: op.cfg, op: op, } ov.SetVimMode(ov.cfg.VimMode) return ov } func (o *opVim) SetVimMode(on bool) { if o.cfg.VimMode && !on { // turn off o.ExitVimMode() } o.cfg.VimMode = on o.vimMode = VIM_INSERT } func (o *opVim) ExitVimMode() { o.vimMode = VIM_INSERT } func (o *opVim) IsEnableVimMode() bool { return o.cfg.VimMode } func (o *opVim) handleVimNormalMovement(r rune, readNext func() rune) (t rune, handled bool) { rb := o.op.buf handled = true switch r { case 'h': t = CharBackward case 'j': t = CharNext case 'k': t = CharPrev case 'l': t = CharForward case '0', '^': rb.MoveToLineStart() case '$': rb.MoveToLineEnd() case 'x': rb.Delete() if rb.IsCursorInEnd() { rb.MoveBackward() } case 'r': rb.Replace(readNext()) case 'd': next := readNext() switch next { case 'd': rb.Erase() case 'w': rb.DeleteWord() case 'h': rb.Backspace() case 'l': rb.Delete() } case 'b', 'B': rb.MoveToPrevWord() case 'w', 'W': rb.MoveToNextWord() case 'e', 'E': rb.MoveToEndWord() case 'f', 'F', 't', 'T': next := readNext() prevChar := r == 't' || r == 'T' reverse := r == 'F' || r == 'T' switch next { case CharEsc: default: rb.MoveTo(next, prevChar, reverse) } default: return r, false } return t, true } func (o *opVim) handleVimNormalEnterInsert(r rune, readNext func() rune) (t rune, handled bool) { rb := o.op.buf handled = true switch r { case 'i': case 'I': rb.MoveToLineStart() case 'a': rb.MoveForward() case 'A': rb.MoveToLineEnd() case 's': rb.Delete() case 'S': rb.Erase() case 'c': next := readNext() switch next { case 'c': rb.Erase() case 'w': rb.DeleteWord() case 'h': rb.Backspace() case 'l': rb.Delete() } default: return r, false } o.EnterVimInsertMode() return } func (o *opVim) HandleVimNormal(r rune, readNext func() rune) (t rune) { switch r { case CharEnter, CharInterrupt: o.ExitVimMode() return r } if r, handled := o.handleVimNormalMovement(r, readNext); handled { return r } if r, handled := o.handleVimNormalEnterInsert(r, readNext); handled { return r } // invalid operation o.op.t.Bell() return 0 } func (o *opVim) EnterVimInsertMode() { o.vimMode = VIM_INSERT } func (o *opVim) ExitVimInsertMode() { o.vimMode = VIM_NORMAL } func (o *opVim) HandleVim(r rune, readNext func() rune) rune { if o.vimMode == VIM_NORMAL { return o.HandleVimNormal(r, readNext) } if r == CharEsc { o.ExitVimInsertMode() return 0 } switch o.vimMode { case VIM_INSERT: return r case VIM_VISUAL: } return r } ================================================ FILE: cmd/nash/vendor/github.com/chzyer/readline/windows_api.go ================================================ // +build windows package readline import ( "reflect" "syscall" "unsafe" ) var ( kernel = NewKernel() stdout = uintptr(syscall.Stdout) stdin = uintptr(syscall.Stdin) ) type Kernel struct { SetConsoleCursorPosition, SetConsoleTextAttribute, FillConsoleOutputCharacterW, FillConsoleOutputAttribute, ReadConsoleInputW, GetConsoleScreenBufferInfo, GetConsoleCursorInfo, GetStdHandle CallFunc } type short int16 type word uint16 type dword uint32 type wchar uint16 type _COORD struct { x short y short } func (c *_COORD) ptr() uintptr { return uintptr(*(*int32)(unsafe.Pointer(c))) } const ( EVENT_KEY = 0x0001 EVENT_MOUSE = 0x0002 EVENT_WINDOW_BUFFER_SIZE = 0x0004 EVENT_MENU = 0x0008 EVENT_FOCUS = 0x0010 ) type _KEY_EVENT_RECORD struct { bKeyDown int32 wRepeatCount word wVirtualKeyCode word wVirtualScanCode word unicodeChar wchar dwControlKeyState dword } // KEY_EVENT_RECORD KeyEvent; // MOUSE_EVENT_RECORD MouseEvent; // WINDOW_BUFFER_SIZE_RECORD WindowBufferSizeEvent; // MENU_EVENT_RECORD MenuEvent; // FOCUS_EVENT_RECORD FocusEvent; type _INPUT_RECORD struct { EventType word Padding uint16 Event [16]byte } type _CONSOLE_SCREEN_BUFFER_INFO struct { dwSize _COORD dwCursorPosition _COORD wAttributes word srWindow _SMALL_RECT dwMaximumWindowSize _COORD } type _SMALL_RECT struct { left short top short right short bottom short } type _CONSOLE_CURSOR_INFO struct { dwSize dword bVisible bool } type CallFunc func(u ...uintptr) error func NewKernel() *Kernel { k := &Kernel{} kernel32 := syscall.NewLazyDLL("kernel32.dll") v := reflect.ValueOf(k).Elem() t := v.Type() for i := 0; i < t.NumField(); i++ { name := t.Field(i).Name f := kernel32.NewProc(name) v.Field(i).Set(reflect.ValueOf(k.Wrap(f))) } return k } func (k *Kernel) Wrap(p *syscall.LazyProc) CallFunc { return func(args ...uintptr) error { var r0 uintptr var e1 syscall.Errno size := uintptr(len(args)) if len(args) <= 3 { buf := make([]uintptr, 3) copy(buf, args) r0, _, e1 = syscall.Syscall(p.Addr(), size, buf[0], buf[1], buf[2]) } else { buf := make([]uintptr, 6) copy(buf, args) r0, _, e1 = syscall.Syscall6(p.Addr(), size, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], ) } if int(r0) == 0 { if e1 != 0 { return error(e1) } else { return syscall.EINVAL } } return nil } } func GetConsoleScreenBufferInfo() (*_CONSOLE_SCREEN_BUFFER_INFO, error) { t := new(_CONSOLE_SCREEN_BUFFER_INFO) err := kernel.GetConsoleScreenBufferInfo( stdout, uintptr(unsafe.Pointer(t)), ) return t, err } func GetConsoleCursorInfo() (*_CONSOLE_CURSOR_INFO, error) { t := new(_CONSOLE_CURSOR_INFO) err := kernel.GetConsoleCursorInfo(stdout, uintptr(unsafe.Pointer(t))) return t, err } func SetConsoleCursorPosition(c *_COORD) error { return kernel.SetConsoleCursorPosition(stdout, c.ptr()) } ================================================ FILE: cmd/nash/vendor.sh ================================================ #!/usr/bin/env nash fn vendor() { cwdir <= pwd | xargs echo -n vendordir = $cwdir + "/vendor" rm -rf $vendordir bindir = $vendordir + "/bin" srcdir = $vendordir + "/src" pkgdir = $vendordir + "/pkg" mkdir -p $bindir $srcdir $pkgdir setenv GOPATH = $vendordir setenv GOBIN = $vendordir go get -v . rawpaths <= ls $srcdir paths <= split($paths, "\n") for path in $paths { mv $srcdir + $path $vendor } rm -rf $bindir $srcdir $pkgdir # because nash library is a dependency of cmd/nash # we need to remove it at end rm -rf vendor/github.com/madlambda } vendor() ================================================ FILE: cmd/nashfmt/main.go ================================================ package main import ( "flag" "fmt" "io" "io/ioutil" "os" "github.com/madlambda/nash/parser" ) var ( overwrite bool version bool // version is set at build time VersionString = "No version provided" ) func init() { flag.BoolVar(&overwrite, "w", false, "overwrite file") flag.BoolVar(&version, "version", false, "Show version") } func main() { var ( file io.ReadCloser err error ) flag.Parse() if version { fmt.Printf("build tag: %s\n", VersionString) return } if len(flag.Args()) <= 0 { flag.PrintDefaults() return } fname := flag.Args()[0] file, err = os.Open(fname) if err != nil { fmt.Fprintf(os.Stderr, "error: %s\n", err.Error()) os.Exit(1) } content, err := ioutil.ReadAll(file) if err != nil { file.Close() fmt.Fprintf(os.Stderr, "error: %s\n", err.Error()) os.Exit(1) } parser := parser.NewParser("nashfmt", string(content)) ast, err := parser.Parse() if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err.Error()) file.Close() os.Exit(1) } file.Close() if !overwrite { fmt.Printf("%s\n", ast.String()) return } if ast.String() != string(content) { err = ioutil.WriteFile(fname, []byte(fmt.Sprintf("%s\n", ast.String())), 0666) if err != nil { fmt.Fprintf(os.Stderr, "error: %s\n", err.Error()) return } } } ================================================ FILE: docs/interactive.md ================================================ # Table of Contents - [Line mode](#line-mode) - [Autocomplete](#autocomplete) - [Hooks](#hooks) - [bindfn](#bindfn) When used as an interactive shell, nash supports a few features to enhance user experience. # Line mode Nash supports line editing with `emacs` and `vim` modes. The default mode is `emacs` but it can be changed by the command `set mode vim`, or setting the environment variable `LINEMODE` with desired value. When in emacs mode, the following shortcuts can be used: | Shortcut | Comment | | ------------------ | --------------------------------- | | `Ctrl`+`A` | Beginning of line | | `Ctrl`+`B` / `←` | Backward one character | | `Meta`+`B` | Backward one word | | `Ctrl`+`C` | Send io.EOF | | `Ctrl`+`D` | Delete one character/Close nash | | `Meta`+`D` | Delete one word | | `Ctrl`+`E` | End of line | | `Ctrl`+`F` / `→` | Forward one character | | `Meta`+`F` | Forward one word | | `Ctrl`+`H` | Delete previous character | | `Ctrl`+`I` / `Tab` | Command line completion | | `Ctrl`+`J` | Line feed | | `Ctrl`+`K` | Cut text to the end of line | | `Ctrl`+`L` | Clear screen | | `Ctrl`+`M` | Same as Enter key | | `Ctrl`+`T` | Transpose characters | | `Ctrl`+`U` | Cut text to the beginning of line | | `Ctrl`+`W` | Cut previous word | | `Backspace` | Delete previous character | | `Meta`+`Backspace` | Cut previous word | | `Enter` | Line feed | # Autocomplete Nash doesn't have autocomplete built in, but it do has triggers to you implement it yourself. Every time the `TAB` or `CTRL-I (in emacs mode)` is pressed, nash looks for a function called `nash_complete` declared in the environment and calls it passing the line buffer and cursor position. The function must make the autocomplete using some external software (like [fzf fuzzy finder](https://github.com/junegunn/fzf)) and then return the characters to be completed. Below is a simple example to autocomplete system binaries using `fzf`: ```sh fn diffword(complete, line) { diff <= echo -n $complete | sed "s#^"+$line+"##g" | tr -d "\n" return $diff } fn nash_complete(line, pos) { ret = () parts <= split($line, "\n") choice <= ( find /bin /usr/bin -maxdepth 1 -type f | sed "s#/.*/##g" | sort -u | -fzf -q "^"+$line -1 -0 --header "Looking for system-wide binaries" --prompt "(λ programs)>" --reverse ) if $status != "0" { return $ret } choice <= diffword($choice, $line) ret = ($choice+" " "0") return $ret } ``` # Hooks There are two functions that can be used to update the environment while typing commands. The function `nash_repl_before` is called every time in the cli main loop *before* the printing of the `PROMPT` variable (and before user can type any command). And the function called `nash_repl_after` is called every time in the cli main loop too, but *after* the command was interpreted and executed. See the examples below: ```sh DEFPROMPT = "λ> " fn nash_repl_before() { # do something before prompt is ready datetime <= date "+%d/%m/%y %H:%M:%S" PROMPT = "("+$datetime+")"+$DEFPROMPT setenv PROMPT } fn nash_repl_after(line, status) { # do something after command was executed # line and status are the command issued and their # exit status (if applicable) } ``` # bindfn Functions are commonly used for nash libraries, but when needed it can be bind'ed to some command name, so it can be used as a command from your shell prompt. For example, lets implement a **cd** using a function and bindfn. First define the function: ```nash fn cd(path) { fullpath <= realpath $path | xargs echo -n chdir($path) PROMPT="[" + $fullpath + "]> " setenv PROMPT } ``` Using the **cd** function above, we can override the built-in **cd** with that function with the **bindfn** statement. ```nash λ> bindfn cd cd λ> cd /var/log [/var/log]> ``` The bindfn syntax is: ```nash bindfn ``` ================================================ FILE: docs/reference.md ================================================ # Table of Contents - [Command line arguments](#command-line-arguments) - [Flow control](#flow-control) - [Branching](#branching) - [Looping](#looping) - [Lists](#lists) - [Forever](#forever) - [Functions](#functions) - [Operators](#operators) - [+](#) - [string](#string) - [Packages](#packages) - [Iterating](#iterating) - [Built-in functions](#builtin-functions) - [print](#print) - [format](#format) - [len](#len) - [append](#append) - [exit](#exit) - [glob](#glob) - [Standard Library](#standard-library) Here lies a comprehensive reference documentation of nash features and built-in functions, and how to use them. There is also some [examples](./examples) that can be useful. # Command line arguments To handle script arguments you can just use the ARGS variable, that is a list populated with the arguments passed to your script when it is executed, like: ```nash echo echo "acessing individual parameter" var somearg = $ARGS[0] echo $somearg echo ``` # Flow control ## Branching To branch you can use **if** statement, it requires a boolean expression, like the comparison operator: ```nash var a = "nash" echo -n $a if $a == "nash" { a = "rocks" } echo $a #Output:"nashrocks" ``` You can also use a junction of boolean expressions: ```nash a = "nash" b = "rocks" if $a == "nash" && $b == "rocks"{ echo "hellyeah" } #Output:"hellyeah" ``` You can also use a disjunction of boolean expressions: ```nash a = "nash" b = "rocks" if $a == "bash" || $b == "rocks"{ echo "hellyeah" } #Output:"hellyeah" ``` ## Looping Right now there are two kind of loops, on lists and the forever kind :-). ### Lists You can iterate lists like this: ```nash a = "" for i in ("nash" "rocks"){ a = $a + $i } echo $a #Output:"nashrocks" ``` ### Forever It would be cool to loop on boolean expressions, but right now we can only loop forever (besides list looping): ```nash for { echo "hi" } ``` # Functions Defining functions is very easy, for example: ```nash fn concat(a, b) { return $a+$b } res <= concat("1","9") echo $res #Output:"19" ``` If a parameter is missing on the function call, it will fail: ```nash fn concat(a, b) { return $a, $b } res <= concat("1") echo $res #Output:"ERROR: Wrong number of arguments for function concat. Expected 2 but found 1" ``` Passing extra parameters will also fail: ```nash fn concat(a, b) { return $a, $b } res <= concat("1","2","3") echo $res #Output:"ERROR: Wrong number of arguments for function concat. Expected 2 but found 3" ``` # Operators ## + The **+** operator behaviour is dependent on the type its operands. It is always invalid to mix types on the operation (like one operand is a string and the other one is a integer). The language is dynamically typed, but it is strongly typed, types can't be mixed on operations, there is no implicit type coercion. ### string String concatenation is pretty straightforward. For example: ```nash a = "1" b = "2" echo $a+$b #Output:"12" ``` # Packages TODO # Iterating TODO # Built-in functions Built-in functions are functions that are embedded on the language. You do not have to import any package to use them. ## print The function **print** is used to print simple messages directly to stdout: ```nash print("hi") #Output:"hi" ``` And supports formatting: ```nash print("%s:%s", "1", "2") #Output:"1:2" ``` ## format The function **format** is used like **print**, but instead of writing to stdout it will return the string according to the format provided: ```nash a <= format("%s:%s", "1", "2") echo $a #Output:"1:2" ``` ## len The function **len** returns the length of a list. An example to check for the length of a list: ``` echo "define one list with two elements" args = ( "one" "two" ) echo "getting list length" argslen <= len($args) echo $argslen ``` ## append The function **append** appends one element to the end of a exist list. Append returns the updated list. An example to append one element to a exist list: ``` example_list = () echo "appending string 1" example_list <= append($example_list, "1") echo $example_list echo "appending string 2" example_list <= append($example_list, "2") echo $example_list ``` ## exit TODO ## glob TODO # Standard Library The standard library is a set of packages that comes with the nash install (although not obligatory). They must be imported explicitly (as any other package) to be used. * [fmt](docs/stdlib/fmt.md) ================================================ FILE: docs/stdlib/fmt.md ================================================ # Table of Contents - [fmt](#fmt) - [fmt_println](#fmtprintln) # fmt ## fmt_println Same behavior as print but adds a newline on the end. ```nash import fmt fmt_println("hi") #Output:"hi\n" ``` ================================================ FILE: errors/error.go ================================================ package errors import ( "fmt" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/scanner" ) type ( NashError struct { reason string format string } unfinished struct{} unfinishedBlockError struct { *NashError unfinished } unfinishedListError struct { *NashError unfinished } unfinishedCmdError struct { *NashError unfinished } ) func NewError(format string, arg ...interface{}) *NashError { e := &NashError{} e.SetReason(format, arg...) return e } func NewEvalError(path string, node ast.Node, format string, arg ...interface{}) *NashError { linenum := fmt.Sprintf("%s:%d:%d: ", path, node.Line(), node.Column()) return NewError(linenum+format, arg...) } func (e *NashError) SetReason(format string, arg ...interface{}) { e.reason = fmt.Sprintf(format, arg...) } func (e *NashError) Error() string { return e.reason } func (e unfinished) Unfinished() bool { return true } func NewUnfinishedBlockError(name string, it scanner.Token) error { return &unfinishedBlockError{ NashError: NewError("%s:%d:%d: Statement's block '{' not finished", name, it.Line(), it.Column()), } } func NewUnfinishedListError(name string, it scanner.Token) error { return &unfinishedListError{ NashError: NewError("%s:%d:%d: List assignment not finished. Found %v", name, it.Line(), it.Column(), it), } } func NewUnfinishedCmdError(name string, it scanner.Token) error { return &unfinishedCmdError{ NashError: NewError("%s:%d:%d: Multi-line command not finished. Found %v but expect ')'", name, it.Line(), it.Column(), it), } } ================================================ FILE: examples/append.sh ================================================ #!/usr/bin/env nash var example_list = () echo "appending string 1" example_list <= append($example_list, "1") echo $example_list echo "appending string 2" example_list <= append($example_list, "2") echo $example_list ================================================ FILE: examples/args.sh ================================================ #!/usr/bin/env nash print("iterating through the arguments list\n\n") for arg in $ARGS { print("%s\n", $arg) } print("\n") print("acessing individual parameter\n") var somearg = $ARGS[0] print("%s\n", $somearg) ================================================ FILE: examples/init ================================================ #!/usr/bin/env nash # Simple init script for you base your own # For a more complete and organized .nash see: # https://github.com/tiago4orion/dotnash # PROMPT is a special variable used by nash command line to # setup your prompt. var RED = "" var GREEN = "" var RESET = "" var PROMPTSYM = "λ> " var DEFPROMPT = $RED+$PROMPTSYM+$RESET setenv PROMPT = $DEFPROMPT # cd overrides built-in cd # Add the current branch before prompt (if current directory is a git repo) fn cd(path) { var branch = "" var prompt = "" if $path == "" { path = $HOME } chdir($path) var _, status <= test -d ./.git if $status != "0" { prompt = $DEFPROMPT } else { branch <= git rev-parse --abbrev-ref HEAD | xargs echo -n prompt = "("+$GREEN+$branch+$RESET+")"+$DEFPROMPT } setenv PROMPT = $prompt } bindfn cd cd # syntax sugar to cd into go project fn gocd(path) { cd $GOPATH+"/src/"+$path } bindfn gocd gocd ================================================ FILE: examples/len.sh ================================================ #!/usr/bin/env nash echo "args: " echo $ARGS if len($ARGS) == "1" { echo "one parameter passed" } else { echo "more parameters passed" } ================================================ FILE: examples_test.go ================================================ package nash_test import ( "os" "io/ioutil" "github.com/madlambda/nash" ) func Example() { nashpath,cleanup := tmpdir() defer cleanup() nashroot, cleanup := tmpdir() defer cleanup() nash, err := nash.New(nashpath, nashroot) if err != nil { panic(err) } // Execute a script from string err = nash.ExecuteString("-input-", `echo Hello World`) if err != nil { panic(err) } // Output: Hello World } func tmpdir() (string, func()) { dir, err := ioutil.TempDir("", "nash-tests") if err != nil { panic(err) } return dir, func() { err := os.RemoveAll(dir) if err != nil { panic(err) } } } ================================================ FILE: fuzz.go ================================================ // +build gofuzz package nash import "github.com/madlambda/nash/parser" func Fuzz(data []byte) int { p := parser.NewParser("fuzz", string(data)) tree, err := p.Parse() if err != nil { if tree != nil { panic("tree != nil") } return 0 } return 1 } ================================================ FILE: hack/check.sh ================================================ #!/bin/bash set -e go test -race -coverprofile=coverage.txt ./... echo "running stdlib and stdbin tests" tests=$(find ./stdlib ./stdbin -name "*_test.sh") for t in ${tests[*]} do echo echo "running test: "$t ./cmd/nash/nash $t echo "success" done ================================================ FILE: hack/install/unix.sh ================================================ #!/bin/bash set -o errexit set -o nounset set -o pipefail which wget >/dev/null || { echo "wget not installed"; exit 1; } which tar >/dev/null || { echo "tar not installed"; exit 1; } which tr >/dev/null || { echo "tr not found"; exit 1; } NASHROOT=${NASHROOT:-$HOME/nashroot} VERSION="v1.1" ARCH="amd64" OS="$(uname | tr '[:upper:]' '[:lower:]')" if [ $# -eq 1 ]; then VERSION=$1 fi echo "installing nash (${OS}): ${VERSION} at NASHROOT: ${NASHROOT}" mkdir -p $NASHROOT cd $NASHROOT tarfile="nash-${VERSION}-${OS}-${ARCH}.tar.gz" wget -c https://github.com/madlambda/nash/releases/download/$VERSION/$tarfile tar xvfz $tarfile rm -f $tarfile ================================================ FILE: hack/releaser.sh ================================================ #!/usr/bin/env nash if len($ARGS) != "2" { print("usage: %s \n\n", $ARGS[0]) exit("1") } var version = $ARGS[1] var supported_os = ("linux" "darwin" "windows") var supported_arch = ("amd64") # Guarantee passing tests at least on the host arch/os make test mkdir -p dist fn prepare_execs(distfiles, os) { if $os == "windows" { var newfiles = () for distfile in $distfiles { var src = $distfile[0] var dst = $distfile[1] var newsrc = $src+".exe" var newdst = $dst+".exe" newfiles <= append($newfiles, ($newsrc $newdst)) } return $newfiles } if $os == "linux" { for distfile in $distfiles { strip $distfile[0] } } return $distfiles } for os in $supported_os { for arch in $supported_arch { setenv GOOS = $os setenv GOARCH = $arch if $os == "linux" { setenv CGO_ENABLED = "1" } else { setenv CGO_ENABLED = "0" } echo "building OS: "+$GOOS+" ARCH : "+$GOARCH make build "version="+$version var pkgdir <= mktemp -d var bindir = $pkgdir+"/bin" var stdlibdir = $pkgdir+"/stdlib" mkdir -p $bindir mkdir -p $stdlibdir var nash_src = "./cmd/nash/nash" var nash_dst = $bindir+"/nash" var nashfmt_src = "./cmd/nashfmt/nashfmt" var nashfmt_dst = $bindir+"/nashfmt" var execfiles = ( ($nash_src $nash_dst) ($nashfmt_src $nashfmt_dst) ) var execfiles <= prepare_execs($execfiles, $os) # TODO: Improve with glob, right now have only two packages =) var distfiles <= append($execfiles, ("./stdlib/io.sh" $stdlibdir)) distfiles <= append($distfiles, ("./stdlib/map.sh" $stdlibdir)) for distfile in $distfiles { var src = $distfile[0] var dst = $distfile[1] cp -pr $src $dst } var projectdir <= pwd var distar <= format("%s/dist/nash-%s-%s-%s.tar.gz", $projectdir, $version, $os, $arch) chdir($pkgdir) var pkgraw <= ls var pkgfiles <= split($pkgraw, "\n") tar cvfz $distar $pkgfiles chdir($projectdir) } } ================================================ FILE: internal/sh/builtin/append.go ================================================ package builtin import ( "io" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( appendFn struct { obj []sh.Obj args []sh.Obj } ) func newAppend() *appendFn { return &appendFn{} } func (appendfn *appendFn) ArgNames() []sh.FnArg { return []sh.FnArg{ sh.NewFnArg("list", false), sh.NewFnArg("value...", true), // variadic } } func (appendfn *appendFn) Run(in io.Reader, out io.Writer, err io.Writer) ([]sh.Obj, error) { newobj := append(appendfn.obj, appendfn.args...) return []sh.Obj{sh.NewListObj(newobj)}, nil } func (appendfn *appendFn) SetArgs(args []sh.Obj) error { if len(args) < 2 { return errors.NewError("append expects at least two arguments") } obj := args[0] if obj.Type() != sh.ListType { return errors.NewError("append expects a list as first argument, but a %s was provided", obj.Type()) } values := args[1:] if objlist, ok := obj.(*sh.ListObj); ok { appendfn.obj = objlist.List() appendfn.args = values return nil } return errors.NewError("internal error: object of wrong type") } ================================================ FILE: internal/sh/builtin/append_test.go ================================================ package builtin_test import ( "bytes" "testing" "github.com/madlambda/nash/internal/sh/internal/fixture" ) type testcase struct { name string code string expectedErr string expectedStdout string expectedStderr string } func testAppend(t *testing.T, tc testcase) { sh, teardown := fixture.SetupShell(t) defer teardown() var ( outb, errb bytes.Buffer ) sh.SetStdout(&outb) sh.SetStderr(&errb) err := sh.Exec(tc.name, tc.code) stdout := string(outb.Bytes()) stderr := errb.String() if stdout != tc.expectedStdout { t.Errorf("String differs: '%s' != '%s'", tc.expectedStdout, stdout) return } if stderr != tc.expectedStderr { t.Errorf("String differs: '%s' != '%s'", tc.expectedStderr, stderr) return } if err != nil { if err.Error() != tc.expectedErr { t.Fatalf("Expected err '%s' but got '%s'", tc.expectedErr, err.Error()) } } else if tc.expectedErr != "" { t.Fatalf("Expected err '%s' but err is nil", tc.expectedErr) } } func TestAppend(t *testing.T) { for _, tc := range []testcase{ { name: "no argument fails", code: `append()`, expectedErr: ":1:0: append expects at least two arguments", }, { name: "one argument fails", code: `append("1")`, expectedErr: ":1:0: append expects at least two arguments", }, { name: "simple append", code: `var a = () a <= append($a, "hello") a <= append($a, "world") echo -n $a...`, expectedErr: "", expectedStdout: "hello world", expectedStderr: "", }, { name: "append is for lists", code: `var a = "something" a <= append($a, "other") echo -n $a...`, expectedErr: ":2:8: append expects a " + "list as first argument, but a StringType was provided", expectedStdout: "", expectedStderr: "", }, { name: "var args", code: `var a <= append((), "1", "2", "3", "4", "5", "6") echo -n $a...`, expectedErr: "", expectedStdout: "1 2 3 4 5 6", expectedStderr: "", }, { name: "append of lists", code: `var a <= append((), (), ()) if len($a) != "2" { print("wrong") } else if len($a[0]) != "0" { print("wrong") } else if len($a[1]) != "0" { print("wrong") } else { print("ok") }`, expectedErr: "", expectedStdout: "ok", }, } { tc := tc t.Run(tc.name, func(t *testing.T) { testAppend(t, tc) }) } } ================================================ FILE: internal/sh/builtin/chdir.go ================================================ package builtin import ( "fmt" "io" "os" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( chdirFn struct { arg string } ) func newChdir() *chdirFn { return &chdirFn{} } func (chdir *chdirFn) ArgNames() []sh.FnArg { return []sh.FnArg{ sh.NewFnArg("dir", false), } } func (chdir *chdirFn) Run(in io.Reader, out io.Writer, ioerr io.Writer) ([]sh.Obj, error) { err := os.Chdir(chdir.arg) if err != nil { err = fmt.Errorf("builtin: chdir: error[%s] path[%s]", err, chdir.arg) } return nil, err } func (chdir *chdirFn) SetArgs(args []sh.Obj) error { if len(args) != 1 { return errors.NewError("chdir expects one argument, but received %q", args) } obj := args[0] if obj.Type() != sh.StringType { return errors.NewError("chdir expects a string, but a %s was provided", obj.Type()) } if objstr, ok := obj.(*sh.StrObj); ok { chdir.arg = objstr.Str() return nil } return errors.NewError("internal error: object of wrong type") } ================================================ FILE: internal/sh/builtin/doc.go ================================================ // builtin is where all built in functions resides package builtin ================================================ FILE: internal/sh/builtin/exec_test.go ================================================ package builtin_test import ( "testing" "github.com/madlambda/nash/internal/sh/internal/fixture" ) func execSuccess(t *testing.T, scriptContents string) string { shell, cleanup := fixture.SetupShell(t) defer cleanup() out, err := shell.ExecOutput("", scriptContents) if err != nil { t.Fatalf("unexpected err: %s", err) } return string(out) } func execFailure(t *testing.T, scriptContents string) { shell, cleanup := fixture.SetupShell(t) defer cleanup() out, err := shell.ExecOutput("", scriptContents) if err == nil { t.Fatalf("expected err, got success, output: %s", string(out)) } if len(out) > 0 { t.Fatalf("expected empty output, got: %s", string(out)) } } ================================================ FILE: internal/sh/builtin/exit.go ================================================ package builtin import ( "io" "os" "strconv" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( exitFn struct { status int } ) func newExit() Fn { return &exitFn{} } func (e *exitFn) ArgNames() []sh.FnArg { return []sh.FnArg{ sh.NewFnArg("status", false), } } func (e *exitFn) Run(in io.Reader, out io.Writer, err io.Writer) ([]sh.Obj, error) { os.Exit(e.status) return nil, nil //Unrecheable code } func (e *exitFn) SetArgs(args []sh.Obj) error { if len(args) != 1 { return errors.NewError("exit expects 1 argument") } obj := args[0] if obj.Type() != sh.StringType { return errors.NewError( "exit expects a status string, but a %s was provided", obj.Type(), ) } statusstr := obj.(*sh.StrObj).Str() status, err := strconv.Atoi(statusstr) if err != nil { return errors.NewError( "exit:error[%s] converting status[%s] to int", err, statusstr, ) } e.status = status return nil } ================================================ FILE: internal/sh/builtin/exit_test.go ================================================ package builtin_test import ( "os" "os/exec" "testing" ) func TestExit(t *testing.T) { type exitDesc struct { script string status string result int fail bool } tests := map[string]exitDesc{ "success": { script: "./testdata/exit.sh", status: "0", result: 0, }, "failure": { script: "./testdata/exit.sh", status: "1", result: 1, }, } // WHY: We need to run Exec because the script will call the exit syscall, // killing the process (the test process on this case). // When calling Exec we need to guarantee that we are using the nash // built directly from the project, not the one installed on the host. projectnash := "../../../cmd/nash/nash" for name, desc := range tests { t.Run(name, func(t *testing.T) { cmd := exec.Command(projectnash, desc.script, desc.status) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr // to know why scripts were failing cmd.Run() if cmd.ProcessState == nil { t.Fatalf("expected cmd[%v] to have a process state, can't validate status code", cmd) } got := cmd.ProcessState.ExitCode() if desc.result != got { t.Fatalf("expected[%d] got[%d]", desc.result, got) } }) } } ================================================ FILE: internal/sh/builtin/format.go ================================================ package builtin import ( "fmt" "io" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( formatFn struct { fmt string args []interface{} } ) func newFormat() *formatFn { return &formatFn{} } func (f *formatFn) ArgNames() []sh.FnArg { return []sh.FnArg{ sh.NewFnArg("fmt", false), sh.NewFnArg("arg...", true), } } func (f *formatFn) Run(in io.Reader, out io.Writer, err io.Writer) ([]sh.Obj, error) { return []sh.Obj{sh.NewStrObj(fmt.Sprintf(f.fmt, f.args...))}, nil } func (f *formatFn) SetArgs(args []sh.Obj) error { if len(args) == 0 { return errors.NewError("format expects at least 1 argument") } f.fmt = args[0].String() f.args = nil for _, arg := range args[1:] { f.args = append(f.args, arg.String()) } return nil } ================================================ FILE: internal/sh/builtin/format_test.go ================================================ package builtin_test import "testing" func TestFormat(t *testing.T) { type formatDesc struct { script string output string } tests := map[string]formatDesc{ "textonly": { script: ` var r <= format("helloworld") echo $r `, output: "helloworld\n", }, "ncallsRegressionTest": { script: ` fn formatstuff() { var r <= format("hello%s", "world") echo $r } formatstuff() formatstuff() `, output: "helloworld\nhelloworld\n", }, "ncallsWithVarsRegressionTest": { script: ` fn formatstuff() { var b = "world" var r <= format("hello%s", $b) var s <= format("hackthe%s", $b) echo $r echo $s } formatstuff() formatstuff() `, output: "helloworld\nhacktheworld\nhelloworld\nhacktheworld\n", }, "fmtstring": { script: ` var r <= format("%s:%s", "hello", "world") echo $r `, output: "hello:world\n", }, "fmtlist": { script: ` var list = ("1" "2" "3") var r <= format("%s:%s", "list", $list) echo $r `, output: "list:1 2 3\n", }, "funconly": { script: ` fn func() {} var r <= format($func) echo $r `, output: "\n", }, "funcfmt": { script: ` fn func() {} var r <= format("calling:%s", $func) echo $r `, output: "calling:\n", }, "listonly": { script: ` var list = ("1" "2" "3") var r <= format($list) echo $r `, output: "1 2 3\n", }, "listoflists": { script: ` var list = (("1" "2" "3") ("4" "5" "6")) var r <= format("%s:%s", "listoflists", $list) echo $r `, output: "listoflists:1 2 3 4 5 6\n", }, "listasfmt": { script: ` var list = ("%s" "%s") var r <= format($list, "1", "2") echo $r `, output: "1 2\n", }, "invalidFmt": { script: ` var r <= format("%d%s", "invalid") echo $r `, output: "%!d(string=invalid)%!s(MISSING)\n", }, } for name, desc := range tests { t.Run(name, func(t *testing.T) { output := execSuccess(t, desc.script) if output != desc.output { t.Fatalf("got %q expected %q", output, desc.output) } }) } } func TestFormatfErrors(t *testing.T) { type formatDesc struct { script string } tests := map[string]formatDesc{ "noParams": {script: `format()`}, } for name, desc := range tests { t.Run(name, func(t *testing.T) { execFailure(t, desc.script) }) } } ================================================ FILE: internal/sh/builtin/glob.go ================================================ package builtin import ( "fmt" "io" "path/filepath" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( globFn struct { pattern string } ) func newGlob() *globFn { return &globFn{} } func (p *globFn) ArgNames() []sh.FnArg { return []sh.FnArg{sh.NewFnArg("pattern", false)} } func (g *globFn) Run(in io.Reader, out io.Writer, e io.Writer) ([]sh.Obj, error) { listobjs := []sh.Obj{} matches, err := filepath.Glob(g.pattern) if err != nil { return []sh.Obj{ sh.NewListObj([]sh.Obj{}), sh.NewStrObj(fmt.Sprintf("glob:error: %q", err)), }, nil } for _, match := range matches { listobjs = append(listobjs, sh.NewStrObj(match)) } return []sh.Obj{sh.NewListObj(listobjs), sh.NewStrObj("")}, nil } func (g *globFn) SetArgs(args []sh.Obj) error { if len(args) != 1 { return errors.NewError("glob expects 1 string argument (the pattern)") } obj := args[0] if obj.Type() != sh.StringType { return errors.NewError( "glob expects a pattern string, but a %s was provided", obj.Type(), ) } g.pattern = obj.(*sh.StrObj).Str() return nil } ================================================ FILE: internal/sh/builtin/glob_test.go ================================================ package builtin_test import ( "fmt" "io/ioutil" "os" "strings" "testing" ) func setup(t *testing.T) (string, func()) { dir, err := ioutil.TempDir("", "globtest") if err != nil { t.Fatalf("error on setup: %s", err) } return dir, func() { os.RemoveAll(dir) } } func createFile(t *testing.T, path string) { f, err := os.Create(path) if err != nil { t.Fatal(err) } f.WriteString("hi") f.Close() } func TestGlobNoResult(t *testing.T) { dir, teardown := setup(t) defer teardown() pattern := dir + "/*.la" out := execSuccess(t, fmt.Sprintf(` var res, err <= glob("%s") print($res) print($err) print(len($res)) `, pattern)) if out != "0" { t.Fatalf("expected no results, got: %q", out) } } func TestGlobOneResult(t *testing.T) { dir, teardown := setup(t) defer teardown() filename := dir + "/whatever.go" createFile(t, filename) pattern := dir + "/*.go" out := execSuccess(t, fmt.Sprintf(` var res, err <= glob("%s") print($res) print($err) `, pattern)) if out != filename { t.Fatalf("expected %q, got: %q", filename, out) } } func TestGlobMultipleResults(t *testing.T) { dir, teardown := setup(t) defer teardown() filename1 := dir + "/whatever.h" filename2 := dir + "/whatever2.h" createFile(t, filename1) createFile(t, filename2) pattern := dir + "/*.h" out := execSuccess(t, fmt.Sprintf(` var res, err <= glob("%s") print($res) print($err) `, pattern)) res := strings.Split(out, " ") if len(res) != 2 { t.Fatalf("expected 2 results, got: %d", len(res)) } found1 := false found2 := false for _, r := range res { if r == filename1 { found1 = true } if r == filename2 { found2 = true } } if !found1 || !found2 { t.Fatalf("unable to found all files, got: %q", out) } } func TestGlobInvalidPatternError(t *testing.T) { out := execSuccess(t, ` var res, err <= glob("*[.go") print($res) if $err != "" { print("got error") } `) if out != "got error" { t.Fatalf("expected error message on glob, got nothing, output[%s]", out) } } func TestFatalFailure(t *testing.T) { type tcase struct { name string code string } cases := []tcase{ tcase{ name: "noParam", code: ` var res <= glob() print($res) `, }, tcase{ name: "multipleParams", code: ` var res <= glob("/a/*", "/b/*") print($res) `, }, tcase{ name: "wrongType", code: ` var param = ("hi") var res <= glob($param) print($res) `, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { execFailure(t, c.code) }) } } ================================================ FILE: internal/sh/builtin/len.go ================================================ package builtin import ( "io" "strconv" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( lenFn struct { arg sh.Collection } ) func newLen() *lenFn { return &lenFn{} } func (l *lenFn) ArgNames() []sh.FnArg { return []sh.FnArg{ sh.NewFnArg("list", false), } } func lenresult(res int) []sh.Obj { return []sh.Obj{sh.NewStrObj(strconv.Itoa(res))} } func (l *lenFn) Run(in io.Reader, out io.Writer, err io.Writer) ([]sh.Obj, error) { return lenresult(l.arg.Len()), nil } func (l *lenFn) SetArgs(args []sh.Obj) error { if len(args) != 1 { return errors.NewError("lenfn expects one argument") } obj := args[0] col, err := sh.NewCollection(obj) if err != nil { return errors.NewError("len:error[%s]", err) } l.arg = col return nil } ================================================ FILE: internal/sh/builtin/len_test.go ================================================ package builtin_test import ( "bytes" "testing" "github.com/madlambda/nash/internal/sh/internal/fixture" ) func TestLen(t *testing.T) { sh, cleanup := fixture.SetupShell(t) defer cleanup() var out bytes.Buffer sh.SetStdout(&out) err := sh.Exec( "test len", `var a = (1 2 3 4 5 6 7 8 9 0) var len_a <= len($a) echo -n $len_a`, ) if err != nil { t.Error(err) return } if "10" != string(out.Bytes()) { t.Errorf("String differs: '%s' != '%s'", "10", string(out.Bytes())) return } // Having to reset is a bad example of why testing N stuff together // is asking for trouble :-). out.Reset() err = sh.Exec( "test len fail", `a = "test" var l <= len($a) echo -n $l `, ) if err != nil { t.Error(err) return } if "4" != string(out.Bytes()) { t.Errorf("String differs: '%s' != '%s'", "4", string(out.Bytes())) return } } ================================================ FILE: internal/sh/builtin/loader.go ================================================ package builtin import ( "io" "github.com/madlambda/nash/sh" ) // Fn is the contract of a built in function, that is simpler // than the core nash Fn. type ( Fn interface { ArgNames() []sh.FnArg SetArgs(args []sh.Obj) error Run( stdin io.Reader, stdout io.Writer, stderr io.Writer, ) ([]sh.Obj, error) } Constructor func() Fn ) // Constructors returns a map of the builtin function name and its constructor func Constructors() map[string]Constructor { return map[string]Constructor{ "glob": func() Fn { return newGlob() }, "print": func() Fn { return newPrint() }, "format": func() Fn { return newFormat() }, "split": func() Fn { return newSplit() }, "len": func() Fn { return newLen() }, "chdir": func() Fn { return newChdir() }, "append": func() Fn { return newAppend() }, "exit": func() Fn { return newExit() }, } } ================================================ FILE: internal/sh/builtin/print.go ================================================ package builtin import ( "fmt" "io" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( printFn struct { fmt string args []interface{} } ) func newPrint() *printFn { return &printFn{} } func (p *printFn) ArgNames() []sh.FnArg { return []sh.FnArg{ sh.NewFnArg("fmt", false), sh.NewFnArg("arg...", true), } } func (p *printFn) Run(in io.Reader, out io.Writer, err io.Writer) ([]sh.Obj, error) { fmt.Fprintf(out, p.fmt, p.args...) return nil, nil } func (p *printFn) SetArgs(args []sh.Obj) error { if len(args) == 0 { return errors.NewError("print expects at least 1 argument") } p.fmt = args[0].String() p.args = nil for _, arg := range args[1:] { p.args = append(p.args, arg.String()) } return nil } ================================================ FILE: internal/sh/builtin/print_test.go ================================================ package builtin_test import "testing" func TestPrint(t *testing.T) { type printDesc struct { script string output string } tests := map[string]printDesc{ "textonly": { script: `print("helloworld")`, output: "helloworld", }, "nCallsRegresion": { script: `print("%s:%s", "hello", "world")`, output: "hello:world", }, "fmtstring": { script: ` print("%s:%s", "hello", "world") print("%s:%s", "hello", "world") `, output: "hello:worldhello:world", }, "fmtlist": { script: ` var list = ("1" "2" "3") print("%s:%s", "list", $list) `, output: "list:1 2 3", }, "funconly": { script: ` fn func() {} print($func) `, output: "", }, "funcfmt": { script: ` fn func() {} print("calling:%s", $func) `, output: "calling:", }, "listonly": { script: ` var list = ("1" "2" "3") print($list) `, output: "1 2 3", }, "listoflists": { script: ` var list = (("1" "2" "3") ("4" "5" "6")) print("%s:%s", "listoflists", $list) `, output: "listoflists:1 2 3 4 5 6", }, "listasfmt": { script: ` var list = ("%s" "%s") print($list, "1", "2") `, output: "1 2", }, "invalidFmt": { script: `print("%d%s", "invalid")`, output: "%!d(string=invalid)%!s(MISSING)", }, } for name, desc := range tests { t.Run(name, func(t *testing.T) { output := execSuccess(t, desc.script) if output != desc.output { t.Fatalf("got %q expected %q", output, desc.output) } }) } } func TestPrintErrors(t *testing.T) { type printDesc struct { script string } tests := map[string]printDesc{ "noParams": { script: `print()`, }, } for name, desc := range tests { t.Run(name, func(t *testing.T) { execFailure(t, desc.script) }) } } ================================================ FILE: internal/sh/builtin/split.go ================================================ package builtin import ( "io" "strings" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( splitFn struct { content string sep sh.Obj } ) func newSplit() *splitFn { return &splitFn{} } func (s *splitFn) ArgNames() []sh.FnArg { return []sh.FnArg{ sh.NewFnArg("sep", false), sh.NewFnArg("content", false), } } func (s *splitFn) Run(in io.Reader, out io.Writer, err io.Writer) ([]sh.Obj, error) { var output []string content := s.content switch s.sep.Type() { case sh.StringType: sep := s.sep.(*sh.StrObj).Str() output = strings.Split(content, sep) case sh.ListType: sepList := s.sep.(*sh.ListObj).List() output = splitByList(content, sepList) case sh.FnType: sepFn := s.sep.(*sh.FnObj).Fn() output = splitByFn(content, sepFn) default: return nil, errors.NewError("Invalid separator value: %v", s.sep) } listobjs := make([]sh.Obj, len(output)) for i := 0; i < len(output); i++ { listobjs[i] = sh.NewStrObj(output[i]) } return []sh.Obj{sh.NewListObj(listobjs)}, nil } func (s *splitFn) SetArgs(args []sh.Obj) error { if len(args) != 2 { return errors.NewError("split: expects 2 parameters") } if args[0].Type() != sh.StringType { return errors.NewError("split: first parameter must be a string") } content := args[0].(*sh.StrObj) s.content = content.Str() s.sep = args[1] return nil } func splitByList(content string, delims []sh.Obj) []string { return strings.FieldsFunc(content, func(r rune) bool { for _, delim := range delims { if delim.Type() != sh.StringType { continue } objstr := delim.(*sh.StrObj) if len(objstr.Str()) > 0 && rune(objstr.Str()[0]) == r { return true } } return false }) } func splitByFn(content string, splitFunc sh.FnDef) []string { return strings.FieldsFunc(content, func(r rune) bool { fn := splitFunc.Build() arg := sh.NewStrObj(string(r)) fn.SetArgs([]sh.Obj{arg}) err := fn.Start() if err != nil { return false } err = fn.Wait() if err != nil { return false } results := fn.Results() if len(results) != 1 { // expects a single return fn return false } result := results[0] //FIXME: It would be cool to only accept booleans // since the splitter is a predicate if result.Type() != sh.StringType { return false } status := result.(*sh.StrObj) if status.Str() == "0" { return true } return false }) } ================================================ FILE: internal/sh/builtin/split_test.go ================================================ package builtin_test import ( "bytes" "testing" "github.com/madlambda/nash/internal/sh/internal/fixture" ) func TestSplit(t *testing.T) { type splitDesc struct { script string word string sep string result string } tests := map[string]splitDesc{ "space": { script: "./testdata/split.sh", word: "a b c", sep: " ", result: "a\nb\nc\n", }, "pipes": { script: "./testdata/split.sh", word: "1|2|3", sep: "|", result: "1\n2\n3\n", }, "nosplit": { script: "./testdata/split.sh", word: "nospaces", sep: " ", result: "nospaces\n", }, "splitfunc": { script: "./testdata/splitfunc.sh", word: "hah", sep: "a", result: "h\nh\n", }, } for name, desc := range tests { t.Run(name, func(t *testing.T) { var output bytes.Buffer shell, cleanup := fixture.SetupShell(t) defer cleanup() shell.SetStdout(&output) err := shell.ExecFile(desc.script, "mock cmd name", desc.word, desc.sep) if err != nil { t.Fatalf("unexpected err: %s", err) } if output.String() != desc.result { t.Fatalf("got %q expected %q", output.String(), desc.result) } }) } } func TestSplitingByFuncWrongWontPanic(t *testing.T) { // Regression for: https://github.com/madlambda/nash/issues/177 badscripts := map[string]string{ "noReturnValue": ` fn b() { echo "oops" } split("lalala", $b) `, "returnValueIsList": ` fn b() { return () } split("lalala", $b) `, "returnValueIsFunc": ` fn b() { fn x () {} return $x } split("lalala", $b) `, } for testname, badscript := range badscripts { t.Run(testname, func(t *testing.T) { shell, cleanup := fixture.SetupShell(t) defer cleanup() _, err := shell.ExecOutput("whatever", badscript) if err != nil { t.Fatal(err) } }) } } ================================================ FILE: internal/sh/builtin/testdata/exit.sh ================================================ #!/usr/bin/env nash exit($ARGS[1]) ================================================ FILE: internal/sh/builtin/testdata/split.sh ================================================ #!/usr/bin/env nash var word = $ARGS[1] var sep = $ARGS[2] var output <= split($word, $sep) for o in $output { echo $o } ================================================ FILE: internal/sh/builtin/testdata/splitfunc.sh ================================================ #!/usr/bin/env nash var word = $ARGS[1] var sep = $ARGS[2] fn splitter(char) { if $char == $sep { return "0" } return "1" } var output <= split($word, $splitter) for o in $output { echo $o } ================================================ FILE: internal/sh/builtin.go ================================================ package sh import ( "fmt" "io" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/internal/sh/builtin" "github.com/madlambda/nash/sh" ) type ( // builtinFn maps a builtin function to a nash sh.FnDef // avoiding a lot of duplicated code and decoupling the // builtin functions of some unnecessary details on how // the sh.Fn works (lots of complexity to provide features of // other kinds of runners/functions). builtinFn struct { stdin io.Reader stdout, stderr io.Writer done chan struct{} err error results []sh.Obj name string fn builtin.Fn } ) func NewBuiltinFn( name string, fn builtin.Fn, in io.Reader, out io.Writer, outerr io.Writer, ) *builtinFn { return &builtinFn{ name: name, fn: fn, stdin: in, stdout: out, stderr: outerr, } } func (f *builtinFn) Name() string { return f.name } func (f *builtinFn) ArgNames() []sh.FnArg { return f.fn.ArgNames() } func (f *builtinFn) Start() error { f.done = make(chan struct{}) go func() { f.results, f.err = f.fn.Run(f.stdin, f.stdout, f.stderr) f.done <- struct{}{} }() return nil } func (f *builtinFn) Wait() error { <-f.done return f.err } func (f *builtinFn) Results() []sh.Obj { return f.results } func (f *builtinFn) String() string { return fmt.Sprintf("", f.Name()) } func (f *builtinFn) SetArgs(args []sh.Obj) error { return f.fn.SetArgs(args) } func (f *builtinFn) SetEnviron(env []string) { // do nothing // terrible design smell having functions that do nothing =/ } func (f *builtinFn) SetStdin(r io.Reader) { f.stdin = r } func (f *builtinFn) SetStderr(w io.Writer) { f.stderr = w } func (f *builtinFn) SetStdout(w io.Writer) { f.stdout = w } func (f *builtinFn) StdoutPipe() (io.ReadCloser, error) { return nil, errors.NewError("builtin functions doesn't works with pipes") } func (f *builtinFn) Stdin() io.Reader { return f.stdin } func (f *builtinFn) Stdout() io.Writer { return f.stdout } func (f *builtinFn) Stderr() io.Writer { return f.stderr } ================================================ FILE: internal/sh/cmd.go ================================================ package sh import ( "io" "os/exec" "path/filepath" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( // Cmd is a nash command. It has maps of input and output file // descriptors that can be set by SetInputfd and SetOutputfd. // This can be used to pipe execution of Cmd commands. Cmd struct { *exec.Cmd argExprs []ast.Expr } // errCmdNotFound is an error indicating the command wasn't found. errCmdNotFound struct { *errors.NashError } ) func newCmdNotFound(format string, arg ...interface{}) error { e := &errCmdNotFound{ NashError: errors.NewError(format, arg...), } return e } func (e *errCmdNotFound) NotFound() bool { return true } func NewCmd(name string) (*Cmd, error) { var ( err error cmdPath = name ) cmd := Cmd{} if !filepath.IsAbs(name) { cmdPath, err = exec.LookPath(name) if err != nil { return nil, newCmdNotFound(err.Error()) } } cmd.Cmd = &exec.Cmd{ Path: cmdPath, } return &cmd, nil } func (c *Cmd) Stdin() io.Reader { return c.Cmd.Stdin } func (c *Cmd) Stdout() io.Writer { return c.Cmd.Stdout } func (c *Cmd) Stderr() io.Writer { return c.Cmd.Stderr } func (c *Cmd) SetStdin(in io.Reader) { c.Cmd.Stdin = in } func (c *Cmd) SetStdout(out io.Writer) { c.Cmd.Stdout = out } func (c *Cmd) SetStderr(err io.Writer) { c.Cmd.Stderr = err } func (c *Cmd) SetArgs(nodeArgs []sh.Obj) error { args := make([]string, 1, len(nodeArgs)+1) args[0] = c.Path for _, obj := range nodeArgs { if obj.Type() == sh.StringType { objstr := obj.(*sh.StrObj) args = append(args, objstr.Str()) } else if obj.Type() == sh.ListType { objlist := obj.(*sh.ListObj) values := objlist.List() for _, l := range values { if l.Type() != sh.StringType { return errors.NewError("Command arguments requires string or list of strings. But received '%v'", l.String()) } lstr := l.(*sh.StrObj) args = append(args, lstr.Str()) } } else if obj.Type() == sh.FnType { return errors.NewError("Function cannot be passed as argument to commands.") } else { return errors.NewError("Invalid command argument '%v'", obj) } } c.Cmd.Args = args return nil } func (c *Cmd) Args() []ast.Expr { return c.argExprs } func (c *Cmd) SetEnviron(env []string) { c.Cmd.Env = env } func (c *Cmd) Wait() error { err := c.Cmd.Wait() if err != nil { return err } return nil } func (c *Cmd) Start() error { err := c.Cmd.Start() if err != nil { return err } return nil } func (c *Cmd) Results() []sh.Obj { return nil } ================================================ FILE: internal/sh/cmd_test.go ================================================ package sh ================================================ FILE: internal/sh/fncall.go ================================================ package sh import ( "fmt" "io" "os" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/sh" ) type ( FnArg struct { Name string IsVariadic bool } UserFn struct { argNames []sh.FnArg // argNames store parameter name done chan error // for async execution results []sh.Obj name string // debugging purposes parent *Shell subshell *Shell environ []string stdin io.Reader stdout, stderr io.Writer body *ast.Tree repr string closeAfterWait []io.Closer } ) func NewUserFn(name string, args []sh.FnArg, body *ast.Tree, parent *Shell) *UserFn { fn := &UserFn{ name: name, argNames: args, body: body, done: make(chan error), parent: parent, stdin: parent.Stdin(), stdout: parent.Stdout(), stderr: parent.Stderr(), subshell: NewSubShell(name, parent), } fn.subshell.SetTree(fn.body) fn.subshell.SetRepr(fn.repr) fn.subshell.SetDebug(fn.parent.debug) fn.subshell.SetStdout(fn.stdout) fn.subshell.SetStderr(fn.stderr) fn.subshell.SetStdin(fn.stdin) fn.subshell.SetEnviron(fn.environ) return fn } func (fn *UserFn) ArgNames() []sh.FnArg { return fn.argNames } func (fn *UserFn) AddArgName(arg sh.FnArg) { fn.argNames = append(fn.argNames, arg) } func (fn *UserFn) SetArgs(args []sh.Obj) error { var ( isVariadic bool countNormalArgs int ) for i, argName := range fn.argNames { if argName.IsVariadic { if i != len(fn.argNames)-1 { return errors.NewError("variadic expansion must be last argument") } isVariadic = true } else { countNormalArgs++ } } if !isVariadic && len(args) != len(fn.argNames) { return errors.NewError("Wrong number of arguments for function %s. "+ "Expected %d but found %d", fn.name, len(fn.argNames), len(args)) } if isVariadic { if len(args) < countNormalArgs { return errors.NewError("Wrong number of arguments for function %s. "+ "Expected at least %d arguments but found %d", fn.name, countNormalArgs, len(args)) } if len(args) == 0 { // there's only a variadic (optional) argument // and user supplied no argument... // then only initialize the variadic variable to // empty list fn.subshell.Newvar(fn.argNames[0].Name, sh.NewListObj([]sh.Obj{})) return nil } } var i int for i = 0; i < len(fn.argNames) && i < len(args); i++ { arg := args[i] argName := fn.argNames[i].Name isVariadic := fn.argNames[i].IsVariadic if isVariadic { var valist []sh.Obj for ; i < len(args); i++ { arg = args[i] valist = append(valist, arg) } valistarg := sh.NewListObj(valist) fn.subshell.Newvar(argName, valistarg) } else { fn.subshell.Newvar(argName, arg) } } // set remaining (variadic) list if len(fn.argNames) > 0 && i < len(fn.argNames) { last := fn.argNames[len(fn.argNames)-1] if !last.IsVariadic { return errors.NewError("internal error: optional arguments only for variadic parameter") } fn.subshell.Newvar(last.Name, sh.NewListObj([]sh.Obj{})) } return nil } func (fn *UserFn) Name() string { return fn.name } func (fn *UserFn) SetRepr(repr string) { fn.repr = repr } func (fn *UserFn) closeDescriptors(closers []io.Closer) { for _, fd := range closers { fd.Close() } } func (fn *UserFn) execute() ([]sh.Obj, error) { if fn.body != nil { return fn.subshell.ExecuteTree(fn.body) } return nil, fmt.Errorf("fn not properly created") } func (fn *UserFn) Start() error { go func() { var err error fn.results, err = fn.execute() fn.done <- err }() return nil } func (fn *UserFn) Results() []sh.Obj { return fn.results } func (fn *UserFn) Wait() error { err := <-fn.done fn.closeDescriptors(fn.closeAfterWait) fn.subshell = nil return err } func (fn *UserFn) SetEnviron(env []string) { fn.environ = env } func (fn *UserFn) SetStderr(w io.Writer) { fn.stderr = w } func (fn *UserFn) SetStdout(w io.Writer) { fn.stdout = w } func (fn *UserFn) SetStdin(r io.Reader) { fn.stdin = r } func (fn *UserFn) Stdin() io.Reader { return fn.stdin } func (fn *UserFn) Stdout() io.Writer { return fn.stdout } func (fn *UserFn) Stderr() io.Writer { return fn.stderr } func (fn *UserFn) String() string { if fn.body != nil { return fn.body.String() } panic("fn not initialized") } func (fn *UserFn) StdoutPipe() (io.ReadCloser, error) { pr, pw, err := os.Pipe() if err != nil { return nil, err } fn.subshell.SetStdout(pw) // As fn doesn't fork, both fd can be closed after wait is called fn.closeAfterWait = append(fn.closeAfterWait, pw, pr) return pr, nil } ================================================ FILE: internal/sh/fndef.go ================================================ package sh import ( "io" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/internal/sh/builtin" "github.com/madlambda/nash/sh" ) type ( fnDef struct { name string Parent *Shell Body *ast.Tree argNames []sh.FnArg stdin io.Reader stdout, stderr io.Writer environ []string } userFnDef struct { *fnDef } builtinFnDef struct { *fnDef constructor builtin.Constructor } ) // newFnDef creates a new function definition func newFnDef(name string, parent *Shell, args []*ast.FnArgNode, body *ast.Tree) (*fnDef, error) { fn := fnDef{ name: name, Parent: parent, Body: body, stdin: parent.stdin, stdout: parent.stdout, stderr: parent.stderr, } for i := 0; i < len(args); i++ { arg := args[i] if i < len(args)-1 && arg.IsVariadic { return nil, errors.NewEvalError(parent.filename, arg, "Vararg '%s' isn't the last argument", arg.String()) } fn.argNames = append(fn.argNames, sh.FnArg{arg.Name, arg.IsVariadic}) } return &fn, nil } func (fnDef *fnDef) Name() string { return fnDef.name } func (fnDef *fnDef) ArgNames() []sh.FnArg { return fnDef.argNames } func (fnDef *fnDef) Environ() []string { return fnDef.environ } func (fnDef *fnDef) SetEnviron(env []string) { fnDef.environ = env } func (fnDef *fnDef) SetStdin(r io.Reader) { fnDef.stdin = r } func (fnDef *fnDef) SetStderr(w io.Writer) { fnDef.stderr = w } func (fnDef *fnDef) SetStdout(w io.Writer) { fnDef.stdout = w } func (fnDef *fnDef) Stdin() io.Reader { return fnDef.stdin } func (fnDef *fnDef) Stdout() io.Writer { return fnDef.stdout } func (fnDef *fnDef) Stderr() io.Writer { return fnDef.stderr } func newUserFnDef(name string, parent *Shell, args []*ast.FnArgNode, body *ast.Tree) (*userFnDef, error) { fnDef, err := newFnDef(name, parent, args, body) if err != nil { return nil, err } ufndef := userFnDef{ fnDef: fnDef, } return &ufndef, nil } func (ufnDef *userFnDef) Build() sh.Fn { userfn := NewUserFn(ufnDef.Name(), ufnDef.ArgNames(), ufnDef.Body, ufnDef.Parent) userfn.SetStdin(ufnDef.stdin) userfn.SetStdout(ufnDef.stdout) userfn.SetStderr(ufnDef.stderr) userfn.SetEnviron(ufnDef.environ) return userfn } func newBuiltinFnDef(name string, parent *Shell, constructor builtin.Constructor) *builtinFnDef { return &builtinFnDef{ fnDef: &fnDef{ name: name, stdin: parent.stdin, stdout: parent.stdout, stderr: parent.stderr, }, constructor: constructor, } } func (bfnDef *builtinFnDef) Build() sh.Fn { return NewBuiltinFn(bfnDef.Name(), bfnDef.constructor(), bfnDef.stdin, bfnDef.stdout, bfnDef.stderr, ) } ================================================ FILE: internal/sh/functions_test.go ================================================ package sh_test import "testing" func TestFunctionsClosures(t *testing.T) { for _, test := range []execTestCase{ { desc: "simpleClosure", code: ` fn func(a) { fn closure() { print($a) } return $closure } var x <= func("1") var y <= func("2") $x() $y() `, expectedStdout: "12", }, { desc: "eachCallCreatesNewVar", code: ` fn func() { var a = () fn add(elem) { a <= append($a, $elem) print("a:%s,",$a) } return $add } var add <= func() $add("1") $add("3") $add("5") `, expectedStdout: "a:1,a:1 3,a:1 3 5,", }, { desc: "adder example", code: ` fn makeAdder(x) { fn add(y) { var ret <= expr $x "+" $y return $ret } return $add } var add1 <= makeAdder("1") var add5 <= makeAdder("5") var add1000 <= makeAdder("1000") print("%s\n", add5("5")) print("%s\n", add5("10")) print("%s\n", add1("10")) print("%s\n", add1("2")) print("%s\n", add1000("50")) print("%s\n", add1000("100")) print("%s\n", add1("10")) `, expectedStdout: `10 15 11 3 1050 1100 11 `, }, { desc: "logger", code: `fn getlogger(prefix) { fn log(fmt, args...) { print($prefix+$fmt+"\n", $args...) } return $log } var info <= getlogger("[info] ") var error <= getlogger("[error] ") var warn <= getlogger("[warn] ") $info("nuke initialized successfully") $warn("temperature above anormal circunstances: %s°", "870") $error("about to explode...") `, expectedStdout: `[info] nuke initialized successfully [warn] temperature above anormal circunstances: 870° [error] about to explode... `, }, } { t.Run(test.desc, func(t *testing.T) { testExec(t, test) }) } } func TestFunctionsVariables(t *testing.T) { for _, test := range []execTestCase{ { desc: "fn stored only as vars", code: ` fn func(a) { echo -n "hello" } func = "teste" echo -n $func func() `, expectedStdout: "teste", expectedErr: ":8:4: Identifier 'func' is not a function", }, } { t.Run(test.desc, func(t *testing.T) { testExec(t, test) }) } } func TestFunctionsStateless(t *testing.T) { for _, test := range []execTestCase{ { desc: "functions have no shared state", code: `fn iter(first, last, func) { var sequence <= seq $first $last var range <= split($sequence, "\n") for i in $range { $func($i) } } fn create_vm(index) { echo "create_vm: "+$index iter("1", "3", $create_disk) } fn create_disk(index) { echo "create_disk: " + $index } iter("1", "2", $create_vm) `, expectedStdout: `create_vm: 1 create_disk: 1 create_disk: 2 create_disk: 3 create_vm: 2 create_disk: 1 create_disk: 2 create_disk: 3 `, }, } { t.Run(test.desc, func(t *testing.T) { testExec(t, test) }) } } ================================================ FILE: internal/sh/internal/fixture/fixture.go ================================================ package fixture import ( "testing" "path/filepath" "github.com/madlambda/nash" "github.com/madlambda/nash/internal/testing/fixture" ) type NashDirs struct { Path string Lib string Root string Stdlib string Cleanup func() } var MkdirAll func(*testing.T, string) = fixture.MkdirAll var Tmpdir func(*testing.T) (string, func()) = fixture.Tmpdir func SetupShell(t *testing.T) (*nash.Shell, func()) { dirs := SetupNashDirs(t) shell, err := nash.New(dirs.Path, dirs.Root) if err != nil { dirs.Cleanup() t.Fatal(err) } return shell, dirs.Cleanup } func SetupNashDirs(t *testing.T) NashDirs { testdir, rmdir := Tmpdir(t) nashpath := filepath.Join(testdir, "nashpath") nashroot := filepath.Join(testdir, "nashroot") nashlib := filepath.Join(nashpath, "lib") nashstdlib := filepath.Join(nashroot, "stdlib") MkdirAll(t, nashlib) MkdirAll(t, nashstdlib) return NashDirs{ Path: nashpath, Lib: nashlib, Root: nashroot, Stdlib: nashstdlib, Cleanup: rmdir, } } ================================================ FILE: internal/sh/ioutils_test.go ================================================ package sh_test import ( "os" "testing" "io/ioutil" ) func writeFile(t *testing.T, filename string, data string) { err := ioutil.WriteFile(filename, []byte(data), os.ModePerm) if err != nil { t.Fatal(err) } } func chdir(t *testing.T, dir string) { t.Helper() err := os.Chdir(dir) if err != nil { t.Fatal(err) } } func getwd(t *testing.T) string { t.Helper() dir, err := os.Getwd() if err != nil { t.Fatal(err) } return dir } ================================================ FILE: internal/sh/log.go ================================================ package sh import ( "log" "os" ) // LogFn is the logger type type LogFn func(format string, args ...interface{}) // NewLog creates a new nash logger func NewLog(ns string, enable bool) LogFn { logger := log.New(os.Stderr, "", 0) return func(format string, args ...interface{}) { if enable { logger.Printf("["+ns+"] "+format, args...) } } } ================================================ FILE: internal/sh/rfork.go ================================================ // +build !linux,!plan9 // package sh import ( "github.com/madlambda/nash/ast" "github.com/madlambda/nash/errors" ) func (sh *Shell) executeRfork(rfork *ast.RforkNode) error { return errors.NewError("rfork only supported on Linux and Plan9") } ================================================ FILE: internal/sh/rfork_linux.go ================================================ // +build linux // nash provides the execution engine package sh import ( "fmt" "io" "net" "os" "os/exec" "strconv" "syscall" "time" "github.com/madlambda/nash/ast" ) func getProcAttrs(flags uintptr) *syscall.SysProcAttr { uid := os.Getuid() gid := os.Getgid() sysproc := &syscall.SysProcAttr{ Cloneflags: flags, } if (flags & syscall.CLONE_NEWUSER) == syscall.CLONE_NEWUSER { sysproc.UidMappings = []syscall.SysProcIDMap{ { ContainerID: 0, HostID: uid, Size: 1, }, } sysproc.GidMappings = []syscall.SysProcIDMap{ { ContainerID: 0, HostID: gid, Size: 1, }, } } return sysproc } func dialRc(sockpath string) (net.Conn, error) { retries := 0 retryRforkDial: client, err := net.Dial("unix", sockpath) if err != nil { if retries < 3 { retries++ time.Sleep(time.Duration(retries) * time.Second) goto retryRforkDial } } return client, err } // executeRfork executes the calling program again but passing // a new name for the process on os.Args[0] and passing an unix // socket file to communicate to. func (sh *Shell) executeRfork(rfork *ast.RforkNode) error { var ( tr *ast.Tree i int nashClient net.Conn copyOut, copyErr bool ) if sh.stdout != os.Stdout { copyOut = true } if sh.stderr != os.Stderr { copyErr = true } if sh.nashdPath == "" { return fmt.Errorf("Nashd not set") } unixfile := "/tmp/nash." + randRunes(4) + ".sock" cmd := exec.Cmd{ Path: sh.nashdPath, Args: append([]string{"-nashd-"}, "-noinit", "-addr", unixfile), Env: buildenv(sh.Environ()), } arg := rfork.Arg() forkFlags, err := getflags(arg.Value()) if err != nil { return err } cmd.SysProcAttr = getProcAttrs(forkFlags) stdoutDone := make(chan bool) stderrDone := make(chan bool) var ( stdout, stderr io.ReadCloser ) if copyOut { stdout, err = cmd.StdoutPipe() if err != nil { return err } } else { cmd.Stdout = sh.stdout close(stdoutDone) } if copyErr { stderr, err = cmd.StderrPipe() if err != nil { return err } } else { cmd.Stderr = sh.stderr close(stderrDone) } cmd.Stdin = sh.stdin err = cmd.Start() if err != nil { return err } if copyOut { go func() { defer close(stdoutDone) io.Copy(sh.stdout, stdout) }() } if copyErr { go func() { defer close(stderrDone) io.Copy(sh.stderr, stderr) }() } nashClient, err = dialRc(unixfile) defer nashClient.Close() tr = rfork.Tree() if tr == nil || tr.Root == nil { return fmt.Errorf("Rfork with no sub block") } for i = 0; i < len(tr.Root.Nodes); i++ { var ( n, status int ) node := tr.Root.Nodes[i] data := []byte(node.String() + "\n") n, err = nashClient.Write(data) if err != nil || n != len(data) { return fmt.Errorf("RPC call failed: Err: %v, bytes written: %d", err, n) } // read response var response [1024]byte n, err = nashClient.Read(response[:]) if err != nil { break } status, err = strconv.Atoi(string(response[0:n])) if err != nil { err = fmt.Errorf("Invalid status: %s", string(response[0:n])) break } if status != 0 { err = fmt.Errorf("nash: Exited with status %d", status) break } } // we're done with rfork daemon nashClient.Write([]byte("quit")) <-stdoutDone <-stderrDone err2 := cmd.Wait() if err != nil { return err } if err2 != nil { return err2 } return nil } func getflags(flags string) (uintptr, error) { var ( lflags uintptr ) for i := 0; i < len(flags); i++ { switch flags[i] { case 'c': lflags |= (syscall.CLONE_NEWUSER | syscall.CLONE_NEWPID | syscall.CLONE_NEWNET | syscall.CLONE_NEWNS | syscall.CLONE_NEWUTS | syscall.CLONE_NEWIPC) case 'u': lflags |= syscall.CLONE_NEWUSER case 'p': lflags |= syscall.CLONE_NEWPID case 'n': lflags |= syscall.CLONE_NEWNET case 'm': lflags |= syscall.CLONE_NEWNS case 's': lflags |= syscall.CLONE_NEWUTS case 'i': lflags |= syscall.CLONE_NEWIPC default: return 0, fmt.Errorf("Wrong rfork flag: %c", flags[i]) } } if lflags == 0 { return 0, fmt.Errorf("Rfork requires some flag") } return lflags, nil } ================================================ FILE: internal/sh/rfork_linux_test.go ================================================ // +build linux package sh import ( "bytes" "strings" "syscall" "testing" ) func getletters() string { var a bytes.Buffer for i := 'a'; i < 'z'; i++ { a.Write(append([]byte{}, byte(i))) } all := string(a.Bytes()) allCap := strings.ToUpper(all) return all + allCap } func getvalid() string { return "cumnpsi" } func testTblFlagsOK(flagstr string, expected uintptr, t *testing.T) { flags, err := getflags(flagstr) if err != nil { t.Error(err) return } if flags != expected { t.Errorf("Flags differ: expected %08x but %08x", expected, flags) return } } func TestRforkFlags(t *testing.T) { _, err := getflags("") if err == nil { t.Error("Empty flags should return error") return } _, err = getflags("a") if err == nil { t.Error("Unknow flag a") return } allchars := getletters() _, err = getflags(allchars) if err == nil { t.Error("Should fail") return } testTblFlagsOK("u", syscall.CLONE_NEWUSER, t) testTblFlagsOK("m", syscall.CLONE_NEWNS, t) testTblFlagsOK("n", syscall.CLONE_NEWNET, t) testTblFlagsOK("i", syscall.CLONE_NEWIPC, t) testTblFlagsOK("s", syscall.CLONE_NEWUTS, t) testTblFlagsOK("p", syscall.CLONE_NEWPID, t) testTblFlagsOK("c", syscall.CLONE_NEWUSER| syscall.CLONE_NEWNS|syscall.CLONE_NEWNET| syscall.CLONE_NEWIPC|syscall.CLONE_NEWUTS| syscall.CLONE_NEWUSER|syscall.CLONE_NEWPID, t) testTblFlagsOK("um", syscall.CLONE_NEWUSER|syscall.CLONE_NEWNS, t) testTblFlagsOK("umn", syscall.CLONE_NEWUSER| syscall.CLONE_NEWNS| syscall.CLONE_NEWNET, t) testTblFlagsOK("umni", syscall.CLONE_NEWUSER| syscall.CLONE_NEWNS| syscall.CLONE_NEWNET| syscall.CLONE_NEWIPC, t) testTblFlagsOK("umnip", syscall.CLONE_NEWUSER| syscall.CLONE_NEWNS| syscall.CLONE_NEWNET| syscall.CLONE_NEWIPC| syscall.CLONE_NEWPID, t) testTblFlagsOK("umnips", syscall.CLONE_NEWUSER| syscall.CLONE_NEWNS| syscall.CLONE_NEWNET| syscall.CLONE_NEWIPC| syscall.CLONE_NEWPID| syscall.CLONE_NEWUTS, t) } ================================================ FILE: internal/sh/rfork_plan9.go ================================================ // +build plan9 package sh import ( "fmt" "syscall" ) func (sh *Shell) executeRfork(rfork *RforkNode) error { return newError("Sorry. Plan9 rfork not implemented yet.") } // getflags converts to Plan9 flags func getflags(flags string) (uintptr, error) { var ( pflags uintptr ) for i := 0; i < len(flags); i++ { switch flags[i] { case 'n': pflags |= syscall.RFNAMEG case 'N': pflags |= syscall.RFCNAMEG case 'e': pflags |= syscall.RFENVG case 'E': pflags |= syscall.RFCENVG case 's': pflags |= syscall.RFNOTEG case 'f': pflags |= syscall.RFFDG case 'F': pflags |= syscall.RFCFDG default: return 0, fmt.Errorf("Wrong rfork flag: %c", flags[i]) } } if pflags == 0 { return 0, fmt.Errorf("Rfork requires some flag") } return pflags, nil } ================================================ FILE: internal/sh/shell.go ================================================ package sh import ( "bytes" "fmt" "io" "io/ioutil" "net" "os" "os/signal" "path/filepath" "runtime" "strconv" "strings" "sync" "syscall" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/internal/sh/builtin" "github.com/madlambda/nash/parser" "github.com/madlambda/nash/sh" "github.com/madlambda/nash/token" ) const ( logNS = "nashell.Shell" defPrompt = "\033[31mλ>\033[0m " ) type ( // Env is the environment map of lists Env map[string]sh.Obj Var Env Fns map[string]sh.FnDef StatusCode uint8 // Shell is the core data structure. Shell struct { name string debug bool interactive bool abortOnErr bool logf LogFn nashdPath string isFn bool filename string // current file being executed or imported sigs chan os.Signal interrupted bool looping bool stdin io.Reader stdout io.Writer stderr io.Writer env Env vars Var binds Fns root *ast.Tree parent *Shell repr string // string representation nashpath string nashroot string *sync.Mutex } errIgnore struct { *errors.NashError } errInterrupted struct { *errors.NashError } errStopWalking struct { *errors.NashError } ) const ( ESuccess StatusCode = 0 ENotFound = 127 ENotStarted = 255 ) func newErrIgnore(format string, arg ...interface{}) error { e := &errIgnore{ NashError: errors.NewError(format, arg...), } return e } func (e *errIgnore) Ignore() bool { return true } func newErrInterrupted(format string, arg ...interface{}) error { return &errInterrupted{ NashError: errors.NewError(format, arg...), } } func (e *errInterrupted) Interrupted() bool { return true } func newErrStopWalking() *errStopWalking { return &errStopWalking{ NashError: errors.NewError("return"), } } func (e *errStopWalking) StopWalking() bool { return true } func NewAbortShell(nashpath string, nashroot string) (*Shell, error) { return newShell(nashpath, nashroot, true) } // NewShell creates a new shell object // nashpath will be used to search libraries and nashroot will be used to // search for the standard library shipped with the language. func NewShell(nashpath string, nashroot string) (*Shell, error) { return newShell(nashpath, nashroot, false) } func newShell(nashpath string, nashroot string, abort bool) (*Shell, error) { shell := &Shell{ name: "parent scope", interactive: false, abortOnErr: abort, isFn: false, logf: NewLog(logNS, false), nashdPath: nashdAutoDiscover(), stdout: os.Stdout, stderr: os.Stderr, stdin: os.Stdin, env: make(Env), vars: make(Var), binds: make(Fns), Mutex: &sync.Mutex{}, sigs: make(chan os.Signal, 1), filename: "", nashpath: nashpath, nashroot: nashroot, } err := shell.setup() if err != nil { return nil, err } shell.setupSignals() err = validateDirs(nashpath, nashroot) if err != nil { if shell.abortOnErr { return nil, err } printerr := func(msg string) { shell.Stderr().Write([]byte(msg + "\n")) } printerr(err.Error()) printerr("please check your NASHPATH and NASHROOT so they point to valid locations") } return shell, nil } // NewSubShell creates a nashell.Shell that inherits the parent shell stdin, // stdout, stderr and mutex lock. // Every variable and function lookup is done first in the subshell and then, if // not found, in the parent shell recursively. func NewSubShell(name string, parent *Shell) *Shell { return &Shell{ name: name, isFn: true, parent: parent, logf: NewLog(logNS, false), nashdPath: nashdAutoDiscover(), stdout: parent.Stdout(), stderr: parent.Stderr(), stdin: parent.Stdin(), env: make(Env), vars: make(Var), binds: make(Fns), Mutex: parent.Mutex, filename: parent.filename, } } func (shell *Shell) NashPath() string { return shell.nashpath } // initEnv creates a new environment from old one func (shell *Shell) initEnv(processEnv []string) error { largs := make([]sh.Obj, len(os.Args)) for i := 0; i < len(os.Args); i++ { largs[i] = sh.NewStrObj(os.Args[i]) } argv := sh.NewListObj(largs) shell.Setenv("argv", argv) shell.Newvar("argv", argv) for _, penv := range processEnv { var value sh.Obj p := strings.Split(penv, "=") if len(p) >= 2 { // TODO(i4k): handle lists correctly in the future // argv is not special, every list must be handled correctly if p[0] == "argv" { continue } value = sh.NewStrObj(strings.Join(p[1:], "=")) shell.Setenv(p[0], value) shell.Newvar(p[0], value) } } pidVal := sh.NewStrObj(strconv.Itoa(os.Getpid())) shell.Setenv("PID", pidVal) shell.Newvar("PID", pidVal) if _, ok := shell.Getenv("SHELL"); !ok { shellVal := sh.NewStrObj(nashdAutoDiscover()) shell.Setenv("SHELL", shellVal) shell.Newvar("SHELL", shellVal) } cwd, err := os.Getwd() if err != nil { return err } cwdObj := sh.NewStrObj(cwd) shell.Setenv("PWD", cwdObj) shell.Newvar("PWD", cwdObj) return nil } // Reset internal state func (shell *Shell) Reset() { shell.vars = make(Var) shell.env = make(Env) shell.binds = make(Fns) } // SetDebug enable/disable debug in the shell func (shell *Shell) SetDebug(d bool) { shell.debug = d shell.logf = NewLog(logNS, d) } // SetInteractive enable/disable shell interactive mode func (shell *Shell) SetInteractive(i bool) { shell.interactive = i if i { _ = shell.setupDefaultBindings() } } func (shell *Shell) Interactive() bool { if shell.parent != nil { return shell.parent.Interactive() } return shell.interactive } func (shell *Shell) SetName(a string) { shell.name = a } func (shell *Shell) Name() string { return shell.name } func (shell *Shell) SetParent(a *Shell) { shell.parent = a } func (shell *Shell) Environ() Env { if shell.parent != nil { return shell.parent.Environ() } return shell.env } func (shell *Shell) Getenv(name string) (sh.Obj, bool) { if shell.parent != nil { return shell.parent.Getenv(name) } value, ok := shell.env[name] return value, ok } func (shell *Shell) Setenv(name string, value sh.Obj) { if shell.parent != nil { shell.parent.Setenv(name, value) return } shell.Newvar(name, value) shell.env[name] = value os.Setenv(name, value.String()) } func (shell *Shell) SetEnviron(processEnv []string) { shell.env = make(Env) for _, penv := range processEnv { var value sh.Obj p := strings.Split(penv, "=") if len(p) == 2 { value = sh.NewStrObj(p[1]) shell.Setenv(p[0], value) shell.Newvar(p[0], value) } } } // GetLocalvar returns a local scoped variable. func (shell *Shell) GetLocalvar(name string) (sh.Obj, bool) { v, ok := shell.vars[name] return v, ok } // Getvar returns the value of the variable name. It could look in their // parent scopes if not found locally. func (shell *Shell) Getvar(name string) (sh.Obj, bool) { if value, ok := shell.vars[name]; ok { return value, ok } if shell.parent != nil { return shell.parent.Getvar(name) } return nil, false } // GetFn returns the function name or error if not found. func (shell *Shell) GetFn(name string) (*sh.FnObj, error) { shell.logf("Looking for function '%s' on shell '%s'\n", name, shell.name) if obj, ok := shell.vars[name]; ok { if obj.Type() == sh.FnType { fnObj := obj.(*sh.FnObj) return fnObj, nil } return nil, errors.NewError("Identifier '%s' is not a function", name) } if shell.parent != nil { return shell.parent.GetFn(name) } return nil, fmt.Errorf("function '%s' not found", name) } func (shell *Shell) Setbindfn(name string, value sh.FnDef) { shell.binds[name] = value } func (shell *Shell) Getbindfn(cmdName string) (sh.FnDef, bool) { if fn, ok := shell.binds[cmdName]; ok { return fn, true } if shell.parent != nil { return shell.parent.Getbindfn(cmdName) } return nil, false } // Newvar creates a new variable in the scope. func (shell *Shell) Newvar(name string, value sh.Obj) { shell.vars[name] = value } // Setvar updates the value of an existing variable. If the variable // doesn't exist in current scope it looks up in their parent scopes. // It returns true if the variable was found and updated. func (shell *Shell) Setvar(name string, value sh.Obj) bool { _, ok := shell.vars[name] if ok { shell.vars[name] = value return true } if shell.parent != nil { return shell.parent.Setvar(name, value) } return false } func (shell *Shell) IsFn() bool { return shell.isFn } func (shell *Shell) SetIsFn(b bool) { shell.isFn = b } // SetNashdPath sets an alternativa path to nashd func (shell *Shell) SetNashdPath(path string) { shell.nashdPath = path } // SetStdin sets the stdin for commands func (shell *Shell) SetStdin(in io.Reader) { shell.stdin = in } // SetStdout sets stdout for commands func (shell *Shell) SetStdout(out io.Writer) { shell.stdout = out } // SetStderr sets stderr for commands func (shell *Shell) SetStderr(err io.Writer) { shell.stderr = err } func (shell *Shell) Stdout() io.Writer { return shell.stdout } func (shell *Shell) Stderr() io.Writer { return shell.stderr } func (shell *Shell) Stdin() io.Reader { return shell.stdin } // SetTree sets the internal tree of the interpreter. It's used for // sub-shells like `fn`. func (shell *Shell) SetTree(t *ast.Tree) { shell.root = t } // Tree returns the internal tree of the subshell. func (shell *Shell) Tree() *ast.Tree { return shell.root } // SetRepr set the string representation of the shell func (shell *Shell) SetRepr(a string) { shell.repr = a } func (shell *Shell) setupBuiltin() { for name, constructor := range builtin.Constructors() { fnDef := newBuiltinFnDef(name, shell, constructor) shell.Newvar(name, sh.NewFnObj(fnDef)) } } func (shell *Shell) setupDefaultBindings() error { // only one builtin fn... no need for advanced machinery yet homeEnvVar := "HOME" if runtime.GOOS == "windows" { homeEnvVar = "HOMEPATH" } err := shell.Exec(shell.name, fmt.Sprintf(`fn nash_builtin_cd(args...) { var path = "" var l <= len($args) if $l == "0" { path = $%s } else { path = $args[0] } chdir($path) } bindfn nash_builtin_cd cd`, homeEnvVar)) return err } func (shell *Shell) setup() error { err := shell.initEnv(os.Environ()) if err != nil { return err } if shell.env["PROMPT"] == nil { pobj := sh.NewStrObj(defPrompt) shell.Setenv("PROMPT", pobj) shell.Newvar("PROMPT", pobj) } _, ok := shell.Getvar("_") if !ok { shell.Newvar("_", sh.NewStrObj("")) } shell.setupBuiltin() return err } func (shell *Shell) setupSignals() { signal.Notify(shell.sigs, syscall.SIGINT) go func() { for { sig := <-shell.sigs switch sig { case syscall.SIGINT: shell.Lock() // TODO(i4k): Review implementation when interrupted inside // function loops if shell.looping { shell.setIntr(true) } shell.Unlock() default: fmt.Printf("%s\n", sig) } } }() } // TriggerCTRLC mock the user pressing CTRL-C in the terminal func (shell *Shell) TriggerCTRLC() error { p, err := os.FindProcess(os.Getpid()) if err != nil { return err } return p.Signal(syscall.SIGINT) } // setIntr *do not lock*. You must do it yourself! func (shell *Shell) setIntr(b bool) { if shell.parent != nil { shell.parent.setIntr(b) return } shell.interrupted = b } // getIntr returns true if nash was interrupted by CTRL-C func (shell *Shell) getIntr() bool { if shell.parent != nil { return shell.parent.getIntr() } return shell.interrupted } // Exec executes the commands specified by string content func (shell *Shell) Exec(path, content string) error { p := parser.NewParser(path, content) tr, err := p.Parse() if err != nil { return err } _, err = shell.ExecuteTree(tr) return err } // Execute the nash file at given path func (shell *Shell) ExecFile(path string) error { bkCurFile := shell.filename content, err := ioutil.ReadFile(path) if err != nil { return err } shell.filename = path defer func() { shell.filename = bkCurFile }() return shell.Exec(path, string(content)) } func (shell *Shell) newvar(name *ast.NameNode, value sh.Obj) error { if name.Index == nil { shell.Newvar(name.Ident, value) return nil } // handles ident[x] = v obj, ok := shell.Getvar(name.Ident) if !ok { return errors.NewEvalError(shell.filename, name, "Variable %s not found", name.Ident) } index, err := shell.evalIndex(name.Index) if err != nil { return err } col, err := sh.NewWriteableCollection(obj) if err != nil { return errors.NewEvalError(shell.filename, name, err.Error()) } err = col.Set(index, value) if err != nil { return errors.NewEvalError( shell.filename, name, "error[%s] setting var", err, ) } shell.Newvar(name.Ident, obj) return nil } func (shell *Shell) setvar(name *ast.NameNode, value sh.Obj) error { if name.Index == nil { if !shell.Setvar(name.Ident, value) { return errors.NewEvalError(shell.filename, name, "Variable '%s' is not initialized. Use 'var %s = '", name, name) } return nil } obj, ok := shell.Getvar(name.Ident) if !ok { return errors.NewEvalError(shell.filename, name, "Variable %s not found", name.Ident) } index, err := shell.evalIndex(name.Index) if err != nil { return err } col, err := sh.NewWriteableCollection(obj) if err != nil { return errors.NewEvalError(shell.filename, name, err.Error()) } err = col.Set(index, value) if err != nil { return errors.NewEvalError( shell.filename, name, "error[%s] setting var", err, ) } if !shell.Setvar(name.Ident, obj) { return errors.NewEvalError(shell.filename, name, "Variable '%s' is not initialized. Use 'var %s = '", name, name) } return nil } func (shell *Shell) setvars(names []*ast.NameNode, values []sh.Obj) error { for i := 0; i < len(names); i++ { err := shell.setvar(names[i], values[i]) if err != nil { return err } } return nil } func (shell *Shell) newvars(names []*ast.NameNode, values []sh.Obj) { for i := 0; i < len(names); i++ { shell.newvar(names[i], values[i]) } } func (shell *Shell) setcmdvars(names []*ast.NameNode, stdout, stderr, status sh.Obj) error { if len(names) == 3 { err := shell.setvar(names[0], stdout) if err != nil { return err } err = shell.setvar(names[1], stderr) if err != nil { return err } return shell.setvar(names[2], status) } else if len(names) == 2 { err := shell.setvar(names[0], stdout) if err != nil { return err } return shell.setvar(names[1], status) } else if len(names) == 1 { return shell.setvar(names[0], stdout) } panic(fmt.Sprintf("internal error: expects 1 <= len(names) <= 3,"+ " but got %d", len(names))) return nil } func (shell *Shell) newcmdvars(names []*ast.NameNode, stdout, stderr, status sh.Obj) { if len(names) == 3 { shell.newvar(names[0], stdout) shell.newvar(names[1], stderr) shell.newvar(names[2], status) } else if len(names) == 2 { shell.newvar(names[0], stdout) shell.newvar(names[1], status) } else if len(names) == 1 { shell.newvar(names[0], stdout) } else { panic(fmt.Sprintf("internal error: expects 1 <= len(names) <= 3,"+ " but got %d", len(names))) } } // evalConcat reveives the AST representation of a concatenation of objects and // returns the string representation, or error. func (shell *Shell) evalConcat(path ast.Expr) (string, error) { var pathStr string if path.Type() != ast.NodeConcatExpr { return "", fmt.Errorf("Invalid node %+v", path) } concatExpr := path.(*ast.ConcatExpr) concat := concatExpr.List() for i := 0; i < len(concat); i++ { part := concat[i] switch part.Type() { case ast.NodeConcatExpr: return "", errors.NewEvalError(shell.filename, part, "Nested concat is not allowed: %s", part) case ast.NodeVarExpr, ast.NodeIndexExpr: partValue, err := shell.evalVariable(part) if err != nil { return "", err } if partValue.Type() == sh.ListType { return "", errors.NewEvalError(shell.filename, part, "Concat of list variables is not allowed: %v = %v", part, partValue) } else if partValue.Type() != sh.StringType { return "", errors.NewEvalError(shell.filename, part, "Invalid concat element: %v", partValue) } strval := partValue.(*sh.StrObj) pathStr += strval.Str() case ast.NodeStringExpr: str, ok := part.(*ast.StringExpr) if !ok { return "", errors.NewEvalError(shell.filename, part, "Failed to eval string: %s", part) } pathStr += str.Value() case ast.NodeFnInv: fnNode := part.(*ast.FnInvNode) result, err := shell.executeFnInv(fnNode) if err != nil { return "", err } if len(result) == 0 || len(result) > 1 { return "", errors.NewEvalError(shell.filename, part, "Function '%s' used in string concat but returns %d values.", fnNode.Name) } obj := result[0] if obj.Type() != sh.StringType { return "", errors.NewEvalError(shell.filename, part, "Function '%s' used in concat but returns a '%s'", obj.Type()) } str := obj.(*sh.StrObj) pathStr += str.Str() case ast.NodeListExpr: return "", errors.NewEvalError(shell.filename, part, "Concat of lists is not allowed: %+v", part.String()) default: return "", errors.NewEvalError(shell.filename, part, "Invalid argument: %+v", part) } } return pathStr, nil } func (shell *Shell) executeNode(node ast.Node) ([]sh.Obj, error) { var ( objs []sh.Obj err error ) shell.logf("Executing node: %v\n", node) switch node.Type() { case ast.NodeImport: err = shell.executeImport(node.(*ast.ImportNode)) case ast.NodeComment: // ignore case ast.NodeSetenv: err = shell.executeSetenv(node.(*ast.SetenvNode)) case ast.NodeVarAssignDecl: err = shell.executeVarAssign(node.(*ast.VarAssignDeclNode)) case ast.NodeVarExecAssignDecl: err = shell.executeVarExecAssign(node.(*ast.VarExecAssignDeclNode)) case ast.NodeAssign: err = shell.executeAssignment(node.(*ast.AssignNode)) case ast.NodeExecAssign: err = shell.executeExecAssign(node.(*ast.ExecAssignNode)) case ast.NodeCommand: _, err = shell.executeCommand(node.(*ast.CommandNode)) case ast.NodePipe: _, err = shell.executePipe(node.(*ast.PipeNode)) case ast.NodeRfork: err = shell.executeRfork(node.(*ast.RforkNode)) case ast.NodeIf: objs, err = shell.executeIf(node.(*ast.IfNode)) case ast.NodeFnDecl: err = shell.executeFnDecl(node.(*ast.FnDeclNode)) case ast.NodeFnInv: // invocation ignoring output _, err = shell.executeFnInv(node.(*ast.FnInvNode)) case ast.NodeFor: objs, err = shell.executeFor(node.(*ast.ForNode)) case ast.NodeBindFn: err = shell.executeBindFn(node.(*ast.BindFnNode)) case ast.NodeReturn: if shell.IsFn() { objs, err = shell.executeReturn(node.(*ast.ReturnNode)) } else { err = errors.NewEvalError(shell.filename, node, "Unexpected return outside of function declaration.") } default: // should never get here return nil, errors.NewEvalError(shell.filename, node, "invalid node: %v.", node.Type()) } return objs, err } func (shell *Shell) ExecuteTree(tr *ast.Tree) ([]sh.Obj, error) { return shell.executeTree(tr, true) } // executeTree evaluates the given tree func (shell *Shell) executeTree(tr *ast.Tree, stopable bool) ([]sh.Obj, error) { if tr == nil || tr.Root == nil { return nil, errors.NewError("empty abstract syntax tree to execute") } root := tr.Root for _, node := range root.Nodes { objs, err := shell.executeNode(node) if err != nil { type ( IgnoreError interface { Ignore() bool } InterruptedError interface { Interrupted() bool } StopWalkingError interface { StopWalking() bool } ) if errIgnore, ok := err.(IgnoreError); ok && errIgnore.Ignore() { continue } if errInterrupted, ok := err.(InterruptedError); ok && errInterrupted.Interrupted() { return nil, err } if errStopWalking, ok := err.(StopWalkingError); stopable && ok && errStopWalking.StopWalking() { return objs, nil } return objs, err } } return nil, nil } func (shell *Shell) executeReturn(n *ast.ReturnNode) ([]sh.Obj, error) { var returns []sh.Obj returnExprs := n.Returns for i := 0; i < len(returnExprs); i++ { retExpr := returnExprs[i] obj, err := shell.evalExpr(retExpr) if err != nil { return nil, err } returns = append(returns, obj) } return returns, newErrStopWalking() } func (shell *Shell) getNashRootFromGOPATH(preverr error) (string, error) { g, hasgopath := shell.Getenv("GOPATH") if !hasgopath { return "", errors.NewError("%s\nno GOPATH env var setted", preverr) } gopath := g.String() return filepath.Join(gopath, filepath.FromSlash("/src/github.com/madlambda/nash")), nil } func isValidNashRoot(nashroot string) bool { _, err := os.Stat(filepath.Join(nashroot, "stdlib")) return err == nil } func (shell *Shell) executeImport(node *ast.ImportNode) error { obj, err := shell.evalExpr(node.Path) if err != nil { return errors.NewEvalError(shell.filename, node, err.Error()) } if obj.Type() != sh.StringType { return errors.NewEvalError(shell.filename, node.Path, "Invalid type on import argument: %s", obj.Type()) } objstr := obj.(*sh.StrObj) fname := objstr.Str() shell.logf("Importing '%s'", fname) var ( tries []string hasExt bool ) hasExt = filepath.Ext(fname) != "" if filepath.IsAbs(fname) { tries = append(tries, fname) if !hasExt { tries = append(tries, fname+".sh") } } if shell.filename != "" { localFile := filepath.Join(filepath.Dir(shell.filename), fname) tries = append(tries, localFile) if !hasExt { tries = append(tries, localFile+".sh") } } tries = append(tries, filepath.Join(shell.nashpath, "lib", fname)) if !hasExt { tries = append(tries, filepath.Join(shell.nashpath, "lib", fname+".sh")) } tries = append(tries, filepath.Join(shell.nashroot, "stdlib", fname+".sh")) shell.logf("Trying %q\n", tries) for _, path := range tries { d, err := os.Stat(path) if err != nil { continue } if m := d.Mode(); !m.IsDir() { return shell.ExecFile(path) } } errmsg := fmt.Sprintf( "Failed to import path '%s'. The locations below have been tried:\n \"%s\"", fname, strings.Join(tries, `", "`), ) return errors.NewEvalError(shell.filename, node, errmsg) } // executePipe executes a pipe of ast.Command's. Each command can be // a path command in the operating system or a function bind to a // command name. // The error of each command can be suppressed prepending it with '-' (dash). // The error returned will be a string representing the errors (or none) of // each command separated by '|'. The $status of pipe execution will be // the $status of each command separated by '|'. func (shell *Shell) executePipe(pipe *ast.PipeNode) (sh.Obj, error) { var ( closeFiles []io.Closer closeAfterWait []io.Closer errIndex int err error ) defer func() { for _, c := range closeAfterWait { c.Close() } }() nodeCommands := pipe.Commands() if len(nodeCommands) < 2 { return sh.NewStrObj(strconv.Itoa(ENotStarted)), errors.NewEvalError(shell.filename, pipe, "Pipe requires at least two commands.") } cmds := make([]sh.Runner, len(nodeCommands)) errs := make([]string, len(nodeCommands)) igns := make([]bool, len(nodeCommands)) // ignoreErrors cods := make([]string, len(nodeCommands)) for i := 0; i < len(nodeCommands); i++ { errs[i] = "not started" cods[i] = strconv.Itoa(ENotStarted) } last := len(nodeCommands) - 1 envVars := buildenv(shell.Environ()) // Create all commands for i := 0; i < len(nodeCommands); i++ { var ( cmd sh.Runner ignore bool args []sh.Obj ) nodeCmd := nodeCommands[i] cmd, ignore, err = shell.getCommand(nodeCmd) igns[i] = ignore if err != nil { errIndex = i cods[i] = strconv.Itoa(ENotFound) goto pipeError } // SetEnviron must be called before SetArgs // otherwise the subshell will have the arguments // shadowed by parent env cmd.SetEnviron(envVars) args, err = shell.evalExprs(nodeCmd.Args()) if err != nil { errIndex = i goto pipeError } err = cmd.SetArgs(args) if err != nil { errIndex = i goto pipeError } cmd.SetStdin(shell.stdin) if i < last { closeFiles, err = shell.setRedirects(cmd, nodeCmd.Redirects()) closeAfterWait = append(closeAfterWait, closeFiles...) if err != nil { errIndex = i goto pipeError } } cmds[i] = cmd } // Shell does not support stdin redirection yet cmds[0].SetStdin(shell.stdin) // Setup the commands. Pointing the stdin of next command to stdout of previous. // Except the stdout of last one for i, cmd := range cmds[:last] { var ( stdin io.ReadCloser ) // StdoutPipe complains if Stdout is already set cmd.SetStdout(nil) stdin, err = cmd.StdoutPipe() if err != nil { errIndex = i goto pipeError } cmds[i+1].SetStdin(stdin) } cmds[last].SetStdout(shell.stdout) cmds[last].SetStderr(shell.stderr) closeFiles, err = shell.setRedirects(cmds[last], nodeCommands[last].Redirects()) closeAfterWait = append(closeAfterWait, closeFiles...) if err != nil { errIndex = last goto pipeError } for i := 0; i < len(cmds); i++ { cmd := cmds[i] err = cmd.Start() if err != nil { errIndex = i goto pipeError } errs[i] = "success" cods[i] = "0" } for i, cmd := range cmds { err = cmd.Wait() if err != nil { errIndex = i goto pipeError } errs[i] = "success" cods[i] = "0" } return sh.NewStrObj("0"), nil pipeError: if igns[errIndex] { errs[errIndex] = "none" } else { errs[errIndex] = err.Error() } cods[errIndex] = getErrStatus(err, cods[errIndex]) err = errors.NewEvalError(shell.filename, pipe, strings.Join(errs, "|")) // verify if all status codes are the same uniqCodes := make(map[string]struct{}) var uniqCode string for i := 0; i < len(cods); i++ { uniqCodes[cods[i]] = struct{}{} uniqCode = cods[i] } var status sh.Obj if len(uniqCodes) == 1 { // if all status are the same status = sh.NewStrObj(uniqCode) } else { status = sh.NewStrObj(strings.Join(cods, "|")) } if igns[errIndex] { return status, nil } return status, err } func (shell *Shell) openRedirectLocation(location ast.Expr) (io.WriteCloser, error) { var protocol string locationObj, err := shell.evalExpr(location) if err != nil { return nil, err } if locationObj.Type() != sh.StringType { return nil, errors.NewEvalError(shell.filename, location, "Redirection to invalid object type: %v (%s)", locationObj, locationObj.Type()) } objstr := locationObj.(*sh.StrObj) locationStr := objstr.Str() if len(locationStr) > 6 { if locationStr[0:6] == "tcp://" { protocol = "tcp" } else if locationStr[0:6] == "udp://" { protocol = "udp" } else if len(locationStr) > 7 && locationStr[0:7] == "unix://" { protocol = "unix" } } if protocol == "" { return os.OpenFile(locationStr, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) } switch protocol { case "tcp", "udp": netParts := strings.Split(locationStr[6:], ":") if len(netParts) != 2 { return nil, errors.NewEvalError(shell.filename, location, "Invalid tcp/udp address: %s", locationStr) } url := netParts[0] + ":" + netParts[1] return net.Dial(protocol, url) case "unix": return net.Dial(protocol, locationStr[7:]) } return nil, errors.NewEvalError(shell.filename, location, "Unexpected redirection value: %s", locationStr) } func (shell *Shell) setRedirects(cmd sh.Runner, redirDecls []*ast.RedirectNode) ([]io.Closer, error) { var closeAfterWait []io.Closer for _, r := range redirDecls { closeFiles, err := shell.buildRedirect(cmd, r) closeAfterWait = append(closeAfterWait, closeFiles...) if err != nil { return closeAfterWait, err } } return closeAfterWait, nil } func (shell *Shell) buildRedirect(cmd sh.Runner, redirDecl *ast.RedirectNode) ([]io.Closer, error) { var closeAfterWait []io.Closer if redirDecl.LeftFD() > 2 || redirDecl.LeftFD() < ast.RedirMapSupress { return closeAfterWait, errors.NewEvalError(shell.filename, redirDecl, "Invalid file descriptor redirection: fd=%d", redirDecl.LeftFD()) } if redirDecl.RightFD() > 2 || redirDecl.RightFD() < ast.RedirMapSupress { return closeAfterWait, errors.NewEvalError(shell.filename, redirDecl, "Invalid file descriptor redirection: fd=%d", redirDecl.RightFD()) } var err error // Note(i4k): We need to remove the repetitive code in some smarter way switch redirDecl.LeftFD() { case 0: return closeAfterWait, fmt.Errorf("Does not support stdin redirection yet") case 1: switch redirDecl.RightFD() { case 0: return closeAfterWait, errors.NewEvalError(shell.filename, redirDecl, "Invalid redirect mapping: %d -> %d", 1, 0) case 1: // do nothing case 2: cmd.SetStdout(cmd.Stderr()) case ast.RedirMapNoValue: if redirDecl.Location() == nil { return closeAfterWait, errors.NewEvalError(shell.filename, redirDecl, "Missing file in redirection: >[%d] ", redirDecl.LeftFD()) } file, err := shell.openRedirectLocation(redirDecl.Location()) if err != nil { return closeAfterWait, err } cmd.SetStdout(file) closeAfterWait = append(closeAfterWait, file) case ast.RedirMapSupress: file := ioutil.Discard cmd.SetStdout(file) } case 2: switch redirDecl.RightFD() { case 0: return closeAfterWait, errors.NewEvalError(shell.filename, redirDecl, "Invalid redirect mapping: %d -> %d", 2, 1) case 1: cmd.SetStderr(cmd.Stdout()) case 2: // do nothing case ast.RedirMapNoValue: if redirDecl.Location() == nil { return closeAfterWait, errors.NewEvalError(shell.filename, redirDecl, "Missing file in redirection: >[%d] ", redirDecl.LeftFD()) } file, err := shell.openRedirectLocation(redirDecl.Location()) if err != nil { return closeAfterWait, err } cmd.SetStderr(file) closeAfterWait = append(closeAfterWait, file) case ast.RedirMapSupress: cmd.SetStderr(ioutil.Discard) } case ast.RedirMapNoValue: if redirDecl.Location() == nil { return closeAfterWait, errors.NewEvalError(shell.filename, redirDecl, "Missing file in redirection: >[%d] ", redirDecl.LeftFD()) } file, err := shell.openRedirectLocation(redirDecl.Location()) if err != nil { return closeAfterWait, err } cmd.SetStdout(file) closeAfterWait = append(closeAfterWait, file) } return closeAfterWait, err } func (shell *Shell) newBindfnRunner( c *ast.CommandNode, cmdName string, fnDef sh.FnDef, ) (sh.Runner, error) { shell.logf("Executing bind %s", cmdName) shell.logf("%s bind to %s", cmdName, fnDef.Name()) if !shell.Interactive() { err := errors.NewEvalError(shell.filename, c, "'%s' is a bind to '%s'. "+ "No binds allowed in non-interactive mode.", cmdName, fnDef.Name()) return nil, err } return fnDef.Build(), nil } func (shell *Shell) getCommand(c *ast.CommandNode) (sh.Runner, bool, error) { var ( ignoreError bool cmd sh.Runner err error ) cmdName := c.Name() shell.logf("Executing: %s\n", c.Name()) if len(cmdName) > 1 && cmdName[0] == '-' { ignoreError = true cmdName = cmdName[1:] shell.logf("Ignoring error\n") } if cmdName == "" { return nil, false, errors.NewEvalError(shell.filename, c, "Empty command name...") } if fnDef, ok := shell.Getbindfn(cmdName); ok { runner, err := shell.newBindfnRunner(c, cmdName, fnDef) return runner, ignoreError, err } cmd, err = NewCmd(cmdName) if err != nil { type NotFound interface { NotFound() bool } shell.logf("Command fails: %s", err.Error()) if errNotFound, ok := err.(NotFound); ok && errNotFound.NotFound() { return nil, ignoreError, err } return nil, ignoreError, err } cmd.SetStdin(shell.stdin) cmd.SetStdout(shell.stdout) cmd.SetStderr(shell.stderr) return cmd, ignoreError, nil } func (shell *Shell) executeCommand(c *ast.CommandNode) (sh.Obj, error) { var ( ignoreError bool status = "127" envVars []string closeAfterWait []io.Closer cmd sh.Runner err error args []sh.Obj ) defer func() { for _, c := range closeAfterWait { c.Close() } }() cmd, ignoreError, err = shell.getCommand(c) if err != nil { goto cmdError } // SetEnviron must be called before SetArgs // otherwise the subshell will have the arguments // shadowed by parent env envVars = buildenv(shell.Environ()) cmd.SetEnviron(envVars) args, err = shell.evalExprs(c.Args()) if err != nil { goto cmdError } err = cmd.SetArgs(args) if err != nil { goto cmdError } closeAfterWait, err = shell.setRedirects(cmd, c.Redirects()) if err != nil { goto cmdError } err = cmd.Start() if err != nil { goto cmdError } err = cmd.Wait() if err != nil { goto cmdError } return sh.NewStrObj("0"), nil cmdError: statusObj := sh.NewStrObj(getErrStatus(err, status)) if ignoreError { return statusObj, newErrIgnore(err.Error()) } return statusObj, err } func (shell *Shell) evalList(argList *ast.ListExpr) (sh.Obj, error) { values := make([]sh.Obj, 0, len(argList.List)) for _, arg := range argList.List { obj, err := shell.evalExpr(arg) if err != nil { return nil, err } values = append(values, obj) } return sh.NewListObj(values), nil } func (shell *Shell) evalArgList(argList *ast.ListExpr) ([]sh.Obj, error) { values := make([]sh.Obj, 0, len(argList.List)) for _, arg := range argList.List { obj, err := shell.evalExpr(arg) if err != nil { return nil, err } values = append(values, obj) } if argList.IsVariadic { return values, nil } return []sh.Obj{sh.NewListObj(values)}, nil } func (shell *Shell) evalIndex(index ast.Expr) (int, error) { if index.Type() != ast.NodeIntExpr && index.Type() != ast.NodeVarExpr && index.Type() != ast.NodeIndexExpr { return 0, errors.NewEvalError(shell.filename, index, "Invalid indexing type: %s", index.Type()) } if index.Type() == ast.NodeIntExpr { idxArg := index.(*ast.IntExpr) return idxArg.Value(), nil } idxObj, err := shell.evalVariable(index) if err != nil { return 0, err } if idxObj.Type() != sh.StringType { return 0, errors.NewEvalError(shell.filename, index, "Invalid object type on index value: %s", idxObj.Type()) } objstr := idxObj.(*sh.StrObj) indexNum, err := strconv.Atoi(objstr.Str()) if err != nil { return 0, err } return indexNum, nil } func (shell *Shell) evalIndexedVar(indexVar *ast.IndexExpr) (sh.Obj, error) { v, err := shell.evalVariable(indexVar.Var) if err != nil { return nil, err } col, err := sh.NewCollection(v) if err != nil { return nil, errors.NewEvalError(shell.filename, indexVar.Var, err.Error()) } indexNum, err := shell.evalIndex(indexVar.Index) if err != nil { return nil, err } val, err := col.Get(indexNum) if err != nil { return nil, errors.NewEvalError(shell.filename, indexVar.Var, err.Error()) } return val, nil } func (shell *Shell) evalArgIndexedVar(indexVar *ast.IndexExpr) ([]sh.Obj, error) { v, err := shell.evalVariable(indexVar.Var) if err != nil { return nil, err } col, err := sh.NewCollection(v) if err != nil { return nil, errors.NewEvalError(shell.filename, indexVar.Var, err.Error()) } indexNum, err := shell.evalIndex(indexVar.Index) if err != nil { return nil, err } retval, err := col.Get(indexNum) if err != nil { return nil, errors.NewEvalError(shell.filename, indexVar.Var, err.Error()) } if indexVar.IsVariadic { if retval.Type() != sh.ListType { return nil, errors.NewEvalError(shell.filename, indexVar, "Use of '...' on a non-list variable") } retlist := retval.(*sh.ListObj) return retlist.List(), nil } return []sh.Obj{retval}, nil } func (shell *Shell) evalVariable(a ast.Expr) (sh.Obj, error) { var ( value sh.Obj ok bool ) if a.Type() == ast.NodeIndexExpr { return shell.evalIndexedVar(a.(*ast.IndexExpr)) } if a.Type() != ast.NodeVarExpr { return nil, errors.NewEvalError(shell.filename, a, "Invalid eval of non variable argument: %s", a) } vexpr := a.(*ast.VarExpr) varName := vexpr.Name if value, ok = shell.Getvar(varName[1:]); !ok { return nil, errors.NewEvalError(shell.filename, a, "Variable %s not set on shell %s", varName, shell.name) } return value, nil } func (shell *Shell) evalArgVariable(a ast.Expr) ([]sh.Obj, error) { var ( value sh.Obj ok bool ) if a.Type() == ast.NodeIndexExpr { return shell.evalArgIndexedVar(a.(*ast.IndexExpr)) } if a.Type() != ast.NodeVarExpr { return nil, errors.NewEvalError(shell.filename, a, "Invalid eval of non variable argument: %s", a) } vexpr := a.(*ast.VarExpr) if value, ok = shell.Getvar(vexpr.Name[1:]); !ok { return nil, errors.NewEvalError(shell.filename, a, "Variable %s not set on shell %s", vexpr.Name, shell.name) } if vexpr.IsVariadic { if value.Type() != sh.ListType { return nil, errors.NewEvalError(shell.filename, a, "Variable expansion (%s) on a non-list object", vexpr.String()) } return value.(*sh.ListObj).List(), nil } return []sh.Obj{value}, nil } func (shell *Shell) evalExprs(exprs []ast.Expr) ([]sh.Obj, error) { objs := make([]sh.Obj, 0, len(exprs)) for _, expr := range exprs { obj, err := shell.evalExpr(expr) if err != nil { return nil, err } objs = append(objs, obj) } return objs, nil } func (shell *Shell) evalArgExprs(exprs []ast.Expr) ([]sh.Obj, error) { ret := make([]sh.Obj, 0, len(exprs)) for _, expr := range exprs { objs, err := shell.evalArgExpr(expr) if err != nil { return nil, err } ret = append(ret, objs...) } return ret, nil } func (shell *Shell) evalArgExpr(expr ast.Expr) ([]sh.Obj, error) { switch expr.Type() { case ast.NodeStringExpr: if str, ok := expr.(*ast.StringExpr); ok { return []sh.Obj{ sh.NewStrObj(str.Value()), }, nil } case ast.NodeConcatExpr: if concat, ok := expr.(*ast.ConcatExpr); ok { argVal, err := shell.evalConcat(concat) if err != nil { return nil, err } return []sh.Obj{ sh.NewStrObj(argVal), }, nil } case ast.NodeVarExpr: return shell.evalArgVariable(expr) case ast.NodeIndexExpr: if indexedVar, ok := expr.(*ast.IndexExpr); ok { return shell.evalArgIndexedVar(indexedVar) } case ast.NodeListExpr: if listExpr, ok := expr.(*ast.ListExpr); ok { return shell.evalArgList(listExpr) } case ast.NodeFnInv: if fnInv, ok := expr.(*ast.FnInvNode); ok { objs, err := shell.executeFnInv(fnInv) if err != nil { return nil, err } if len(objs) == 0 { return nil, errors.NewEvalError(shell.filename, expr, "Function used in"+ " expression but do not return any value: %s", fnInv) } else if len(objs) != 1 { return nil, errors.NewEvalError(shell.filename, expr, "Function used in"+ " expression but it returns %d values: %10q", len(objs), objs) } return []sh.Obj{objs[0]}, nil } } return nil, errors.NewEvalError(shell.filename, expr, "Failed to eval expression: %+v", expr) } func (shell *Shell) evalExpr(expr ast.Expr) (sh.Obj, error) { switch expr.Type() { case ast.NodeStringExpr: if str, ok := expr.(*ast.StringExpr); ok { return sh.NewStrObj(str.Value()), nil } case ast.NodeConcatExpr: if concat, ok := expr.(*ast.ConcatExpr); ok { argVal, err := shell.evalConcat(concat) if err != nil { return nil, err } return sh.NewStrObj(argVal), nil } case ast.NodeVarExpr: return shell.evalVariable(expr) case ast.NodeIndexExpr: if indexedVar, ok := expr.(*ast.IndexExpr); ok { return shell.evalIndexedVar(indexedVar) } case ast.NodeListExpr: if listExpr, ok := expr.(*ast.ListExpr); ok { return shell.evalList(listExpr) } case ast.NodeFnInv: if fnInv, ok := expr.(*ast.FnInvNode); ok { objs, err := shell.executeFnInv(fnInv) if err != nil { return nil, err } if len(objs) == 0 { return nil, errors.NewEvalError(shell.filename, expr, "Function used in"+ " expression but do not return any value: %s", fnInv) } else if len(objs) != 1 { return nil, errors.NewEvalError(shell.filename, expr, "Function used in"+ " expression but it returns %d values: %10q", len(objs), objs) } return objs[0], nil } } return nil, errors.NewEvalError(shell.filename, expr, "Failed to eval expression: %+v", expr) } func (shell *Shell) executeSetenvAssign(assign *ast.AssignNode) error { for i := 0; i < len(assign.Names); i++ { name := assign.Names[i] value := assign.Values[i] err := shell.initVar(name, value) if err != nil { return err } obj, ok := shell.GetLocalvar(name.Ident) if !ok { return errors.NewEvalError(shell.filename, assign, "internal error: Setenv not setting local variable '%s'", name.Ident, ) } shell.Setenv(name.Ident, obj) } return nil } func (shell *Shell) executeSetenvExec(assign *ast.ExecAssignNode) error { err := shell.executeExecAssign(assign) if err != nil { return err } for i := 0; i < len(assign.Names); i++ { name := assign.Names[i] obj, ok := shell.GetLocalvar(name.Ident) if !ok { return errors.NewEvalError(shell.filename, assign, "internal error: Setenv not setting local variable '%s'", name.Ident, ) } shell.Setenv(name.Ident, obj) } return nil } func (shell *Shell) executeSetenv(v *ast.SetenvNode) error { var ( varValue sh.Obj ok bool assign = v.Assignment() ) if assign != nil { switch assign.Type() { case ast.NodeAssign: return shell.executeSetenvAssign(assign.(*ast.AssignNode)) case ast.NodeExecAssign: return shell.executeSetenvExec(assign.(*ast.ExecAssignNode)) } return errors.NewEvalError(shell.filename, v, "Failed to eval setenv, invalid assignment type: %+v", assign) } varValue, ok = shell.Getvar(v.Name) if !ok { return errors.NewEvalError(shell.filename, v, "Variable '%s' not set on shell %s", v.Name, shell.name, ) } shell.Setenv(v.Name, varValue) return nil } func (shell *Shell) concatElements(expr *ast.ConcatExpr) (string, error) { value := "" list := expr.List() for i := 0; i < len(list); i++ { ec := list[i] obj, err := shell.evalExpr(ec) if err != nil { return "", err } if obj.Type() != sh.StringType { return "", errors.NewEvalError(shell.filename, expr, "Impossible to concat elements of type %s", obj.Type()) } value = value + obj.String() } return value, nil } func (shell *Shell) execCmdOutput(cmd ast.Node, getstderr, ignoreError bool) ([]byte, []byte, sh.Obj, error) { var ( outBuf, errBuf bytes.Buffer err error status sh.Obj ) if cmd.Type() != ast.NodeCommand && cmd.Type() != ast.NodePipe { return nil, nil, nil, errors.NewEvalError(shell.filename, cmd, "Invalid node type (%v). Expected command or pipe", cmd) } bkStdout, bkStderr := shell.stdout, shell.stderr shell.SetStdout(&outBuf) if getstderr { shell.SetStderr(&errBuf) } defer func() { shell.SetStdout(bkStdout) shell.SetStderr(bkStderr) }() if cmd.Type() == ast.NodeCommand { status, err = shell.executeCommand(cmd.(*ast.CommandNode)) } else { status, err = shell.executePipe(cmd.(*ast.PipeNode)) } outb := outBuf.Bytes() errb := errBuf.Bytes() trimnl := func(data []byte) []byte { if len(data) > 0 && data[len(data)-1] == '\n' { // remove the trailing new line // Why? because it's what user wants in 99.99% of times... data = data[0 : len(data)-1] } return data[:] } if ignoreError { err = nil } return trimnl(outb), trimnl(errb), status, err } func (shell *Shell) executeExecAssignCmd(v ast.Node) (stdout, stderr, status sh.Obj, err error) { assign := v.(*ast.ExecAssignNode) cmd := assign.Command() mustIgnoreErr := len(assign.Names) > 1 collectStderr := len(assign.Names) == 3 outb, errb, status, err := shell.execCmdOutput(cmd, collectStderr, mustIgnoreErr) if err != nil { return nil, nil, nil, err } return sh.NewStrObj(string(outb)), sh.NewStrObj(string(errb)), status, nil } func (shell *Shell) executeExecAssignFn(assign *ast.ExecAssignNode) ([]sh.Obj, error) { var ( err error fnValues []sh.Obj ) cmd := assign.Command() if cmd.Type() != ast.NodeFnInv { return nil, errors.NewEvalError(shell.filename, cmd, "Invalid node type (%v). Expected function call", cmd) } fnValues, err = shell.executeFnInv(cmd.(*ast.FnInvNode)) if err != nil { return nil, err } if len(fnValues) != len(assign.Names) { return nil, errors.NewEvalError(shell.filename, assign, "Functions returns %d objects, but statement expects %d", len(fnValues), len(assign.Names)) } return fnValues, nil } func (shell *Shell) executeExecAssign(v *ast.ExecAssignNode) (err error) { exec := v.Command() switch exec.Type() { case ast.NodeFnInv: var values []sh.Obj values, err = shell.executeExecAssignFn(v) if err != nil { return err } err = shell.setvars(v.Names, values) case ast.NodeCommand, ast.NodePipe: var stdout, stderr, status sh.Obj stdout, stderr, status, err = shell.executeExecAssignCmd(v) if err != nil { return err } err = shell.setcmdvars(v.Names, stdout, stderr, status) default: err = errors.NewEvalError(shell.filename, exec, "Invalid node type (%v). Expected function call, command or pipe", exec) } return err } func (shell *Shell) initVar(name *ast.NameNode, value ast.Expr) error { obj, err := shell.evalExpr(value) if err != nil { return err } return shell.newvar(name, obj) } func (shell *Shell) executeVarAssign(v *ast.VarAssignDeclNode) error { assign := v.Assign if len(assign.Names) != len(assign.Values) { return errors.NewEvalError(shell.filename, assign, "Invalid multiple assignment. Different amount of variables and values: %s", assign, ) } for i := 0; i < len(assign.Names); i++ { name := assign.Names[i] value := assign.Values[i] err := shell.initVar(name, value) if err != nil { return err } } return nil } func (shell *Shell) executeVarExecAssign(v *ast.VarExecAssignDeclNode) (err error) { assign := v.ExecAssign exec := assign.Command() switch exec.Type() { case ast.NodeFnInv: var values []sh.Obj values, err = shell.executeExecAssignFn(assign) if err != nil { return err } shell.newvars(assign.Names, values) case ast.NodeCommand, ast.NodePipe: var stdout, stderr, status sh.Obj stdout, stderr, status, err = shell.executeExecAssignCmd(assign) if err != nil { return err } shell.newcmdvars(assign.Names, stdout, stderr, status) default: err = errors.NewEvalError(shell.filename, exec, "Invalid node type (%v). Expected function call, command or pipe", exec) } return err } func (shell *Shell) executeAssignment(v *ast.AssignNode) error { if len(v.Names) != len(v.Values) { return errors.NewEvalError(shell.filename, v, "Invalid multiple assignment. Different amount of variables and values: %s", v, ) } for i := 0; i < len(v.Names); i++ { name := v.Names[i] value := v.Values[i] obj, err := shell.evalExpr(value) if err != nil { return err } err = shell.setvar(name, obj) if err != nil { return err } } return nil } func (shell *Shell) evalIfArgument(arg ast.Node) (sh.Obj, error) { var ( obj sh.Obj err error ) obj, err = shell.evalExpr(arg) if err != nil { return nil, err } else if obj == nil { return nil, errors.NewEvalError(shell.filename, arg, "lvalue doesn't yield value (%s)", arg) } return obj, nil } func (shell *Shell) evalIfArguments(n *ast.IfNode) (string, string, error) { var ( lobj, robj sh.Obj err error ) lobj, err = shell.evalIfArgument(n.Lvalue()) if err != nil { return "", "", err } robj, err = shell.evalIfArgument(n.Rvalue()) if err != nil { return "", "", err } if lobj.Type() != sh.StringType { return "", "", errors.NewEvalError(shell.filename, n, "lvalue is not comparable: (%v) -> %s.", lobj, lobj.Type()) } if robj.Type() != sh.StringType { return "", "", errors.NewEvalError(shell.filename, n, "rvalue is not comparable: (%v) -> %s.", lobj, lobj.Type()) } lobjstr := lobj.(*sh.StrObj) robjstr := robj.(*sh.StrObj) return lobjstr.Str(), robjstr.Str(), nil } func (shell *Shell) executeIfEqual(n *ast.IfNode) ([]sh.Obj, error) { lstr, rstr, err := shell.evalIfArguments(n) if err != nil { return nil, err } if lstr == rstr { return shell.executeTree(n.IfTree(), false) } else if n.ElseTree() != nil { return shell.executeTree(n.ElseTree(), false) } return nil, nil } func (shell *Shell) executeIfNotEqual(n *ast.IfNode) ([]sh.Obj, error) { lstr, rstr, err := shell.evalIfArguments(n) if err != nil { return nil, err } if lstr != rstr { return shell.executeTree(n.IfTree(), false) } else if n.ElseTree() != nil { return shell.executeTree(n.ElseTree(), false) } return nil, nil } func (shell *Shell) executeFnInv(n *ast.FnInvNode) ([]sh.Obj, error) { var fnDef sh.FnDef fnName := n.Name() if len(fnName) > 1 && fnName[0] == '$' { argVar := ast.NewVarExpr(token.NewFileInfo(n.Line(), n.Column()), fnName) obj, err := shell.evalVariable(argVar) if err != nil { return nil, err } if obj.Type() != sh.FnType { return nil, errors.NewEvalError(shell.filename, n, "Variable '%s' is not a function.", fnName) } objfn := obj.(*sh.FnObj) fnDef = objfn.Fn() } else { fnObj, err := shell.GetFn(fnName) if err != nil { return nil, errors.NewEvalError(shell.filename, n, err.Error()) } fnDef = fnObj.Fn() } fn := fnDef.Build() args, err := shell.evalArgExprs(n.Args()) if err != nil { return nil, err } err = fn.SetArgs(args) if err != nil { return nil, errors.NewEvalError(shell.filename, n, err.Error()) } fn.SetStdin(shell.stdin) fn.SetStdout(shell.stdout) fn.SetStderr(shell.stderr) err = fn.Start() if err != nil { return nil, errors.NewEvalError(shell.filename, n, err.Error()) } err = fn.Wait() if err != nil { return nil, errors.NewEvalError(shell.filename, n, err.Error()) } return fn.Results(), nil } func (shell *Shell) executeInfLoop(tr *ast.Tree) ([]sh.Obj, error) { var ( err error objs []sh.Obj ) for { objs, err = shell.executeTree(tr, false) runtime.Gosched() type ( interruptedError interface { Interrupted() bool } stopWalkingError interface { StopWalking() bool } ) if errInterrupted, ok := err.(interruptedError); ok && errInterrupted.Interrupted() { break } if errStopWalking, ok := err.(stopWalkingError); ok && errStopWalking.StopWalking() { return objs, err } shell.Lock() if shell.getIntr() { shell.setIntr(false) if err != nil { err = newErrInterrupted(err.Error()) } else { err = newErrInterrupted("loop interrupted") } } shell.Unlock() if err != nil { break } } return nil, err } func (shell *Shell) executeFor(n *ast.ForNode) ([]sh.Obj, error) { shell.Lock() shell.looping = true shell.Unlock() defer func() { shell.Lock() defer shell.Unlock() shell.looping = false }() if n.InExpr() == nil { return shell.executeInfLoop(n.Tree()) } id := n.Identifier() inExpr := n.InExpr() var ( obj sh.Obj err error ) if inExpr.Type() == ast.NodeVarExpr { obj, err = shell.evalVariable(inExpr.(*ast.VarExpr)) } else if inExpr.Type() == ast.NodeListExpr { obj, err = shell.evalList(inExpr.(*ast.ListExpr)) } else if inExpr.Type() == ast.NodeFnInv { var objs []sh.Obj objs, err = shell.executeFnInv(inExpr.(*ast.FnInvNode)) if err != nil { return nil, err } if len(objs) != 1 { return nil, errors.NewEvalError(shell.filename, inExpr, "Functions with multiple returns do not work as for 'in expression' yet: %v", inExpr) } obj = objs[0] } else { return nil, errors.NewEvalError(shell.filename, inExpr, "Invalid expression in for loop: %s", inExpr.Type()) } if err != nil { return nil, err } col, err := sh.NewCollection(obj) if err != nil { return nil, errors.NewEvalError(shell.filename, inExpr, "error[%s] trying to iterate", err) } for i := 0; i < col.Len(); i++ { val, err := col.Get(i) if err != nil { return nil, errors.NewEvalError(shell.filename, inExpr, "unexpected error[%s] during iteration", err) } shell.Newvar(id, val) objs, err := shell.executeTree(n.Tree(), false) type ( interruptedError interface { Interrupted() bool } stopWalkingError interface { StopWalking() bool } ) if errInterrupted, ok := err.(interruptedError); ok && errInterrupted.Interrupted() { return nil, err } if errStopWalking, ok := err.(stopWalkingError); ok && errStopWalking.StopWalking() { return objs, err } shell.Lock() if shell.getIntr() { shell.setIntr(false) shell.Unlock() if err != nil { return nil, newErrInterrupted(err.Error()) } return nil, newErrInterrupted("loop interrupted") } shell.Unlock() if err != nil { return nil, err } } return nil, nil } func (shell *Shell) executeFnDecl(n *ast.FnDeclNode) error { fnDef, err := newUserFnDef(n.Name(), shell, n.Args(), n.Tree()) if err != nil { return err } shell.Newvar(n.Name(), sh.NewFnObj(fnDef)) shell.logf("Function %s declared on '%s'", n.Name(), shell.name) return nil } func (shell *Shell) executeBindFn(n *ast.BindFnNode) error { if !shell.Interactive() { return errors.NewEvalError(shell.filename, n, "'bindfn' is not allowed in non-interactive mode.") } fnDef, err := shell.GetFn(n.Name()) if err != nil { return errors.NewEvalError(shell.filename, n, err.Error()) } shell.Setbindfn(n.CmdName(), fnDef.Fn()) return nil } func (shell *Shell) executeIf(n *ast.IfNode) ([]sh.Obj, error) { op := n.Op() if op == "==" { return shell.executeIfEqual(n) } else if op == "!=" { return shell.executeIfNotEqual(n) } return nil, fmt.Errorf("invalid operation '%s'", op) } func validateDirs(nashpath string, nashroot string) error { if nashpath == nashroot { return fmt.Errorf("invalid nashpath and nashroot, they are both[%s] but they must differ", nashpath) } err := validateDir(nashpath) if err != nil { return fmt.Errorf("invalid nashpath, user's config won't be loaded: error: %s", err) } err = validateDir(nashroot) if err != nil { return fmt.Errorf("invalid nashroot, stdlib/stdbin won't be available: error: %s", err) } return nil } func validateDir(dir string) error { dir, err := filepath.EvalSymlinks(dir) if err != nil { return err } info, err := os.Stat(dir) if err != nil { return err } if !info.IsDir() { return fmt.Errorf("%s is a file, expected a dir", dir) } if !filepath.IsAbs(dir) { return fmt.Errorf("%s is a relative path, expected a absolute path", dir) } return nil } ================================================ FILE: internal/sh/shell_import_test.go ================================================ package sh_test import ( "bytes" "path/filepath" "strings" "testing" "github.com/madlambda/nash/internal/sh" "github.com/madlambda/nash/internal/sh/internal/fixture" ) func TestImportsLibFromNashPathLibDir(t *testing.T) { nashdirs := fixture.SetupNashDirs(t) defer nashdirs.Cleanup() writeFile(t, filepath.Join(nashdirs.Lib, "lib.sh"), ` fn test() { echo "hasnashpath" } `) newTestShell(t, nashdirs.Path, nashdirs.Root).ExecCheckingOutput(t, ` import lib test() `, "hasnashpath\n") } func TestImportsLibFromNashPathLibDirBeforeNashRootStdlib(t *testing.T) { nashdirs := fixture.SetupNashDirs(t) defer nashdirs.Cleanup() writeFile(t, filepath.Join(nashdirs.Lib, "lib.sh"), ` fn test() { echo "libcode" } `) writeFile(t, filepath.Join(nashdirs.Stdlib, "lib.sh"), ` fn test() { echo "stdlibcode" } `) newTestShell(t, nashdirs.Path, nashdirs.Root).ExecCheckingOutput(t, ` import lib test() `, "libcode\n") } func TestImportsLibFromNashRootStdlib(t *testing.T) { nashdirs := fixture.SetupNashDirs(t) defer nashdirs.Cleanup() writeFile(t, filepath.Join(nashdirs.Stdlib, "lib.sh"), ` fn test() { echo "stdlibcode" } `) newTestShell(t, nashdirs.Path, nashdirs.Root).ExecCheckingOutput(t, ` import lib test() `, "stdlibcode\n") } func TestImportsLibFromWorkingDirBeforeLibAndStdlib(t *testing.T) { workingdir, rmdir := fixture.Tmpdir(t) defer rmdir() curwd := getwd(t) chdir(t, workingdir) defer chdir(t, curwd) nashdirs := fixture.SetupNashDirs(t) defer nashdirs.Cleanup() writeFile(t, filepath.Join(workingdir, "lib.sh"), ` fn test() { echo "localcode" } `) writeFile(t, filepath.Join(nashdirs.Lib, "lib.sh"), ` fn test() { echo "libcode" } `) writeFile(t, filepath.Join(nashdirs.Stdlib, "lib.sh"), ` fn test() { echo "stdlibcode" } `) newTestShell(t, nashdirs.Path, nashdirs.Root).ExecCheckingOutput(t, ` import lib test() `, "localcode\n") } func TestStdErrOnInvalidSearchPaths(t *testing.T) { type testCase struct { name string nashpath string nashroot string errmsg string } const nashrooterr = "invalid nashroot" const nashpatherr = "invalid nashpath" validDir, rmdir := fixture.Tmpdir(t) defer rmdir() validfile := filepath.Join(validDir, "notdir") writeFile(t, validfile, "whatever") cases := []testCase{ { name: "EmptyNashPath", nashpath: "", nashroot: validDir, errmsg: nashpatherr, }, { name: "NashPathDontExists", nashpath: filepath.Join(validDir, "dontexists"), nashroot: validDir, errmsg: nashpatherr, }, { name: "EmptyNashRoot", nashpath: validDir, nashroot: "", errmsg: nashrooterr, }, { name: "NashRootDontExists", nashroot: filepath.Join(validDir, "dontexists"), nashpath: validDir, errmsg: nashrooterr, }, { name: "NashPathIsFile", nashroot: validDir, nashpath: validfile, errmsg: nashpatherr, }, { name: "NashRootIsFile", nashroot: validfile, nashpath: validDir, errmsg: nashrooterr, }, { name: "NashPathIsRelative", nashroot: validDir, nashpath: "./", errmsg: nashpatherr, }, { name: "NashRootIsRelative", nashroot: "./", nashpath: validDir, errmsg: nashrooterr, }, { name: "NashRootAndNashPathAreEqual", nashroot: validDir, nashpath: validDir, errmsg: "invalid nashpath and nashroot", }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { _, err := sh.NewAbortShell(c.nashpath, c.nashroot) if c.errmsg != "" { if err == nil { t.Fatalf("expected err[%s]", c.errmsg) } if !strings.HasPrefix(err.Error(), c.errmsg) { t.Fatalf("errors mismatch: [%s] didnt contains [%s]", err, c.errmsg) } } else if err != nil { t.Fatalf("got unexpected error[%s]", err) } }) } } type testshell struct { shell *sh.Shell stdout *bytes.Buffer } func (s *testshell) ExecCheckingOutput(t *testing.T, code string, expectedOutupt string) { err := s.shell.Exec("shellenvtest", code) if err != nil { t.Fatal(err) } output := s.stdout.String() s.stdout.Reset() if output != expectedOutupt { t.Fatalf( "expected output: [%s] got: [%s]", expectedOutupt, output, ) } } func newTestShell(t *testing.T, nashpath string, nashroot string) *testshell { shell, err := sh.NewShell(nashpath, nashroot) if err != nil { t.Fatal(err) } var out bytes.Buffer shell.SetStdout(&out) return &testshell{shell: shell, stdout: &out} } ================================================ FILE: internal/sh/shell_linux_test.go ================================================ // +build linux package sh_test import ( "bytes" "fmt" "io/ioutil" "os" "os/exec" "strings" "testing" ) var ( enableUserNS bool ) func init() { const usernsOk = "1" const kernelcfg = "CONFIG_USER_NS" logUsernsDetection := func(err error) { if enableUserNS { fmt.Printf("Linux user namespaces enabled!") return } fmt.Printf("Warning: Impossible to know if kernel support USER namespace.\n") fmt.Printf("Warning: USER namespace tests will not run.\n") if err != nil { fmt.Printf("ERROR: %s\n", err) } } usernsCfg := "/proc/sys/kernel/unprivileged_userns_clone" val, permerr := ioutil.ReadFile(usernsCfg) // Travis build doesn't support /proc/config.gz but kernel has userns if os.Getenv("TRAVIS_BUILD") == "1" { enableUserNS = permerr == nil && string(val) == usernsOk logUsernsDetection(permerr) return } if permerr == nil { enableUserNS = string(val) == usernsOk logUsernsDetection(permerr) return } // old kernels dont have sysctl configurations // than just checking the /proc/config suffices usernsCmd := exec.Command("zgrep", kernelcfg, "/proc/config.gz") content, err := usernsCmd.CombinedOutput() if err != nil { enableUserNS = false logUsernsDetection(fmt.Errorf("Failed to get kernel config: %s", err)) return } cfgVal := strings.Trim(string(content), "\n\t ") enableUserNS = cfgVal == kernelcfg+"=y" logUsernsDetection(fmt.Errorf("%s not enabled in kernel config", kernelcfg)) } func TestExecuteRforkUserNS(t *testing.T) { if !enableUserNS { t.Skip("User namespace not enabled") return } f, teardown := setup(t) defer teardown() err := f.shell.Exec("rfork test", ` rfork u { id -u } `) if err != nil { t.Error(err) return } if string(f.shellOut.Bytes()) != "0\n" { t.Errorf("User namespace not supported in your kernel: %s", string(f.shellOut.Bytes())) return } } func TestExecuteRforkEnvVars(t *testing.T) { if !enableUserNS { t.Skip("User namespace not enabled") return } f, teardown := setup(t) defer teardown() sh := f.shell err := sh.Exec("test env", `var abra = "cadabra" setenv abra rfork up { echo $abra }`) if err != nil { t.Error(err) return } } func TestExecuteRforkUserNSNested(t *testing.T) { if !enableUserNS { t.Skip("User namespace not enabled") return } var out bytes.Buffer f, teardown := setup(t) defer teardown() sh := f.shell sh.SetStdout(&out) err := sh.Exec("rfork userns nested", ` rfork u { id -u rfork u { id -u } } `) if err != nil { t.Error(err) return } if string(out.Bytes()) != "0\n0\n" { t.Errorf("User namespace not supported in your kernel") return } } ================================================ FILE: internal/sh/shell_regression_test.go ================================================ package sh_test import ( "io/ioutil" "os" "strings" "testing" "path" "fmt" ) func TestExecuteIssue68(t *testing.T) { f, cleanup := setup(t) defer cleanup() sh := f.shell tmpDir, err := ioutil.TempDir("", "nash-tests") if err != nil { t.Fatal(err) } file := path.Join(tmpDir, "la") err = sh.Exec("-input-", fmt.Sprintf(`echo lalalala | grep la > %s`, file)) if err != nil { t.Error(err) return } defer os.Remove(file) contents, err := ioutil.ReadFile(file) if err != nil { t.Fatal(err) } contentStr := strings.TrimSpace(string(contents)) if contentStr != "lalalala" { t.Errorf("Strings differ: '%s' != '%s'", contentStr, "lalalala") return } } func TestExecuteErrorSuppression(t *testing.T) { f, cleanup := setup(t) defer cleanup() sh := f.shell err := sh.Exec("-input-", `-bllsdlfjlsd`) if err != nil { t.Errorf("Expected to not fail...: %s", err.Error()) return } // issue #72 err = sh.Exec("-input-", `echo lalala | -grep lelele`) if err != nil { t.Errorf("Expected to not fail...:(%s)", err.Error()) return } } ================================================ FILE: internal/sh/shell_test.go ================================================ package sh_test import ( "bytes" "fmt" "io/ioutil" "net" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" "testing" "time" // FIXME: depending on other sh package on the internal sh tests seems very odd shtypes "github.com/madlambda/nash/sh" "github.com/madlambda/nash/internal/sh" "github.com/madlambda/nash/internal/sh/internal/fixture" "github.com/madlambda/nash/tests" ) type ( execTestCase struct { desc string code string expectedStdout string expectedStderr string expectedErr string expectedPrefixErr string } testFixture struct { shell *sh.Shell shellOut *bytes.Buffer dir string envDirs fixture.NashDirs nashdPath string } ) func TestInitEnv(t *testing.T) { os.Setenv("TEST", "abc=123=") f, teardown := setup(t) defer teardown() testEnv, ok := f.shell.Getenv("TEST") if !ok { t.Fatal("environment TEST not found") } expectedTestEnv := "abc=123=" if testEnv.String() != expectedTestEnv { t.Fatalf("Expected TEST Env differs: '%s' != '%s'", testEnv, expectedTestEnv) } } func TestExecuteFile(t *testing.T) { type fileTests struct { path string expected string execBefore string } f, teardown := setup(t) defer teardown() for _, ftest := range []fileTests{ {path: "/ex1.sh", expected: "hello world\n"}, {path: "/sieve.sh", expected: "\n", execBefore: `var ARGS = ("" "0")`}, {path: "/sieve.sh", expected: "\n", execBefore: `var ARGS = ("" "1")`}, {path: "/sieve.sh", expected: "2 \n", execBefore: `var ARGS = ("" "2")`}, {path: "/sieve.sh", expected: "2 3 \n", execBefore: `var ARGS = ("" "3")`}, {path: "/sieve.sh", expected: "2 3 \n", execBefore: `var ARGS = ("" "4")`}, {path: "/sieve.sh", expected: "2 3 5 \n", execBefore: `var ARGS = ("" "5")`}, {path: "/sieve.sh", expected: "2 3 5 7 \n", execBefore: `var ARGS = ("" "10")`}, {path: "/fibonacci.sh", expected: "1 \n", execBefore: `var ARGS = ("" "1")`}, {path: "/fibonacci.sh", expected: "1 2 \n", execBefore: `var ARGS = ("" "2")`}, {path: "/fibonacci.sh", expected: "1 2 3 \n", execBefore: `var ARGS = ("" "3")`}, {path: "/fibonacci.sh", expected: "1 2 3 5 8 \n", execBefore: `var ARGS = ("" "5")`}, } { testExecuteFile(t, f.dir+ftest.path, ftest.expected, ftest.execBefore) } } func TestExecuteCommand(t *testing.T) { echopath, err := exec.LookPath("echo") if err != nil { t.Fatal(err) } echodir := filepath.Dir(echopath) for _, test := range []execTestCase{ { desc: "command failed", code: `non-existing-program`, expectedStdout: "", expectedStderr: "", expectedPrefixErr: `exec: "non-existing-program": executable file not found in `, }, { desc: "err ignored", code: `-non-existing-program`, expectedStdout: "", expectedStderr: "", expectedErr: "", }, { desc: "hello world", code: "echo -n hello world", expectedStdout: "hello world", expectedStderr: "", expectedErr: "", }, { desc: "cmd with concat", code: `echo -n "hello " + "world"`, expectedStdout: "hello world", expectedStderr: "", expectedErr: "", }, { desc: "local command", code: fmt.Sprintf(`var echodir = "%s" chdir($echodir) ./echo -n hello `, strings.Replace(echodir, "\\", "\\\\", -1)), expectedStdout: "hello", expectedStderr: "", expectedErr: "", }, } { testExec(t, test) } } func TestExecuteAssignment(t *testing.T) { for _, test := range []execTestCase{ { // wrong assignment desc: "wrong assignment", code: `var name=i4k`, expectedStdout: "", expectedStderr: "", expectedErr: "wrong assignment:1:9: Unexpected token IDENT. Expecting VARIABLE, STRING or (", }, { desc: "assignment", code: `var name="i4k" echo $name`, expectedStdout: "i4k\n", expectedStderr: "", expectedErr: "", }, { desc: "list assignment", code: `var name=(honda civic) echo -n $name`, expectedStdout: "honda civic", expectedStderr: "", expectedErr: "", }, { desc: "list of lists", code: `var l = ( (name Archlinux) (arch amd64) (kernel 4.7.1) ) echo $l[0] echo $l[1] echo -n $l[2]`, expectedStdout: `name Archlinux arch amd64 kernel 4.7.1`, expectedStderr: "", expectedErr: "", }, { desc: "list assignment", code: `var l = (0 1 2 3) l[0] = "666" echo -n $l`, expectedStdout: `666 1 2 3`, expectedStderr: "", expectedErr: "", }, { desc: "list assignment", code: `var l = (0 1 2 3) var a = "2" l[$a] = "666" echo -n $l`, expectedStdout: `0 1 666 3`, expectedStderr: "", expectedErr: "", }, } { testExec(t, test) } } func TestExecuteMultipleAssignment(t *testing.T) { for _, test := range []execTestCase{ { desc: "multiple assignment", code: `var _1, _2 = "1", "2" echo -n $_1 $_2`, expectedStdout: "1 2", expectedStderr: "", expectedErr: "", }, { desc: "multiple assignment", code: `var _1, _2, _3 = "1", "2", "3" echo -n $_1 $_2 $_3`, expectedStdout: "1 2 3", expectedStderr: "", expectedErr: "", }, { desc: "multiple assignment", code: `var _1, _2 = (), () echo -n $_1 $_2`, expectedStdout: "", expectedStderr: "", expectedErr: "", }, { desc: "multiple assignment", code: `var _1, _2 = (1 2 3 4 5), (6 7 8 9 10) echo -n $_1 $_2`, expectedStdout: "1 2 3 4 5 6 7 8 9 10", expectedStderr: "", expectedErr: "", }, { desc: "multiple assignment", code: `var _1, _2, _3, _4, _5, _6, _7, _8, _9, _10 = "1", "2", "3", "4", "5", "6", "7", "8", "9", "10" echo -n $_1 $_2 $_3 $_4 $_5 $_6 $_7 $_8 $_9 $_10`, expectedStdout: "1 2 3 4 5 6 7 8 9 10", expectedStderr: "", expectedErr: "", }, { desc: "multiple assignment", code: `var _1, _2 = (a b c), "d" echo -n $_1 $_2`, expectedStdout: "a b c d", expectedStderr: "", expectedErr: "", }, { desc: "multiple assignment", code: `fn a() { echo -n "a" } fn b() { echo -n "b" } var _a, _b = $a, $b $_a(); $_b()`, expectedStdout: "ab", expectedStderr: "", expectedErr: "", }, } { testExec(t, test) } } func TestExecuteCmdAssignment(t *testing.T) { for _, test := range []execTestCase{ { desc: "cmd assignment", code: `var name <= echo -n i4k echo -n $name`, expectedStdout: "i4k", expectedStderr: "", expectedErr: "", }, { desc: "list cmd assignment", code: `var name <= echo "honda civic" echo -n $name`, expectedStdout: "honda civic", expectedStderr: "", expectedErr: "", }, { desc: "wrong cmd assignment", code: `var name <= ""`, expectedStdout: "", expectedStderr: "", expectedErr: "wrong cmd assignment:1:13: Invalid token STRING. Expected command or function invocation", }, { desc: "fn must return value", code: `fn e() {} var v <= e()`, expectedStdout: "", expectedStderr: "", expectedErr: ":2:29: Functions returns 0 objects, but statement expects 1", }, { desc: "list assignment", code: `var l = (0 1 2 3) l[0] <= echo -n 666 echo -n $l`, expectedStdout: `666 1 2 3`, expectedStderr: "", expectedErr: "", }, { desc: "list assignment", code: `var l = (0 1 2 3) var a = "2" l[$a] <= echo -n "666" echo -n $l`, expectedStdout: `0 1 666 3`, expectedStderr: "", expectedErr: "", }, } { testExec(t, test) } } func TestExecuteCmdMultipleAssignment(t *testing.T) { for _, test := range []execTestCase{ { desc: "cmd assignment", code: `var name, err <= echo -n i4k if $err == "0" { echo -n $name }`, expectedStdout: "i4k", expectedStderr: "", expectedErr: "", }, { desc: "list cmd assignment", code: `var name, err2 <= echo "honda civic" if $err2 == "0" { echo -n $name }`, expectedStdout: "honda civic", expectedStderr: "", expectedErr: "", }, { desc: "wrong cmd assignment", code: `var name, err <= ""`, expectedStdout: "", expectedStderr: "", expectedErr: "wrong cmd assignment:1:18: Invalid token STRING. Expected command or function invocation", }, { desc: "fn must return value", code: `fn e() {} var v, err <= e()`, expectedStdout: "", expectedStderr: "", expectedErr: ":2:29: Functions returns 0 objects, but statement expects 2", }, { desc: "list assignment", code: `var l = (0 1 2 3) var l[0], err <= echo -n 666 if $err == "0" { echo -n $l }`, expectedStdout: `666 1 2 3`, expectedStderr: "", expectedErr: "", }, { desc: "list assignment", code: `var l = (0 1 2 3) var a = "2" var l[$a], err <= echo -n "666" if $err == "0" { echo -n $l }`, expectedStdout: `0 1 666 3`, expectedStderr: "", expectedErr: "", }, { desc: "cmd assignment works with 1 or 2 variables", code: "var out, err, status <= echo something", expectedStdout: "", expectedStderr: "", expectedErr: "", }, { desc: "ignore error", code: `var out, _ <= cat /file-not-found/test >[2=] echo -n $out`, expectedStdout: "", expectedStderr: "", expectedErr: "", }, { desc: "exec without '-' and getting status still fails", code: `var out <= cat /file-not-found/test >[2=] echo $out`, expectedStdout: "", expectedStderr: "", expectedErr: "exit status 1", }, { desc: "check status", code: `var out, status <= cat /file-not-found/test >[2=] if $status == "0" { echo -n "must fail.. sniff" } else if $status == "1" { echo -n "it works" } else { echo -n "unexpected status:" $status } `, expectedStdout: "it works", expectedStderr: "", expectedErr: "", }, { desc: "multiple return in functions", code: `fn fun() { return "1", "2" } var a, b <= fun() echo -n $a $b`, expectedStdout: "1 2", expectedStderr: "", expectedErr: "", }, } { testExec(t, test) } } func TestExecuteRedirection(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell pathobj, err := ioutil.TempFile("", "nash-redir") if err != nil { t.Fatal(err) } path := strings.Replace(pathobj.Name(), "\\", "\\\\", -1) defer os.Remove(path) err = shell.Exec("redirect", fmt.Sprintf(` echo -n "hello world" > %s `, path)) if err != nil { t.Fatal(err) } content, err := ioutil.ReadFile(path) if err != nil { t.Fatal(err) } if string(content) != "hello world" { t.Fatalf("File differ: '%s' != '%s'", string(content), "hello world") } // Test redirection truncate the file err = shell.Exec("redirect", fmt.Sprintf(` echo -n "a" > %s `, path)) if err != nil { t.Fatal(err) } content, err = ioutil.ReadFile(path) if err != nil { t.Fatal(err) } if string(content) != "a" { t.Fatalf("File differ: '%s' != '%s'", string(content), "a") } // Test redirection to variable err = shell.Exec("redirect", ` var location = "`+path+`" echo -n "hello world" > $location `) if err != nil { t.Fatal(err) } content, err = ioutil.ReadFile(path) if err != nil { t.Error(err) return } if string(content) != "hello world" { t.Errorf("File differ: '%s' != '%s'", string(content), "hello world") return } // Test redirection to concat err = shell.Exec("redirect", fmt.Sprintf(` location = "%s" var a = ".2" echo -n "hello world" > $location+$a `, path)) if err != nil { t.Fatal(err) } defer os.Remove(path + ".2") content, err = ioutil.ReadFile(path + ".2") if err != nil { t.Fatal(err) } if string(content) != "hello world" { t.Fatalf("File differ: '%s' != '%s'", string(content), "hello world") } } func TestExecuteRedirectionMap(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell tmpfile, err := ioutil.TempFile("", "nash-redir-map") if err != nil { t.Fatal(err) } //path := strings.Replace(tmpfile.Name(), "\\", "\\\\", -1) defer os.Remove(tmpfile.Name()) err = shell.Exec("redirect map", fmt.Sprintf(` echo -n "hello world" > %s `, tmpfile.Name())) if err != nil { t.Error(err) return } content, err := ioutil.ReadFile(tmpfile.Name()) if err != nil { t.Fatal(err) } if string(content) != "hello world" { t.Fatalf("File differ: '%s' != '%s'", string(content), "hello world") } } func TestExecuteSetenv(t *testing.T) { f, teardown := setup(t) defer teardown() for _, test := range []execTestCase{ { desc: "test setenv basic", code: `var setenvtest = "hello" setenv setenvtest ` + f.nashdPath + ` -c "echo $setenvtest"`, expectedStdout: "hello\n", expectedStderr: "", expectedErr: "", }, { desc: "test setenv assignment", code: `setenv setenvtest = "hello" ` + f.nashdPath + ` -c "echo $setenvtest"`, expectedStdout: "hello\n", expectedStderr: "", expectedErr: "", }, { desc: "test setenv exec cmd", code: `setenv setenvtest <= echo -n "hello" ` + f.nashdPath + ` -c "echo $setenvtest"`, expectedStdout: "hello\n", expectedStderr: "", expectedErr: "", }, { desc: "test setenv semicolon", code: `setenv a setenv b`, expectedStdout: "", expectedStderr: "", expectedErr: "test setenv semicolon:1:9: Unexpected token setenv, expected semicolon (;) or EOL", }, } { testExec(t, test) } } func TestExecuteCd(t *testing.T) { tmpdir, err := ioutil.TempDir("", "nash-cd") if err != nil { t.Fatal(err) } tmpdir, err = filepath.EvalSymlinks(tmpdir) if err != nil { t.Fatal(err) } tmpdirEscaped := strings.Replace(tmpdir, "\\", "\\\\", -1) homeEnvVar := "HOME" if runtime.GOOS == "windows" { homeEnvVar = "HOMEPATH" // hack to use nash's pwd instead of gnu on windows projectDir := filepath.FromSlash(tests.Projectpath) pwdDir := filepath.Join(projectDir, "stdbin", "pwd") path := os.Getenv("Path") defer os.Setenv("Path", path) // TODO(i4k): very unsafe os.Setenv("Path", pwdDir+";"+path) } for _, test := range []execTestCase{ { desc: "test cd 1", code: fmt.Sprintf(`cd %s pwd`, tmpdir), expectedStdout: tmpdir + "\n", expectedStderr: "", expectedErr: "", }, { desc: "test cd 2", code: fmt.Sprintf(`%s = "%s" setenv %s cd pwd`, homeEnvVar, tmpdirEscaped, homeEnvVar), expectedStdout: tmpdir + "\n", expectedStderr: "", expectedErr: "", }, { desc: "test cd into $var", code: fmt.Sprintf(` var v = "%s" cd $v pwd`, tmpdirEscaped), expectedStdout: tmpdir + "\n", expectedStderr: "", expectedErr: "", }, } { t.Run(test.desc, func(t *testing.T) { test := test testInteractiveExec(t, test) }) } } func TestExecuteImport(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut tmpfile, err := ioutil.TempFile("", "nash-import") if err != nil { t.Fatal(err) } defer os.Remove(tmpfile.Name()) _, err = tmpfile.Write([]byte(`var TESTE="teste"`)) if err != nil { t.Fatal(err) } fnameEscaped := strings.Replace(tmpfile.Name(), "\\", "\\\\", -1) err = shell.Exec("test import", fmt.Sprintf(`import %s echo $TESTE `, fnameEscaped)) if err != nil { t.Error(err) return } if strings.TrimSpace(string(out.Bytes())) != "teste" { t.Error("Import does not work") return } } func TestExecuteIfEqual(t *testing.T) { for _, test := range []execTestCase{ { desc: "if equal", code: ` if "" == "" { echo "empty string works" }`, expectedStdout: "empty string works\n", expectedStderr: "", expectedErr: "", }, { desc: "if equal", code: ` if "i4k" == "_i4k_" { echo "do not print" }`, expectedStdout: "", expectedStderr: "", expectedErr: "", }, { desc: "if lvalue concat", code: ` if "i4"+"k" == "i4k" { echo -n "ok" }`, expectedStdout: "ok", expectedStderr: "", expectedErr: "", }, { desc: "if lvalue concat", code: `var name = "something" if $name+"k" == "somethingk" { echo -n "ok" }`, expectedStdout: "ok", expectedStderr: "", expectedErr: "", }, { desc: "if lvalue concat", code: `var name = "something" if $name+"k"+"k" == "somethingkk" { echo -n "ok" }`, expectedStdout: "ok", expectedStderr: "", expectedErr: "", }, { desc: "if rvalue concat", code: ` if "i4k" == "i4"+"k" { echo -n "ok" }`, expectedStdout: "ok", expectedStderr: "", expectedErr: "", }, { desc: "if lvalue funcall", code: `var a = () if len($a) == "0" { echo -n "ok" }`, expectedStdout: "ok", expectedStderr: "", expectedErr: "", }, { desc: "if rvalue funcall", code: `var a = ("1") if "1" == len($a) { echo -n "ok" }`, expectedStdout: "ok", expectedStderr: "", expectedErr: "", }, { desc: "if lvalue funcall with concat", code: `var a = () if len($a)+"1" == "01" { echo -n "ok" }`, expectedStdout: "ok", expectedStderr: "", expectedErr: "", }, } { testExec(t, test) } } func TestExecuteIfElse(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("test if else", ` if "" == "" { echo "if still works" } else { echo "nop" }`) if err != nil { t.Error(err) return } if strings.TrimSpace(string(out.Bytes())) != "if still works" { t.Errorf("'%s' != 'if still works'", strings.TrimSpace(string(out.Bytes()))) return } out.Reset() err = shell.Exec("test if equal 2", ` if "i4k" == "_i4k_" { echo "do not print" } else { echo "print this" }`) if err != nil { t.Error(err) return } if strings.TrimSpace(string(out.Bytes())) != "print this" { t.Errorf("Error: '%s' != 'print this'", strings.TrimSpace(string(out.Bytes()))) return } } func TestExecuteIfElseIf(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("test if else", ` if "" == "" { echo "if still works" } else if "bleh" == "bloh" { echo "nop" }`) if err != nil { t.Error(err) return } if strings.TrimSpace(string(out.Bytes())) != "if still works" { t.Errorf("'%s' != 'if still works'", strings.TrimSpace(string(out.Bytes()))) return } out.Reset() err = shell.Exec("test if equal 2", ` if "i4k" == "_i4k_" { echo "do not print" } else if "a" != "b" { echo "print this" }`) if err != nil { t.Error(err) return } if strings.TrimSpace(string(out.Bytes())) != "print this" { t.Errorf("Error: '%s' != 'print this'", strings.TrimSpace(string(out.Bytes()))) return } } func TestExecuteFnDecl(t *testing.T) { f, teardown := setup(t) defer teardown() err := f.shell.Exec("test fnDecl", ` fn build(image, debug) { ls }`) if err != nil { t.Error(err) return } } func TestExecuteFnInv(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("test fn inv", ` fn getints() { return ("1" "2" "3" "4" "5" "6" "7" "8" "9" "0") } var integers <= getints() echo -n $integers `) if err != nil { t.Error(err) return } if string(out.Bytes()) != "1 2 3 4 5 6 7 8 9 0" { t.Errorf("'%s' != '%s'", string(out.Bytes()), "1 2 3 4 5 6 7 8 9 0") return } out.Reset() // Test fn scope err = shell.Exec("test fn inv", ` var OUTSIDE = "some value" fn getOUTSIDE() { return $OUTSIDE } var val <= getOUTSIDE() echo -n $val `) if err != nil { t.Error(err) return } if string(out.Bytes()) != "some value" { t.Errorf("'%s' != '%s'", string(out.Bytes()), "some value") return } err = shell.Exec("test fn inv", ` fn notset() { var INSIDE = "camshaft" } notset() echo -n $INSIDE `) if err == nil { t.Error("Must fail") return } out.Reset() // test variables shadow the global ones err = shell.Exec("test shadow", `var _path="AAA" fn test(_path) { echo -n $_path } test("BBB") `) if string(out.Bytes()) != "BBB" { t.Errorf("String differs: '%s' != '%s'", string(out.Bytes()), "BBB") return } out.Reset() err = shell.Exec("test shadow", ` fn test(_path) { echo -n $_path } _path="AAA" test("BBB") `) if string(out.Bytes()) != "BBB" { t.Errorf("String differs: '%s' != '%s'", string(out.Bytes()), "BBB") return } out.Reset() err = shell.Exec("test fn list arg", ` var ids_luns = () var id = "1" var lun = "lunar" var ids_luns <= append($ids_luns, ($id $lun)) print(len($ids_luns))`) if err != nil { t.Error(err) return } got := string(out.Bytes()) expected := "1" if got != expected { t.Fatalf("String differs: '%s' != '%s'", got, expected) } } func TestFnComposition(t *testing.T) { for _, test := range []execTestCase{ { desc: "composition", code: ` fn a(b) { echo -n $b } fn b() { return "hello" } a(b()) `, expectedStdout: "hello", expectedStderr: "", expectedErr: "", }, { desc: "composition", code: ` fn a(b, c) { echo -n $b $c } fn b() { return "hello" } fn c() { return "world" } a(b(), c()) `, expectedStdout: "hello world", expectedStderr: "", expectedErr: "", }, } { testExec(t, test) } } func TestExecuteFnInvOthers(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("test fn inv", ` fn _getints() { return ("1" "2" "3" "4" "5" "6" "7" "8" "9" "0") } fn getints() { var values <= _getints() return $values } var integers <= getints() echo -n $integers `) if err != nil { t.Error(err) return } if string(out.Bytes()) != "1 2 3 4 5 6 7 8 9 0" { t.Errorf("'%s' != '%s'", string(out.Bytes()), "1 2 3 4 5 6 7 8 9 0") return } } func TestNonInteractive(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell shell.SetInteractive(true) testShellExec(t, shell, execTestCase{ desc: "test bindfn interactive", code: ` fn greeting() { echo "Hello" } bindfn greeting hello`, }) shell.SetInteractive(false) // FIXME: using private stuff on tests ? // shell.filename = "" t.Skip("FIXME: TEST USES PRIVATE STUFF") expectedErr := ":1:0: " + "'hello' is a bind to 'greeting'." + " No binds allowed in non-interactive mode." testShellExec(t, shell, execTestCase{ desc: "test 'binded' function non-interactive", code: `hello`, expectedStdout: "", expectedStderr: "", expectedErr: expectedErr, }) expectedErr = ":6:8: 'bindfn' is not allowed in" + " non-interactive mode." testShellExec(t, shell, execTestCase{ desc: "test bindfn non-interactive", code: ` fn goodbye() { echo "Ciao" } bindfn goodbye ciao`, expectedStdout: "", expectedStderr: "", expectedErr: expectedErr, }) } func TestExecuteBindFn(t *testing.T) { for _, test := range []execTestCase{ { desc: "test bindfn", code: ` fn cd() { echo "override builtin cd" } bindfn cd cd cd `, expectedStdout: "override builtin cd\n", }, { desc: "test bindfn vargs", code: ` fn echoargs(args...) { for a in $args { echo $a } } bindfn echoargs echoargs echoargs echoargs "a" echoargs "b" "c" `, expectedStdout: "a\nb\nc\n", }, { desc: "test empty bindfn vargs len", code: ` fn echoargs(args...) { var l <= len($args) echo $l } bindfn echoargs echoargs echoargs `, expectedStdout: "0\n", }, { desc: "test bindfn args", code: ` fn foo(line) { echo $line } bindfn foo bar bar test test `, expectedErr: "Wrong number of arguments for function foo. Expected 1 but found 2", }, } { t.Run(test.desc, func(t *testing.T) { testInteractiveExec(t, test) }) } } func TestExecutePipe(t *testing.T) { var stderr bytes.Buffer var stdout bytes.Buffer f, teardown := setup(t) defer teardown() // Case 1 cmd := exec.Command(f.nashdPath, "-c", `echo hello | tr -d "[:space:]"`) cmd.Stderr = &stderr cmd.Stdout = &stdout err := cmd.Run() if err != nil { t.Errorf("Unexpected error: %s", err.Error()) } expectedOutput := "hello" actualOutput := string(stdout.Bytes()) if actualOutput != expectedOutput { t.Errorf("'%s' != '%s'", actualOutput, expectedOutput) return } stdout.Reset() stderr.Reset() // Case 2 cmd = exec.Command(f.nashdPath, "-c", `echo hello | wc -l | tr -d "[:space:]"`) cmd.Stderr = &stderr cmd.Stdout = &stdout err = cmd.Run() if err != nil { t.Errorf("Unexpected error: %s", err.Error()) } expectedOutput = "1" actualOutput = string(stdout.Bytes()) if actualOutput != expectedOutput { t.Errorf("'%s' != '%s'", actualOutput, expectedOutput) return } } func TestExecuteRedirectionPipe(t *testing.T) { f, teardown := setup(t) defer teardown() err := f.shell.Exec("test", `cat stuff >[2=] | grep file`) expectedErr := ":1:16: exit status 1|success" if err == nil { t.Fatalf("expected err[%s]", expectedErr) } if err.Error() != expectedErr { t.Errorf("Expected stderr to be '%s' but got '%s'", expectedErr, err.Error()) return } } func testTCPRedirection(t *testing.T, port, command string) { message := "hello world" done := make(chan error) l, err := net.Listen("tcp", port) if err != nil { t.Fatal(err) } defer l.Close() go func() { f, teardown := setup(t) defer teardown() err := <-done if err != nil { t.Fatal(err) } done <- f.shell.Exec("test net redirection", command) }() done <- nil // synchronize peers conn, err := l.Accept() if err != nil { done <- err t.Fatal(err) } defer conn.Close() err = <-done if err != nil { t.Fatal(err) } buf, err := ioutil.ReadAll(conn) if err != nil { t.Fatal(err) } if msg := string(buf[:]); msg != message { t.Fatalf("Unexpected message:\nGot:\t\t%s\nExpected:\t%s\n", msg, message) } } func TestTCPRedirection(t *testing.T) { testTCPRedirection(t, ":4666", `echo -n "hello world" >[1] "tcp://localhost:4666"`) testTCPRedirection(t, ":4667", `echo -n "hello world" > "tcp://localhost:4667"`) } func TestExecuteUnixRedirection(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("windows does not support unix socket") return } message := "hello world" sockDir, err := ioutil.TempDir("", "nash-tests") if err != nil { t.Error(err) return } sockFile := sockDir + "/listen.sock" defer func() { os.Remove(sockFile) os.RemoveAll(sockDir) }() done := make(chan bool) writeDone := make(chan bool) go func() { f, teardown := setup(t) defer teardown() defer func() { writeDone <- true }() <-done err = f.shell.Exec("test net redirection", `echo -n "`+message+`" >[1] "unix://`+sockFile+`"`) if err != nil { t.Error(err) return } }() l, err := net.Listen("unix", sockFile) if err != nil { t.Error(err) return } defer l.Close() go func() { conn, err := l.Accept() if err != nil { return } defer conn.Close() buf, err := ioutil.ReadAll(conn) if err != nil { t.Fatal(err) } fmt.Println(string(buf[:])) if msg := string(buf[:]); msg != message { t.Fatalf("Unexpected message:\nGot:\t\t%s\nExpected:\t%s\n", msg, message) } return // Done }() done <- true <-writeDone } func TestExecuteUDPRedirection(t *testing.T) { message := "hello world" done := make(chan bool) writeDone := make(chan bool) go func() { f, teardown := setup(t) defer teardown() defer func() { writeDone <- true }() <-done err := f.shell.Exec("test net redirection", `echo -n "`+message+`" >[1] "udp://localhost:6667"`) if err != nil { t.Error(err) return } }() serverAddr, err := net.ResolveUDPAddr("udp", ":6667") if err != nil { t.Error(err) return } l, err := net.ListenUDP("udp", serverAddr) if err != nil { t.Fatal(err) } go func() { defer l.Close() buf := make([]byte, 1024) nb, _, err := l.ReadFromUDP(buf) if err != nil { t.Error(err) return } received := string(buf[:nb]) if received != message { t.Errorf("Unexpected message:\nGot:\t\t'%s'\nExpected:\t'%s'\n", received, message) } }() time.Sleep(time.Second * 1) done <- true <-writeDone } func TestExecuteReturn(t *testing.T) { for _, test := range []execTestCase{ { desc: "return invalid", code: `return`, expectedStdout: "", expectedStderr: "", expectedErr: ":1:0: Unexpected return outside of function declaration.", }, { desc: "test simple return", code: `fn test() { return } test()`, expectedStdout: "", expectedStderr: "", expectedErr: "", }, { desc: "return must finish func evaluation", code: `fn test() { if "1" == "1" { return "1" } return "0" } var res <= test() echo -n $res`, expectedStdout: "1", expectedStderr: "", expectedErr: "", }, { desc: "ret from for", code: `fn test() { var values = (0 1 2 3 4 5 6 7 8 9) for i in $values { if $i == "5" { return $i } } return "0" } var a <= test() echo -n $a`, expectedStdout: "5", expectedStderr: "", expectedErr: "", }, { desc: "inf loop ret", code: `fn test() { for { if "1" == "1" { return "1" } } # never happen return "bleh" } var a <= test() echo -n $a`, expectedStdout: "1", expectedStderr: "", expectedErr: "", }, { desc: "test returning funcall", code: `fn a() { return "1" } fn b() { return a() } var c <= b() echo -n $c`, expectedStdout: "1", expectedStderr: "", expectedErr: "", }, } { testExec(t, test) } } func TestExecuteFnAsFirstClass(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("test fn by arg", ` fn printer(val) { echo -n $val } fn success(print, val) { $print("[SUCCESS] " + $val) } success($printer, "Command executed!") `) if err != nil { t.Error(err) return } expected := `[SUCCESS] Command executed!` if expected != string(out.Bytes()) { t.Errorf("Differs: '%s' != '%s'", expected, string(out.Bytes())) return } } func TestExecuteConcat(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("", `var a = "A" var b = "B" var c = $a + $b + "C" echo -n $c`) if err != nil { t.Error(err) return } if string(out.Bytes()) != "ABC" { t.Errorf("Must be equal. '%s' != '%s'", string(out.Bytes()), "ABC") return } out.Reset() err = shell.Exec("concat indexed var", `var tag = (Name some) echo -n "Key="+$tag[0]+",Value="+$tag[1]`) if err != nil { t.Error(err) return } expected := "Key=Name,Value=some" if expected != string(out.Bytes()) { t.Errorf("String differs: '%s' != '%s'", expected, string(out.Bytes())) return } } func TestExecuteFor(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("simple loop", `var files = (/etc/passwd /etc/shells) for f in $files { echo $f echo "loop" }`) if err != nil { t.Error(err) return } expected := `/etc/passwd loop /etc/shells loop` value := strings.TrimSpace(string(out.Bytes())) if value != expected { t.Errorf("String differs: '%s' != '%s'", expected, value) return } } func TestExecuteInfiniteLoop(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell doneCtrlc := make(chan bool) doneLoop := make(chan bool) go func() { fmt.Printf("Waiting 2 second to abort infinite loop") time.Sleep(2 * time.Second) err := shell.TriggerCTRLC() if err != nil { t.Fatal(err) } doneCtrlc <- true }() go func() { err := shell.Exec("simple loop", `for { echo "infinite loop" >[1=] sleep 1 }`) doneLoop <- true if err == nil { t.Errorf("Must fail with interrupted error") return } type interrupted interface { Interrupted() bool } if errInterrupted, ok := err.(interrupted); !ok || !errInterrupted.Interrupted() { t.Errorf("Loop not interrupted properly") return } }() for i := 0; i < 2; i++ { select { case <-doneCtrlc: fmt.Printf("CTRL-C Sent to subshell\n") case <-doneLoop: fmt.Printf("Loop finished.\n") case <-time.After(5 * time.Second): t.Errorf("Failed to stop infinite loop") return } } } func TestExecuteVariableIndexing(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("indexing", `var list = ("1" "2" "3") echo -n $list[0]`) if err != nil { t.Error(err) return } result := strings.TrimSpace(string(out.Bytes())) expected := "1" if expected != result { t.Errorf("Fail: '%s' != '%s'", expected, result) return } out.Reset() err = shell.Exec("indexing", `var i = "0" echo -n $list[$i]`) if err != nil { t.Error(err) return } result = strings.TrimSpace(string(out.Bytes())) expected = "1" if expected != result { t.Errorf("Fail: '%s' != '%s'", expected, result) return } out.Reset() err = shell.Exec("indexing", `var tmp <= seq 0 2 var seq <= split($tmp, "\n") for i in $seq { echo -n $list[$i] }`) if err != nil { t.Error(err) return } result = strings.TrimSpace(string(out.Bytes())) expected = "123" if expected != result { t.Errorf("Fail: '%s' != '%s'", expected, result) return } out.Reset() err = shell.Exec("indexing", `echo -n $list[5]`) if err == nil { t.Error("Must fail. Out of bounds") return } out.Reset() err = shell.Exec("indexing", `var a = ("0") echo -n $list[$a[0]]`) if err != nil { t.Error(err) return } result = strings.TrimSpace(string(out.Bytes())) expected = "1" if expected != result { t.Errorf("Fail: '%s' != '%s'", expected, result) return } } func TestExecuteSubShellDoesNotOverwriteparentEnv(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("set env", `setenv SHELL = "bleh"`) if err != nil { t.Error(err) return } err = shell.Exec("set env from fn", `fn test() { # test() should not call the setup func in Nash } test() echo -n $SHELL`) if err != nil { t.Error(err) return } if string(out.Bytes()) != "bleh" { t.Errorf("Differ: '%s' != '%s'", "bleh", string(out.Bytes())) return } } func TestExecuteInterruptDoesNotCancelLoop(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell shell.TriggerCTRLC() time.Sleep(time.Second * 1) err := shell.Exec("interrupting loop", `var seq = (1 2 3 4 5) for i in $seq {}`) if err != nil { t.Error(err) return } } func TestExecuteErrorSuppressionAll(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell err := shell.Exec("-input-", `var _, status <= command-not-exists`) if err != nil { t.Errorf("Expected to not fail...: %s", err.Error()) return } // FIXME: depending on other sh package on the internal sh tests seems very odd scode, ok := shell.GetLocalvar("status") if !ok || scode.Type() != shtypes.StringType || scode.String() != strconv.Itoa(sh.ENotFound) { t.Errorf("Invalid status code %v", scode) return } err = shell.Exec("-input-", `var _, status <= echo works`) if err != nil { t.Error(err) return } // FIXME: depending on other sh package on the internal sh tests seems very odd scode, ok = shell.GetLocalvar("status") if !ok || scode.Type() != shtypes.StringType || scode.String() != "0" { t.Errorf("Invalid status code %v", scode) return } err = shell.Exec("-input-", `echo works | cmd-does-not-exists`) if err == nil { t.Errorf("Must fail") return } expectedError := `:1:11: not started|exec: "cmd-does-not-exists": executable file not found in` if !strings.HasPrefix(err.Error(), expectedError) { t.Errorf("Unexpected error: %s", err.Error()) return } } func TestExecuteGracefullyError(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell err := shell.Exec("someinput.sh", "(") if err == nil { t.Errorf("Must fail...") return } expectErr := "someinput.sh:1:1: Multi-line command not finished. Found EOF but expect ')'" if err.Error() != expectErr { t.Errorf("Expect error: %s, but got: %s", expectErr, err.Error()) return } err = shell.Exec("input", "echo(") if err == nil { t.Errorf("Must fail...") return } if err.Error() != "input:1:5: Unexpected token EOF. Expecting STRING, VARIABLE or )" { t.Errorf("Unexpected error: %s", err.Error()) return } } func TestExecuteMultilineCmd(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("test", `(echo -n hello world)`) if err != nil { t.Error(err) return } expected := "hello world" if expected != string(out.Bytes()) { t.Errorf("Expected '%s' but got '%s'", expected, string(out.Bytes())) return } out.Reset() err = shell.Exec("test", `( echo -n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 )`) if err != nil { t.Error(err) return } expected = "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20" if expected != string(out.Bytes()) { t.Errorf("Expected '%s' but got '%s'", expected, string(out.Bytes())) return } } func TestExecuteMultilineCmdAssign(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell out := f.shellOut err := shell.Exec("test", `var val <= (echo -n hello world) echo -n $val`) if err != nil { t.Error(err) return } expected := "hello world" if expected != string(out.Bytes()) { t.Errorf("Expected '%s' but got '%s'", expected, string(out.Bytes())) return } out.Reset() err = shell.Exec("test", `val <= ( echo -n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 ) echo -n $val`) if err != nil { t.Error(err) return } expected = "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20" if expected != string(out.Bytes()) { t.Errorf("Expected '%s' but got '%s'", expected, string(out.Bytes())) return } } func TestExecuteMultiReturnUnfinished(t *testing.T) { f, teardown := setup(t) defer teardown() shell := f.shell err := shell.Exec("test", "(") if err == nil { t.Errorf("Must fail... Must return an unfinished paren error") return } type unfinished interface { Unfinished() bool } if e, ok := err.(unfinished); !ok || !e.Unfinished() { t.Errorf("Must fail with unfinished paren error. Got %s", err.Error()) return } err = shell.Exec("test", `( echo`) if err == nil { t.Errorf("Must fail... Must return an unfinished paren error") return } if e, ok := err.(unfinished); !ok || !e.Unfinished() { t.Errorf("Must fail with unfinished paren error. Got %s", err.Error()) return } err = shell.Exec("test", `( echo hello world`) if err == nil { t.Errorf("Must fail... Must return an unfinished paren error") return } if e, ok := err.(unfinished); !ok || !e.Unfinished() { t.Errorf("Must fail with unfinished paren error. Got %s", err.Error()) return } } func TestExecuteVariadicFn(t *testing.T) { for _, test := range []execTestCase{ { desc: "println", code: `fn println(fmt, arg...) { print($fmt+"\n", $arg...) } println("%s %s", "test", "test")`, expectedStdout: "test test\n", expectedStderr: "", expectedErr: "", }, { desc: "lots of args", code: `fn println(fmt, arg...) { print($fmt+"\n", $arg...) } println("%s%s%s%s%s%s%s%s%s%s", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10")`, expectedStdout: "12345678910\n", expectedStderr: "", expectedErr: "", }, { desc: "passing list to var arg fn", code: `fn puts(arg...) { for a in $arg { echo $a } } var a = ("1" "2" "3" "4" "5") puts($a...)`, expectedErr: "", expectedStdout: "1\n2\n3\n4\n5\n", expectedStderr: "", }, { desc: "passing empty list to var arg fn", code: `fn puts(arg...) { for a in $arg { echo $a } } var a = () puts($a...)`, expectedErr: "", expectedStdout: "", expectedStderr: "", }, { desc: "... expansion", code: `var args = ("plan9" "from" "outer" "space") print("%s %s %s %s", $args...)`, expectedStdout: "plan9 from outer space", }, { desc: "literal ... expansion", code: `print("%s:%s:%s", ("a" "b" "c")...)`, expectedStdout: "a:b:c", }, { desc: "varargs only as last argument", code: `fn println(arg..., fmt) {}`, expectedErr: ":1:11: Vararg 'arg...' isn't the last argument", }, { desc: "variadic argument are optional", code: `fn println(b...) { for v in $b { print($v) } print("\n") } println()`, expectedStdout: "\n", }, { desc: "the first argument isn't optional", code: `fn a(b, c...) { print($b, $c...) } a("test")`, expectedStdout: "test", }, { desc: "the first argument isn't optional", code: `fn a(b, c...) { print($b, $c...) } a()`, expectedErr: ":4:0: Wrong number of arguments for function a. Expected at least 1 arguments but found 0", }, } { testExec(t, test) } } func setup(t *testing.T) (testFixture, func()) { dirs := fixture.SetupNashDirs(t) shell, err := sh.NewAbortShell(dirs.Path, dirs.Root) if err != nil { t.Fatal(err) } var out bytes.Buffer shell.SetStdout(&out) return testFixture{ shell: shell, shellOut: &out, dir: tests.Testdir, envDirs: dirs, nashdPath: tests.Nashcmd, }, dirs.Cleanup } func testExecuteFile(t *testing.T, path, expected string, before string) { f, teardown := setup(t) defer teardown() if before != "" { f.shell.Exec("", before) } err := f.shell.ExecFile(path) if err != nil { t.Error(err) return } if string(f.shellOut.Bytes()) != expected { t.Errorf("Wrong command output: '%s' != '%s'", string(f.shellOut.Bytes()), expected) return } } func testShellExec(t *testing.T, shell *sh.Shell, testcase execTestCase) { t.Helper() var bout bytes.Buffer var berr bytes.Buffer shell.SetStderr(&berr) shell.SetStdout(&bout) err := shell.Exec(testcase.desc, testcase.code) if err != nil { if testcase.expectedPrefixErr != "" { if !strings.HasPrefix(err.Error(), testcase.expectedPrefixErr) { t.Errorf("[%s] Prefix of error differs: Expected prefix '%s' in '%s'", testcase.desc, testcase.expectedPrefixErr, err.Error()) } } else if err.Error() != testcase.expectedErr { t.Errorf("[%s] Error differs: Expected '%s' but got '%s'", testcase.desc, testcase.expectedErr, err.Error()) } } else if testcase.expectedErr != "" { t.Fatalf("Expected error[%s] but got nil", testcase.expectedErr) } if testcase.expectedStdout != string(bout.Bytes()) { t.Errorf("[%s] Stdout differs: '%s' != '%s'", testcase.desc, testcase.expectedStdout, string(bout.Bytes())) return } if testcase.expectedStderr != string(berr.Bytes()) { t.Errorf("[%s] Stderr differs: '%s' != '%s'", testcase.desc, testcase.expectedStderr, string(berr.Bytes())) return } bout.Reset() berr.Reset() } func testExec(t *testing.T, testcase execTestCase) { t.Helper() f, teardown := setup(t) defer teardown() testShellExec(t, f.shell, testcase) } func testInteractiveExec(t *testing.T, testcase execTestCase) { t.Helper() f, teardown := setup(t) defer teardown() f.shell.SetInteractive(true) testShellExec(t, f.shell, testcase) } ================================================ FILE: internal/sh/shell_var_test.go ================================================ package sh_test import ( "fmt" "testing" "github.com/madlambda/nash/tests" ) func TestVarAssign(t *testing.T) { for _, test := range []execTestCase{ { desc: "simple init", code: `var a = "1"; echo -n $a`, expectedStdout: "1", }, { desc: "variable does not exists", code: `a = "1"; echo -n $a`, expectedErr: `:1:0: Variable 'a' is not initialized. Use 'var a = '`, }, { desc: "variable already initialized", code: `var a = "1"; var a = "2"; echo -n $a`, expectedStdout: "2", }, { desc: "variable set", code: `var a = "1"; a = "2"; echo -n $a`, expectedStdout: "2", }, { desc: "global variable set", code: `var global = "1" fn somefunc() { global = "2" } somefunc() echo -n $global`, expectedStdout: "2", }, } { t.Run(test.desc, func(t *testing.T) { testExec(t, test) }) } } func TestVarExecAssign(t *testing.T) { for _, test := range []execTestCase{ { desc: "simple exec var", code: `var heart <= echo -n "feed both wolves" echo -n $heart`, expectedStdout: "feed both wolves", }, { desc: "var do not exists", code: `__a <= echo -n "fury"`, expectedErr: ":1:0: Variable '__a' is not initialized. Use 'var __a = '", }, { desc: "multiple var same name", code: `var a = "1" var a = "2" var a = "3" echo -n $a`, expectedStdout: "3", }, { desc: "multiple var same name with exec", code: `var a <= echo -n "1" var a <= echo -n "hello" echo -n $a`, expectedStdout: "hello", }, { desc: "first variable is stdout", code: `var out <= echo -n "hello" echo -n $out`, expectedStdout: "hello", }, { desc: "two variable, first stdout and second is status", code: `var stdout, status <= echo -n "bleh" echo -n $stdout $status`, expectedStdout: "bleh 0", }, { desc: "three variables, stdout empty, stderr with data, status", code: fmt.Sprintf(`var out, err, st <= %s/write/write /dev/stderr "hello" echo $out echo $err echo -n $st`, tests.Stdbindir), expectedStdout: "\nhello\n0", }, { desc: "three variables, stdout with data, stderr empty, status", code: fmt.Sprintf(`var out, err, st <= %s/write/write /dev/stdout "hello" echo $out echo $err echo -n $st`, tests.Stdbindir), expectedStdout: "hello\n\n0", }, } { t.Run(test.desc, func(t *testing.T) { testExec(t, test) }) } } ================================================ FILE: internal/sh/util.go ================================================ package sh import ( "fmt" "io" "math/rand" "os" "os/exec" "strconv" "syscall" "time" "github.com/madlambda/nash/sh" ) var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") func init() { rand.Seed(time.Now().UnixNano()) } func randRunes(n int) string { b := make([]rune, n) for i := range b { b[i] = letterRunes[rand.Intn(len(letterRunes))] } return string(b) } func buildenv(e Env) []string { env := make([]string, 0, len(e)) for k, v := range e { if v == nil { continue } if v.Type() != sh.ListType && v.Type() != sh.StringType { continue } if v.Type() == sh.ListType { vlist := v.(*sh.ListObj) env = append(env, k+"=("+vlist.String()+")") } else { vstr := v.(*sh.StrObj) env = append(env, k+"="+vstr.String()) } } return env } func printVar(out io.Writer, name string, val sh.Obj) { if val.Type() == sh.StringType { valstr := val.(*sh.StrObj) fmt.Fprintf(out, "%s = \"%s\"\n", name, valstr.Str()) } else if val.Type() == sh.ListType { vallist := val.(*sh.ListObj) fmt.Fprintf(out, "%s = (%s)\n", name, vallist.String()) } } func printEnv(out io.Writer, name string) { fmt.Fprintf(out, "setenv %s\n", name) } func getErrStatus(err error, def string) string { status := def if exiterr, ok := err.(*exec.ExitError); ok { if statusObj, ok := exiterr.Sys().(syscall.WaitStatus); ok { status = strconv.Itoa(statusObj.ExitStatus()) } } return status } func nashdAutoDiscover() string { path, err := os.Readlink("/proc/self/exe") if err != nil { path = os.Args[0] if _, err := os.Stat(path); err != nil { return "" } } return path } ================================================ FILE: internal/sh/util_test.go ================================================ package sh import ( "sort" "testing" "github.com/madlambda/nash/sh" ) func TestBuildEnv(t *testing.T) { env := Env{ "teste": nil, } penv := buildenv(env) if len(penv) != 0 { t.Errorf("Invalid env length") return } env = Env{ "PATH": sh.NewStrObj("/bin:/usr/bin"), } penv = buildenv(env) if len(penv) != 1 { t.Errorf("Invalid env length") return } if penv[0] != "PATH=/bin:/usr/bin" { t.Errorf("Invalid env value: %s", penv[0]) return } env = Env{ "PATH": sh.NewListObj([]sh.Obj{ sh.NewStrObj("/bin"), sh.NewStrObj("/usr/bin"), }), } penv = buildenv(env) if len(penv) != 1 { t.Errorf("Invalid env length") return } if penv[0] != "PATH=(/bin /usr/bin)" { t.Errorf("Invalid env value: %s", penv[0]) return } env = Env{ "PATH": sh.NewListObj([]sh.Obj{ sh.NewStrObj("/bin"), sh.NewStrObj("/usr/bin"), }), "path": sh.NewStrObj("abracadabra"), } penv = buildenv(env) if len(penv) != 2 { t.Errorf("Invalid env length") return } sort.Strings(penv) if penv[0] != "PATH=(/bin /usr/bin)" { t.Errorf("Invalid env value: '%s'", penv[0]) return } if penv[1] != "path=abracadabra" { t.Errorf("Invalid env value: '%s'", penv[1]) return } } ================================================ FILE: internal/testing/fixture/io.go ================================================ package fixture import ( "fmt" "io/ioutil" "math/rand" "os" "path/filepath" "testing" ) // Tmpdir creates a temporary dir and returns a function that can be used // to remove it after usage. Any error on any operation returns on a Fatal // call on the given testing.T. func Tmpdir(t *testing.T) (string, func()) { t.Helper() dir, err := ioutil.TempDir("", "nash-tests") if err != nil { t.Fatal(err) } dir, err = filepath.EvalSymlinks(dir) if err != nil { t.Fatal(err) } return dir, func() { err := os.RemoveAll(dir) if err != nil { t.Fatal(err) } } } // MkdirAll will do the same thing as os.Mkdirall but calling Fatal on // the given testing.T if something goes wrong. func MkdirAll(t *testing.T, nashlib string) { t.Helper() err := os.MkdirAll(nashlib, os.ModePerm) if err != nil { t.Fatal(err) } } // CreateFiles will create all files and its dirs if // necessary calling Fatal on the given testing if anything goes wrong. // // The files contents will be randomly generated strings (not a lot random, // just for test purposes) and will be returned on the map that will map // the filepath to its contents func CreateFiles(t *testing.T, filepaths []string) map[string]string { t.Helper() createdFiles := map[string]string{} for _, f := range filepaths { contents := CreateFile(t, f) createdFiles[f] = contents } return createdFiles } // CreateFile will create the file and its dirs if // necessary calling Fatal on the given testing if anything goes wrong. // // The file content will be randomly generated strings (not a lot random, // just for test purposes) and will be returned on the map that will map // the filepath to its contents. // // Return the contents generated for the file (and that has been written on it). func CreateFile(t *testing.T, f string) string { t.Helper() dir := filepath.Dir(f) MkdirAll(t, dir) contents := fmt.Sprintf("randomContents=%d", rand.Int()) err := ioutil.WriteFile(f, []byte(contents), 0644) if err != nil { t.Fatalf("error[%s] writing file[%s]", err, f) } return contents } func WorkingDir(t *testing.T) string { t.Helper() wd, err := os.Getwd() if err != nil { t.Fatal(err) } return wd } func ChangeDir(t *testing.T, path string) { t.Helper() err := os.Chdir(path) if err != nil { t.Fatal(err) } } func Chmod(t *testing.T, path string, mode os.FileMode) { t.Helper() err := os.Chmod(path, mode) if err != nil { t.Fatal(err) } } ================================================ FILE: nash.go ================================================ // Package nash provides a library to embed the `nash` scripting language // within your program or create your own nash cli. package nash import ( "bytes" "fmt" "io" "github.com/madlambda/nash/ast" shell "github.com/madlambda/nash/internal/sh" "github.com/madlambda/nash/sh" ) type ( // Shell is the execution engine of the scripting language. Shell struct { interp *shell.Shell } ) func newShell(nashpath string, nashroot string, abort bool) (*Shell, error) { var ( nash Shell err error ) if abort { nash.interp, err = shell.NewAbortShell(nashpath, nashroot) } else { nash.interp, err = shell.NewShell(nashpath, nashroot) } if err != nil { return nil, err } return &nash, nil } // New creates a new `nash.Shell` instance. func New(nashpath string, nashroot string) (*Shell, error) { return newShell(nashpath, nashroot, false) } // NewAbort creates a new shell that aborts in case of error on initialization. // Useful for tests, to avoid trashing the output log. func NewAbort(nashpath string, nashroot string) (*Shell, error) { return newShell(nashpath, nashroot, true) } // SetDebug enable some logging for debug purposes. func (nash *Shell) SetDebug(b bool) { nash.interp.SetDebug(b) } // SetInteractive enables interactive (shell) mode. func (nash *Shell) SetInteractive(b bool) { nash.interp.SetInteractive(b) } func (nash *Shell) NashPath() string { return nash.interp.NashPath() } // Environ returns the set of environment variables in the shell func (nash *Shell) Environ() shell.Env { return nash.interp.Environ() } // GetFn gets the function object. func (nash *Shell) GetFn(name string) (sh.FnDef, error) { fnObj, err := nash.interp.GetFn(name) if err != nil { return nil, err } return fnObj.Fn(), nil } // Prompt returns the environment prompt or the default one func (nash *Shell) Prompt() string { value, ok := nash.interp.Getenv("PROMPT") if ok { return value.String() } return " " } // SetNashdPath sets an alternativa path to nashd func (nash *Shell) SetNashdPath(path string) { nash.interp.SetNashdPath(path) } // Exec executes the code specified by string content. // By default, nash uses os.Stdin, os.Stdout and os.Stderr as input, output // and error file descriptors. You can change it with SetStdin, SetStdout and Stderr, // respectively. // The path is only used for error line reporting. If content represents a file, then // setting path to this filename should improve debugging (or no). func (nash *Shell) Exec(path, content string) error { return nash.interp.Exec(path, content) } // ExecOutput executes the code specified by string content. // // It behaves like **Exec** with the exception that it will ignore any // stdout parameter (and the default os.Stdout) and will return the // whole stdout output in memory. // // This method has no side effects, it will preserve any previously // setted stdout, it will only ignore the configured stdout to run // the provided script content; func (nash *Shell) ExecOutput(path, content string) ([]byte, error) { oldstdout := nash.Stdout() defer nash.SetStdout(oldstdout) var output bytes.Buffer nash.SetStdout(&output) err := nash.interp.Exec(path, content) return output.Bytes(), err } // ExecuteString executes the script content. // Deprecated: Use Exec instead. func (nash *Shell) ExecuteString(path, content string) error { return nash.interp.Exec(path, content) } // ExecFile executes the script content of the file specified by path // and passes as arguments to the script the given args slice. func (nash *Shell) ExecFile(path string, args ...string) error { if len(args) > 0 { err := nash.ExecuteString("setting args", `var ARGS = `+args2Nash(args)) if err != nil { return fmt.Errorf("Failed to set nash arguments: %s", err.Error()) } } return nash.interp.ExecFile(path) } // ExecuteFile executes the given file. // Deprecated: Use ExecFile instead. func (nash *Shell) ExecuteFile(path string) error { return nash.interp.ExecFile(path) } // ExecuteTree executes the given tree. // Deprecated: Use ExecTree instead. func (nash *Shell) ExecuteTree(tr *ast.Tree) ([]sh.Obj, error) { return nash.interp.ExecuteTree(tr) } // ExecTree evaluates the given abstract syntax tree. // it returns the object result of eval or nil when not applied and error. func (nash *Shell) ExecTree(tree *ast.Tree) ([]sh.Obj, error) { return nash.interp.ExecuteTree(tree) } // SetStdout set the stdout of the nash engine. func (nash *Shell) SetStdout(out io.Writer) { nash.interp.SetStdout(out) } // SetStderr set the stderr of nash engine func (nash *Shell) SetStderr(err io.Writer) { nash.interp.SetStderr(err) } // SetStdin set the stdin of the nash engine func (nash *Shell) SetStdin(in io.Reader) { nash.interp.SetStdin(in) } // Stdin is the interpreter standard input func (nash *Shell) Stdin() io.Reader { return nash.interp.Stdin() } // Stdout is the interpreter standard output func (nash *Shell) Stdout() io.Writer { return nash.interp.Stdout() } // Stderr is the interpreter standard error func (nash *Shell) Stderr() io.Writer { return nash.interp.Stderr() } // Setvar sets or updates the variable in the nash session. It // returns true if variable was found and properly updated. func (nash *Shell) Setvar(name string, value sh.Obj) bool { return nash.interp.Setvar(name, value) } // Newvar creates a new variable in the interpreter scope func (nash *Shell) Newvar(name string, value sh.Obj) { nash.interp.Newvar(name, value) } // Getvar retrieves a variable from nash session func (nash *Shell) Getvar(name string) (sh.Obj, bool) { return nash.interp.Getvar(name) } func args2Nash(args []string) string { ret := "(" for i := 0; i < len(args); i++ { ret += `"` + args[i] + `"` if i < (len(args) - 1) { ret += " " } } return ret + ")" } ================================================ FILE: nash_test.go ================================================ package nash import ( "bytes" "io/ioutil" "os" "testing" "github.com/madlambda/nash/sh" "github.com/madlambda/nash/tests" ) // only testing the public API // bypass to internal sh.Shell func TestExecuteFile(t *testing.T) { testfile := tests.Testdir + "/ex1.sh" var out bytes.Buffer shell, cleanup := newTestShell(t) defer cleanup() shell.SetNashdPath(tests.Nashcmd) shell.SetStdout(&out) shell.SetStderr(os.Stderr) shell.SetStdin(os.Stdin) err := shell.ExecuteFile(testfile) if err != nil { t.Error(err) return } if string(out.Bytes()) != "hello world\n" { t.Errorf("Wrong command output: '%s'", string(out.Bytes())) return } } func TestExecuteString(t *testing.T) { shell, cleanup := newTestShell(t) defer cleanup() var out bytes.Buffer shell.SetStdout(&out) err := shell.ExecuteString("-ínput-", "echo -n AAA") if err != nil { t.Error(err) return } if string(out.Bytes()) != "AAA" { t.Errorf("Unexpected '%s'", string(out.Bytes())) return } out.Reset() err = shell.ExecuteString("-input-", ` PROMPT="humpback> " setenv PROMPT `) if err != nil { t.Error(err) return } prompt := shell.Prompt() if prompt != "humpback> " { t.Errorf("Invalid prompt = %s", prompt) return } } func TestSetvar(t *testing.T) { shell, cleanup := newTestShell(t) defer cleanup() shell.Newvar("__TEST__", sh.NewStrObj("something")) var out bytes.Buffer shell.SetStdout(&out) err := shell.Exec("TestSetvar", `echo -n $__TEST__`) if err != nil { t.Error(err) return } if string(out.Bytes()) != "something" { t.Errorf("Value differ: '%s' != '%s'", string(out.Bytes()), "something") return } val, ok := shell.Getvar("__TEST__") if !ok || val.String() != "something" { t.Errorf("Getvar doesn't work: '%s' != '%s'", val, "something") return } } func newTestShell(t *testing.T) (*Shell, func()) { t.Helper() nashpath, pathclean := tmpdir(t) nashroot, rootclean := tmpdir(t) s, err := NewAbort(nashpath, nashroot) if err != nil { t.Fatal(err) } return s, func() { pathclean() rootclean() } } func tmpdir(t *testing.T) (string, func()) { t.Helper() dir, err := ioutil.TempDir("", "nash-tests") if err != nil { t.Fatal(err) } return dir, func() { err := os.RemoveAll(dir) if err != nil { t.Fatal(err) } } } ================================================ FILE: parser/parse.go ================================================ package parser import ( "fmt" "runtime" "strconv" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/errors" "github.com/madlambda/nash/scanner" "github.com/madlambda/nash/token" ) type ( // Parser parses an nash file Parser struct { name string // filename or name of the buffer content string l *scanner.Lexer tok *scanner.Token // token saved for lookahead openblocks int insidePipe bool keywordParsers map[token.Token]parserFn } parserFn func(tok scanner.Token) (ast.Node, error) exprConfig struct { allowArg bool allowVariadic bool allowFuncall bool allowConcat bool } ) // NewParser creates a new parser func NewParser(name, content string) *Parser { p := &Parser{ name: name, content: content, l: scanner.Lex(name, content), } p.keywordParsers = map[token.Token]parserFn{ token.For: p.parseFor, token.If: p.parseIf, token.Fn: p.parseFnDecl, token.Var: p.parseVar, token.Return: p.parseReturn, token.Import: p.parseImport, token.SetEnv: p.parseSetenv, token.Rfork: p.parseRfork, token.BindFn: p.parseBindFn, token.Comment: p.parseComment, token.Illegal: p.parseError, } return p } // Parse starts the parsing. func (p *Parser) Parse() (tr *ast.Tree, err error) { var root *ast.BlockNode defer func() { if r := recover(); r != nil { if _, ok := r.(runtime.Error); ok { panic(r) } err = r.(error) } }() root, err = p.parseBlock(1, 0) if err != nil { return nil, err } tr = ast.NewTree(p.name) tr.Root = root return tr, nil } // next returns the next item from lookahead buffer if not empty or // from the Lexer func (p *Parser) next() scanner.Token { if p.tok != nil { t := p.tok p.tok = nil return *t } tok := <-p.l.Tokens if tok.Type() == token.Illegal { panic(errors.NewError(tok.Value())) } return tok } // backup puts the item into the lookahead buffer func (p *Parser) backup(it scanner.Token) error { if p.tok != nil { panic(errors.NewError("only one slot for backup/lookahead: %s", it)) } p.tok = &it return nil } // ignores the next item func (p *Parser) ignore() { if p.tok != nil { p.tok = nil } else { <-p.l.Tokens } } // peek gets but do not discards the next item (lookahead) func (p *Parser) peek() scanner.Token { i := p.next() p.tok = &i return i } func (p *Parser) parseBlock(lineStart, columnStart int) (*ast.BlockNode, error) { ln := ast.NewBlockNode(token.NewFileInfo(lineStart, columnStart)) for { it := p.peek() switch it.Type() { case token.EOF: goto finish case token.LBrace: p.ignore() return nil, newParserError(it, p.name, "Unexpected '{'") case token.RBrace: p.ignore() if p.openblocks <= 0 { return nil, newParserError(it, p.name, "No block open for close") } p.openblocks-- return ln, nil default: n, err := p.parseStatement() if err != nil { return nil, err } ln.Push(n) } } finish: if p.openblocks != 0 { return nil, errors.NewUnfinishedBlockError(p.name, p.peek()) } return ln, nil } func (p *Parser) parseStatement() (ast.Node, error) { it := p.next() next := p.peek() if fn, ok := p.keywordParsers[it.Type()]; ok { return fn(it) } // statement starting with ident: // - fn call // - variable assignment // - variable exec assignment // - Command if isFuncall(it.Type(), next.Type()) { return p.parseFnInv(it, true) } if it.Type() == token.Ident { if isAssignment(next.Type()) { return p.parseAssignment(it) } return p.parseCommand(it) } else if it.Type() == token.Arg { return p.parseCommand(it) } // statement starting with '(' // -multiline command (echo hello) if it.Type() == token.LParen { return p.parseCommand(it) } return nil, newParserError(it, p.name, "Unexpected token parsing statement '%+v'", it) } func (p *Parser) parseIndexing() (ast.Expr, error) { it := p.next() if it.Type() != token.Number && it.Type() != token.Variable { return nil, newParserError(it, p.name, "Expected number or variable in index. Found %v", it) } var ( index ast.Expr err error ) if it.Type() == token.Number { // only supports base10 intval, err := strconv.Atoi(it.Value()) if err != nil { return nil, err } index = ast.NewIntExpr(it.FileInfo, intval) } else { index, err = p.parseVariable(&it, false) if err != nil { return nil, err } } it = p.next() if it.Type() != token.RBrack { return nil, newParserError(it, p.name, "Unexpected token %v. Expecting ']'", it) } return index, nil } func (p *Parser) parseVariable(tok *scanner.Token, allowVararg bool) (ast.Expr, error) { var it scanner.Token if tok == nil { it = p.next() } else { it = *tok } if it.Type() != token.Variable { return nil, newParserError(it, p.name, "Unexpected token %v. Expected VARIABLE", it) } variadicErr := func(tok scanner.Token) (ast.Node, error) { return nil, newParserError(it, p.name, "Unexpected token '...'. Varargs allowed only in fn call and fn decl") } varTok := it it = p.peek() if it.Type() == token.LBrack { variable := ast.NewVarExpr(varTok.FileInfo, varTok.Value()) p.ignore() index, err := p.parseIndexing() if err != nil { return nil, err } isVariadic := p.peek().Type() == token.Dotdotdot if isVariadic && !allowVararg { return variadicErr(p.peek()) } indexedVar := ast.NewIndexVariadicExpr(variable.FileInfo, variable, index, isVariadic) if isVariadic { p.ignore() } return indexedVar, nil } isVariadic := p.peek().Type() == token.Dotdotdot if isVariadic { if !allowVararg { return variadicErr(p.peek()) } p.ignore() } return ast.NewVarVariadicExpr(varTok.FileInfo, varTok.Value(), isVariadic), nil } func (p *Parser) parsePipe(first *ast.CommandNode) (ast.Node, error) { it := p.next() n := ast.NewPipeNode(it.FileInfo, first.IsMulti()) first.SetMulti(false) n.AddCmd(first) for it = p.peek(); it.Type() == token.Ident || it.Type() == token.Arg; it = p.peek() { p.next() cmd, err := p.parseCommand(it) if err != nil { return nil, err } n.AddCmd(cmd.(*ast.CommandNode)) if !p.insidePipe { break } } if n.IsMulti() { it = p.peek() if it.Type() != token.RParen { if it.Type() == token.EOF { return nil, errors.NewUnfinishedCmdError(p.name, it) } return nil, newParserError(it, p.name, "Unexpected symbol '%s'", it) } p.ignore() } it = p.peek() if it.Type() == token.RBrace { return n, nil } if it.Type() != token.Semicolon { return nil, newParserError(it, p.name, "Unexpected symbol %s", it) } p.ignore() return n, nil } func (p *Parser) parseCommand(it scanner.Token) (ast.Node, error) { isMulti := false if it.Type() == token.LParen { // multiline command isMulti = true it = p.next() } if it.Type() != token.Ident && it.Type() != token.Arg { if isMulti && it.Type() == token.EOF { return nil, errors.NewUnfinishedCmdError(p.name, it) } return nil, newParserError(it, p.name, "Unexpected token %v. Expecting IDENT or ARG", it) } n := ast.NewCommandNode(it.FileInfo, it.Value(), isMulti) cmdLoop: for { it = p.peek() switch typ := it.Type(); { case typ == token.RBrace: if p.openblocks > 0 { if p.insidePipe { p.insidePipe = false } return n, nil } break cmdLoop case isValidArgument(it): arg, err := p.getArgument(nil, exprConfig{ allowConcat: true, allowArg: true, allowVariadic: true, allowFuncall: false, }) if err != nil { return nil, err } n.AddArg(arg) case typ == token.Plus: return nil, newParserError(it, p.name, "Unexpected '+'") case typ == token.Gt: p.next() redir, err := p.parseRedirection(it) if err != nil { return nil, err } n.AddRedirect(redir) case typ == token.Pipe: if p.insidePipe { p.next() // TODO(i4k): test against pipes and multiline cmds return n, nil } p.insidePipe = true return p.parsePipe(n) case typ == token.EOF: break cmdLoop case typ == token.Illegal: return nil, errors.NewError(it.Value()) default: break cmdLoop } } it = p.peek() if isMulti { if it.Type() != token.RParen { if it.Type() == token.EOF { return nil, errors.NewUnfinishedCmdError(p.name, it) } return nil, newParserError(it, p.name, "Unexpected symbol '%s'", it) } p.ignore() it = p.peek() } if p.insidePipe { p.insidePipe = false return n, nil } if it.Type() != token.Semicolon { return nil, newParserError(it, p.name, "Unexpected symbol '%s'", it) } p.ignore() return n, nil } func (p *Parser) parseRedirection(it scanner.Token) (*ast.RedirectNode, error) { var ( lval, rval int = ast.RedirMapNoValue, ast.RedirMapNoValue err error ) redir := ast.NewRedirectNode(it.FileInfo) it = p.peek() if !isValidArgument(it) && it.Type() != token.LBrack { return nil, newParserError(it, p.name, "Unexpected token: %v", it) } // [ if it.Type() == token.LBrack { p.next() it = p.peek() if it.Type() != token.Number { return nil, newParserError(it, p.name, "Expected lefthand side of redirection map, but found '%s'", it.Value()) } lval, err = strconv.Atoi(it.Value()) if err != nil { return nil, newParserError(it, p.name, "Redirection map expects integers. Found: %s", it.Value()) } p.next() it = p.peek() if it.Type() != token.Assign && it.Type() != token.RBrack { return nil, newParserError(it, p.name, "Unexpected token %v. Expecting ASSIGN or ]", it) } // [xxx= if it.Type() == token.Assign { p.next() it = p.peek() if it.Type() != token.Number && it.Type() != token.RBrack { return nil, newParserError(it, p.name, "Unexpected token %v. Expecting REDIRMAPRSIDE or ]", it) } if it.Type() == token.Number { rval, err = strconv.Atoi(it.Value()) if err != nil { return nil, newParserError(it, p.name, "Redirection map expects integers. Found: %s", it.Value()) } p.next() it = p.peek() } else { rval = ast.RedirMapSupress } } if it.Type() != token.RBrack { return nil, newParserError(it, p.name, "Unexpected token %v. Expecting ]", it) } // [xxx=yyy] redir.SetMap(lval, rval) p.next() it = p.peek() } if !isValidArgument(it) { if rval != ast.RedirMapNoValue || lval != ast.RedirMapNoValue { return redir, nil } return nil, newParserError(it, p.name, "Unexpected token %v. Expecting STRING or ARG or VARIABLE", it) } arg, err := p.getArgument(nil, exprConfig{ allowConcat: true, allowArg: true, allowVariadic: false, allowFuncall: false, }) if err != nil { return nil, err } redir.SetLocation(arg) return redir, nil } func (p *Parser) parseImport(importToken scanner.Token) (ast.Node, error) { it := p.next() if it.Type() != token.Arg && it.Type() != token.String && it.Type() != token.Ident { return nil, newParserError(it, p.name, "Unexpected token %v. Expecting ARG or STRING", it) } var arg *ast.StringExpr if it.Type() == token.String { arg = ast.NewStringExpr(it.FileInfo, it.Value(), true) } else if it.Type() == token.Arg || it.Type() == token.Ident { arg = ast.NewStringExpr(it.FileInfo, it.Value(), false) } else { return nil, newParserError(it, p.name, "Parser error: Invalid token '%v' for import path", it) } if p.peek().Type() == token.Semicolon { p.ignore() } return ast.NewImportNode(importToken.FileInfo, arg), nil } func (p *Parser) parseSetenv(it scanner.Token) (ast.Node, error) { var ( setenv *ast.SetenvNode assign ast.Node err error fileInfo = it.FileInfo ) it = p.next() next := p.peek() if it.Type() != token.Ident { return nil, newParserError(it, p.name, "Unexpected token %v, expected identifier", it) } if next.Type() == token.Assign || next.Type() == token.AssignCmd { assign, err = p.parseAssignment(it) if err != nil { return nil, err } setenv, err = ast.NewSetenvNode(fileInfo, it.Value(), assign) } else { setenv, err = ast.NewSetenvNode(fileInfo, it.Value(), nil) if p.peek().Type() != token.Semicolon { return nil, newParserError(p.peek(), p.name, "Unexpected token %v, expected semicolon (;) or EOL", p.peek()) } p.ignore() } if err != nil { return nil, err } return setenv, nil } func (p *Parser) getArgument(tok *scanner.Token, cfg exprConfig) (ast.Expr, error) { var ( err error it scanner.Token isFuncall bool ) if tok != nil { it = *tok } else { it = p.next() } if !isValidArgument(it) { return nil, newParserError(it, p.name, "Unexpected token %v. Expected %s, %s, %s or %s", it, token.Ident, token.String, token.Variable, token.Arg) } firstToken := it var arg ast.Expr if firstToken.Type() == token.Variable { next := p.peek() if cfg.allowFuncall && next.Type() == token.LParen { arg, err = p.parseFnInv(firstToken, false) isFuncall = true } else { // makes "echo $list" == "echo $list..." arg, err = p.parseVariable(&firstToken, cfg.allowVariadic) } } else if firstToken.Type() == token.String { arg = ast.NewStringExpr(firstToken.FileInfo, firstToken.Value(), true) } else { // Arg, Ident, Number, Dotdotdot, etc next := p.peek() if cfg.allowFuncall && next.Type() == token.LParen { arg, err = p.parseFnInv(firstToken, false) isFuncall = true } else { arg = ast.NewStringExpr(firstToken.FileInfo, firstToken.Value(), false) } } if err != nil { return nil, err } it = p.peek() if it.Type() == token.Plus && cfg.allowConcat { return p.getConcatArg(arg) } if (firstToken.Type() == token.Arg || firstToken.Type() == token.Ident) && (!cfg.allowArg && !isFuncall) { return nil, newParserError(it, p.name, "Unquoted string not allowed at pos %d (%s)", it.FileInfo, it.Value()) } return arg, nil } func (p *Parser) getConcatArg(firstArg ast.Expr) (ast.Expr, error) { var ( it scanner.Token parts []ast.Expr ) parts = append(parts, firstArg) hasConcat: it = p.peek() if it.Type() == token.Plus { p.ignore() arg, err := p.getArgument(nil, exprConfig{ allowArg: false, allowConcat: false, allowFuncall: true, allowVariadic: false, }) if err != nil { return nil, err } parts = append(parts, arg) goto hasConcat } return ast.NewConcatExpr(token.NewFileInfo(firstArg.Line(), firstArg.Column()), parts), nil } func (p *Parser) parseAssignment(ident scanner.Token) (ast.Node, error) { // we're here // | // V // ident = ... // ident <= ... // ident, ident2, ..., identN = ... // ident, ident2, ..., identN <= ... it := p.next() if !isAssignment(it.Type()) { return nil, newParserError(it, p.name, "Unexpected token %v, expected '=' ,'<=', ',' or '['", it) } var ( index ast.Expr err error ) if it.Type() == token.LBrack { index, err = p.parseIndexing() if err != nil { return nil, err } it = p.next() } names := []*ast.NameNode{ ast.NewNameNode(ident.FileInfo, ident.Value(), index), } if it.Type() != token.Comma { goto assignOp } for it = p.next(); it.Type() == token.Ident; it = p.next() { var index ast.Expr name := it it = p.next() if it.Type() == token.LBrack { index, err = p.parseIndexing() if err != nil { return nil, err } it = p.next() } names = append(names, ast.NewNameNode(name.FileInfo, name.Value(), index)) if it.Type() != token.Comma { break } } assignOp: if it.Type() != token.AssignCmd && it.Type() != token.Assign { return nil, newParserError(it, p.name, "Unexpected token %v, expected ',' '=' or '<='", it) } if it.Type() == token.AssignCmd { return p.parseAssignCmdOut(names) } return p.parseAssignValues(names) } func (p *Parser) parseList(tok *scanner.Token) (ast.Node, error) { var ( arg ast.Expr err error lit scanner.Token ) if tok != nil { lit = *tok } else { lit = p.next() } if lit.Type() != token.LParen { return nil, newParserError(lit, p.name, "Unexpected token %v. Expecting (", lit) } var values []ast.Expr it := p.peek() for isValidArgument(it) || it.Type() == token.LParen { if it.Type() == token.LParen { arg, err = p.parseList(nil) } else { arg, err = p.getArgument(nil, exprConfig{ allowArg: true, allowConcat: true, allowFuncall: false, allowVariadic: false, }) } if err != nil { return nil, err } it = p.peek() values = append(values, arg) } if it.Type() != token.RParen { if it.Type() == token.EOF { return nil, errors.NewUnfinishedListError(p.name, it) } return nil, newParserError(it, p.name, "Expected ) but found %s", it) } p.ignore() var isVariadic bool if p.peek().Type() == token.Dotdotdot { isVariadic = true p.ignore() } return ast.NewListVariadicExpr(lit.FileInfo, values, isVariadic), nil } func (p *Parser) parseAssignValues(names []*ast.NameNode) (ast.Node, error) { var values []ast.Expr if len(names) == 0 { return nil, newParserError(p.peek(), p.name, "parser error: expect names non nil") } for it := p.peek(); isExpr(it.Type()); it = p.peek() { var ( value ast.Expr err error ) if it.Type() == token.Variable || it.Type() == token.String { value, err = p.getArgument(nil, exprConfig{ allowArg: false, allowFuncall: true, allowVariadic: false, allowConcat: true, }) } else if it.Type() == token.LParen { // list value, err = p.parseList(nil) } else { return nil, newParserError(it, p.name, "Unexpected token %v. Expecting VARIABLE or STRING or (", it) } if err != nil { return nil, err } values = append(values, value) if p.peek().Type() != token.Comma { break } p.ignore() } if len(values) == 0 { return nil, newParserError(p.peek(), p.name, "Unexpected token %v. Expecting VARIABLE, STRING or (", p.peek()) } else if len(values) != len(names) { return nil, newParserError(p.peek(), p.name, "assignment count mismatch: %d = %d", len(names), len(values)) } if p.peek().Type() == token.Semicolon { p.ignore() } return ast.NewAssignNode(names[0].FileInfo, names, values), nil } func (p *Parser) parseAssignCmdOut(identifiers []*ast.NameNode) (ast.Node, error) { var ( exec ast.Node err error ) it := p.next() if it.Type() != token.Ident && it.Type() != token.Arg && it.Type() != token.Variable && it.Type() != token.LParen { return nil, newParserError(it, p.name, "Invalid token %v. Expected command or function invocation", it) } if it.Type() == token.LParen { // command invocation exec, err = p.parseCommand(it) } else { nextIt := p.peek() if nextIt.Type() != token.LParen { // it == (Ident || Arg) exec, err = p.parseCommand(it) } else { // () // () // () exec, err = p.parseFnInv(it, true) } } if err != nil { return nil, err } if len(identifiers) == 0 { // should not happen... pray panic("internal error parsing assignment") } return ast.NewExecAssignNode(identifiers[0].FileInfo, identifiers, exec) } func (p *Parser) parseRfork(it scanner.Token) (ast.Node, error) { n := ast.NewRforkNode(it.FileInfo) it = p.next() if it.Type() != token.Ident { return nil, newParserError(it, p.name, "rfork requires one or more of the following flags: %s", ast.RforkFlags) } arg := ast.NewStringExpr(it.FileInfo, it.Value(), false) n.SetFlags(arg) it = p.peek() if it.Type() == token.LBrace { blockPos := it.FileInfo p.ignore() // ignore lookaheaded symbol p.openblocks++ tree := ast.NewTree("rfork block") r, err := p.parseBlock(blockPos.Line(), blockPos.Column()) if err != nil { return nil, err } tree.Root = r n.SetTree(tree) } if p.peek().Type() == token.Semicolon { p.ignore() } return n, nil } func (p *Parser) parseIfExpr() (ast.Node, error) { it := p.peek() if it.Type() != token.Ident && it.Type() != token.String && it.Type() != token.Variable { return nil, newParserError(it, p.name, "if requires lhs/rhs of type string, variable or function invocation. Found %v", it) } return p.getArgument(nil, exprConfig{ allowArg: false, allowVariadic: false, allowFuncall: true, allowConcat: true, }) } func (p *Parser) parseIf(it scanner.Token) (ast.Node, error) { n := ast.NewIfNode(it.FileInfo) lvalue, err := p.parseIfExpr() if err != nil { return nil, err } n.SetLvalue(lvalue) it = p.next() if it.Type() != token.Equal && it.Type() != token.NotEqual { return nil, newParserError(it, p.name, "Expected comparison, but found %v", it) } if it.Value() != "==" && it.Value() != "!=" { return nil, newParserError(it, p.name, "Invalid if operator '%s'. Valid comparison operators are '==' and '!='", it.Value()) } n.SetOp(it.Value()) rvalue, err := p.parseIfExpr() if err != nil { return nil, err } n.SetRvalue(rvalue) it = p.next() if it.Type() != token.LBrace { return nil, newParserError(it, p.name, "Expected '{' but found %v", it) } p.openblocks++ r, err := p.parseBlock(it.Line(), it.Column()) if err != nil { return nil, err } ifTree := ast.NewTree("if block") ifTree.Root = r n.SetIfTree(ifTree) it = p.peek() if it.Type() == token.Else { p.next() elseBlock, elseIf, err := p.parseElse() if err != nil { return nil, err } elseTree := ast.NewTree("else tree") elseTree.Root = elseBlock n.SetElseif(elseIf) n.SetElseTree(elseTree) } return n, nil } func (p *Parser) parseFnArgs() ([]*ast.FnArgNode, error) { var args []*ast.FnArgNode if p.peek().Type() == token.RParen { // no argument p.ignore() return args, nil } for { it := p.next() if it.Type() == token.Ident { argName := it.Value() isVariadic := false if p.peek().Type() == token.Dotdotdot { isVariadic = true p.ignore() } args = append(args, ast.NewFnArgNode(it.FileInfo, argName, isVariadic)) } else { return nil, newParserError(it, p.name, "Unexpected token %v. Expected identifier or ')'", it) } it = p.peek() if it.Type() == token.Comma { p.ignore() it = p.peek() if it.Type() == token.RParen { break } continue } if it.Type() != token.RParen { return nil, newParserError(it, p.name, "Unexpected '%v'. Expected ')'", it) } p.ignore() break } return args, nil } func (p *Parser) parseVar(it scanner.Token) (ast.Node, error) { var varTok = it it = p.next() next := p.peek() if it.Type() != token.Ident { return nil, newParserError(it, p.name, "Unexpected token %v. Expected IDENT", next, ) } if !isAssignment(next.Type()) { return nil, newParserError(next, p.name, "Unexpected token %v. Expected '=' or ','", next, ) } assign, err := p.parseAssignment(it) if err != nil { return nil, err } switch assign.Type() { case ast.NodeAssign: return ast.NewVarAssignDecl( varTok.FileInfo, assign.(*ast.AssignNode), ), nil case ast.NodeExecAssign: return ast.NewVarExecAssignDecl( varTok.FileInfo, assign.(*ast.ExecAssignNode), ), nil } return nil, newParserError(next, p.name, "Unexpected token %v. Expected ASSIGN or EXECASSIGN", next, ) } func (p *Parser) parseFnDecl(it scanner.Token) (ast.Node, error) { var n *ast.FnDeclNode it = p.next() if it.Type() == token.Ident { n = ast.NewFnDeclNode(it.FileInfo, it.Value()) it = p.next() } else { n = ast.NewFnDeclNode(it.FileInfo, "") } if it.Type() != token.LParen { return nil, newParserError(it, p.name, "Unexpected token %v. Expected '('", it) } args, err := p.parseFnArgs() if err != nil { return nil, err } for _, arg := range args { n.AddArg(arg) } it = p.next() if it.Type() != token.LBrace { return nil, newParserError(it, p.name, "Unexpected token %v. Expected '{'", it) } p.openblocks++ tree := ast.NewTree(fmt.Sprintf("fn %s body", n.Name())) r, err := p.parseBlock(it.Line(), it.Column()) if err != nil { return nil, err } tree.Root = r n.SetTree(tree) return n, nil } func (p *Parser) parseFnInv(ident scanner.Token, allowSemicolon bool) (ast.Node, error) { n := ast.NewFnInvNode(ident.FileInfo, ident.Value()) it := p.next() if it.Type() != token.LParen { return nil, newParserError(it, p.name, "Invalid token %v. Expected '('", it) } for { it = p.next() next := p.peek() if isFuncall(it.Type(), next.Type()) || isValidArgument(it) { arg, err := p.getArgument(&it, exprConfig{ allowArg: false, allowFuncall: true, allowConcat: true, allowVariadic: true, }) if err != nil { return nil, err } n.AddArg(arg) } else if it.Type() == token.LParen { listArg, err := p.parseList(&it) if err != nil { return nil, err } n.AddArg(listArg) } else if it.Type() == token.RParen { // p.next() break } else if it.Type() == token.EOF { goto parseError } it = p.peek() if it.Type() == token.Comma { p.ignore() continue } if it.Type() == token.RParen { p.next() break } goto parseError } // semicolon is optional here if allowSemicolon && p.peek().Type() == token.Semicolon { p.next() } return n, nil parseError: return nil, newParserError(it, p.name, "Unexpected token %v. Expecting STRING, VARIABLE or )", it) } func (p *Parser) parseElse() (*ast.BlockNode, bool, error) { it := p.next() if it.Type() == token.LBrace { p.openblocks++ elseBlock, err := p.parseBlock(it.Line(), it.Column()) if err != nil { return nil, false, err } return elseBlock, false, nil } if it.Type() == token.If { ifNode, err := p.parseIf(it) if err != nil { return nil, false, err } block := ast.NewBlockNode(it.FileInfo) block.Push(ifNode) return block, true, nil } return nil, false, newParserError(it, p.name, "Unexpected token: %v", it) } func (p *Parser) parseBindFn(bindIt scanner.Token) (ast.Node, error) { nameIt := p.next() if nameIt.Type() != token.Ident { return nil, newParserError(nameIt, p.name, "Expected identifier, but found '%v'", nameIt) } cmdIt := p.next() if cmdIt.Type() != token.Ident { return nil, newParserError(cmdIt, p.name, "Expected identifier, but found '%v'", cmdIt) } if p.peek().Type() == token.Semicolon { p.ignore() } n := ast.NewBindFnNode(bindIt.FileInfo, nameIt.Value(), cmdIt.Value()) return n, nil } func (p *Parser) parseReturn(retTok scanner.Token) (ast.Node, error) { ret := ast.NewReturnNode(retTok.FileInfo) tok := p.peek() // return; // return } // return $v // return "" // return ( ... values ... ) // return () // return "val1", "val2", $val3, test() if tok.Type() != token.Semicolon && tok.Type() != token.RBrace && tok.Type() != token.Variable && tok.Type() != token.String && tok.Type() != token.LParen && tok.Type() != token.Ident { return nil, newParserError(tok, p.name, "Expected ';', STRING, VARIABLE, FUNCALL or LPAREN, but found %v", tok) } var returnExprs []ast.Expr for { tok = p.peek() if tok.Type() == token.Semicolon { p.ignore() break } if tok.Type() == token.RBrace { break } if tok.Type() == token.LParen { listArg, err := p.parseList(nil) if err != nil { return nil, err } returnExprs = append(returnExprs, listArg) } else if tok.Type() == token.Ident { p.next() next := p.peek() if next.Type() != token.LParen { return nil, newParserError(tok, p.name, "Expected FUNCALL, STRING, VARIABLE or LPAREN, but found '%v' %v", tok.Value(), next) } arg, err := p.parseFnInv(tok, true) if err != nil { return nil, err } returnExprs = append(returnExprs, arg) } else { arg, err := p.getArgument(nil, exprConfig{ allowArg: false, allowConcat: true, allowFuncall: true, allowVariadic: false, }) if err != nil { return nil, err } returnExprs = append(returnExprs, arg) } next := p.peek() if next.Type() == token.Comma { p.ignore() continue } if next.Type() == token.Semicolon { p.ignore() } break } ret.Returns = returnExprs return ret, nil } func (p *Parser) parseFor(it scanner.Token) (ast.Node, error) { var ( inExpr ast.Expr err error next scanner.Token ) forStmt := ast.NewForNode(it.FileInfo) it = p.peek() if it.Type() != token.Ident { goto forBlockParse } p.next() forStmt.SetIdentifier(it.Value()) it = p.next() if it.Type() != token.Ident || it.Value() != "in" { return nil, newParserError(it, p.name, "Expected 'in' but found %q", it) } // ignores 'in' keyword // TODO: make 'in' a real keyword it = p.next() next = p.peek() if it.Type() != token.Variable && (it.Type() != token.Ident || (it.Type() == token.Ident && next.Type() != token.LParen)) && it.Type() != token.LParen { return nil, newParserError(it, p.name, "Expected (variable, list or fn invocation) but found %q", it) } if (it.Type() == token.Ident || it.Type() == token.Variable) && next.Type() == token.LParen { inExpr, err = p.parseFnInv(it, false) } else if it.Type() == token.Variable { inExpr, err = p.parseVariable(&it, false) } else if it.Type() == token.LParen { inExpr, err = p.parseList(&it) } if err != nil { return nil, err } forStmt.SetInExpr(inExpr) forBlockParse: it = p.peek() if it.Type() != token.LBrace { return nil, newParserError(it, p.name, "Expected '{' but found %q", it) } blockPos := it.FileInfo p.ignore() // ignore lookaheaded symbol p.openblocks++ tree := ast.NewTree("for block") r, err := p.parseBlock(blockPos.Line(), blockPos.Column()) if err != nil { return nil, err } tree.Root = r forStmt.SetTree(tree) return forStmt, nil } func (p *Parser) parseComment(it scanner.Token) (ast.Node, error) { return ast.NewCommentNode(it.FileInfo, it.Value()), nil } func (p *Parser) parseError(it scanner.Token) (ast.Node, error) { return nil, errors.NewError(it.Value()) } func newParserError(item scanner.Token, name, format string, args ...interface{}) error { if item.Type() == token.Illegal { // scanner error return errors.NewError(item.Value()) } errstr := fmt.Sprintf(format, args...) return errors.NewError("%s:%d:%d: %s", name, item.Line(), item.Column(), errstr) } func isValidArgument(t scanner.Token) bool { if t.Type() == token.String || t.Type() == token.Number || t.Type() == token.Arg || t.Type() == token.Dotdotdot || t.Type() == token.Ident || token.IsKeyword(t.Type()) || t.Type() == token.Variable { return true } return false } func isFuncall(tok, next token.Token) bool { return (tok == token.Ident || tok == token.Variable) && next == token.LParen } func isAssignment(tok token.Token) bool { return tok == token.Assign || tok == token.AssignCmd || tok == token.LBrack || tok == token.Comma } func isExpr(tok token.Token) bool { return tok == token.Variable || tok == token.String || tok == token.LParen } ================================================ FILE: parser/parse_fmt_test.go ================================================ package parser import "testing" type fmtTestTable struct { input, expected string } func testFmt(input string, expected string, t *testing.T) { p := NewParser("fmt test", input) tree, err := p.Parse() if err != nil { t.Error(err) return } fmtval := tree.String() if fmtval != expected { t.Errorf("Fmt differ: '%s' != '%s'", fmtval, expected) return } } func testFmtTable(testTable []fmtTestTable, t *testing.T) { for _, test := range testTable { testFmt(test.input, test.expected, t) } } func TestFmtVariables(t *testing.T) { testTable := []fmtTestTable{ // correct adjust of spaces {`test = "a"`, `test = "a"`}, {`test="a"`, `test = "a"`}, {`test= "a"`, `test = "a"`}, {`test ="a"`, `test = "a"`}, {`test = "a"`, `test = "a"`}, {`test ="a"`, `test = "a"`}, {`test ="a"`, `test = "a"`}, {`test = "a"`, `test = "a"`}, {`test = "a"`, `test = "a"`}, {`test = ()`, `test = ()`}, {`test=()`, `test = ()`}, {`test =()`, `test = ()`}, {`test =()`, `test = ()`}, {`test= ()`, `test = ()`}, {`test = (plan9)`, `test = (plan9)`}, {`test=(plan9)`, `test = (plan9)`}, {`test = (plan9)`, `test = (plan9)`}, {`test = (plan9)`, `test = (plan9)`}, {`test = (plan9)`, `test = (plan9)`}, {`test = ( plan9)`, `test = (plan9)`}, {`test = ( plan9)`, `test = (plan9)`}, {`test = (plan9 )`, `test = (plan9)`}, {`test = (plan9 from bell labs)`, `test = (plan9 from bell labs)`}, {`test = (plan9 from bell labs)`, `test = (plan9 from bell labs)`}, {`test = (plan9 from bell labs)`, `test = (plan9 from bell labs)`}, {`test = (plan9 from bell labs)`, `test = (plan9 from bell labs)`}, {`test = ( plan9 from bell labs )`, `test = (plan9 from bell labs)`}, {`test = (plan9 from bell labs windows linux freebsd netbsd openbsd)`, `test = ( plan9 from bell labs windows linux freebsd netbsd openbsd )`}, {`IFS = ("\n")`, `IFS = ("\n")`}, // multiple variables {`test = "a" testb = "b"`, `test = "a" testb = "b"`}, } testFmtTable(testTable, t) } func TestFmtGroupVariables(t *testing.T) { testTable := []fmtTestTable{ { `test = "a" test2 = "b" fn cd() { echo "hello" }`, `test = "a" test2 = "b" fn cd() { echo "hello" }`, }, { `#!/usr/bin/env nash echo "hello"`, `#!/usr/bin/env nash echo "hello"`, }, } testFmtTable(testTable, t) } func TestFmtFn(t *testing.T) { testTable := []fmtTestTable{ { `fn lala() { echo hello } fn lele() { echo lele }`, `fn lala() { echo hello } fn lele() { echo lele }`, }, { `vv = "" fn t() { echo t }`, `vv = "" fn t() { echo t }`, }, } testFmtTable(testTable, t) } func TestFmtImports(t *testing.T) { testTable := []fmtTestTable{ { `import test import test import test`, `import test import test import test`, }, { `import nashlib/all import klb/aws/all vpcTags = ((Name klb-vpc-example) (Env testing)) `, `import nashlib/all import klb/aws/all vpcTags = ( (Name klb-vpc-example) (Env testing) )`, }, } testFmtTable(testTable, t) } func TestFmtFnComments(t *testing.T) { testTable := []fmtTestTable{ { `PATH = "/bin" # isolated comment # Comment for fn fn test() { echo "hello" } `, `PATH = "/bin" # isolated comment # Comment for fn fn test() { echo "hello" }`, }, } testFmtTable(testTable, t) } func TestFmtSamples(t *testing.T) { testTable := []fmtTestTable{ { `#!/usr/bin/env nash import nashlib/all import klb/aws/all vpcTags = ((Name klb-vpc-example) (Env testing)) igwTags = ((Name klb-igw-example) (Env testing)) routeTblTags = ((Name klb-rtbl-example) (Env testing)) appSubnetTags = ((Name klb-app-subnet-example) (Env testing)) dbSubnetTags = ((Name klb-db-subnet-example) (Env testing)) sgTags = ((Name klb-sg-example) (Env testing)) fn print_resource(name, id) { printf "Created %s: %s%s%s\n" $name $NASH_GREEN $id $NASH_RESET } fn create_prod() { vpcid <= aws_vpc_create("10.0.0.1/16", $vpcTags) appnet <= aws_subnet_create($vpcid, "10.0.1.0/24", $appSubnetTags) dbnet <= aws_subnet_create($vpcid, "10.0.2.0/24", $dbSubnetTags) igwid <= aws_igw_create($igwTags) tblid <= aws_routetbl_create($vpcid, $routeTblTags) aws_igw_attach($igwid, $vpcid) aws_route2igw($tblid, "0.0.0.0/0", $igwid) grpid <= aws_secgroup_create("klb-default-sg", "sg description", $vpcid, $sgTags) print_resource("VPC", $vpcid) print_resource("app subnet", $appnet) print_resource("db subnet", $dbnet) print_resource("Internet Gateway", $igwid) print_resource("Routing table", $tblid) print_resource("Security group", $grpid) } create_prod() `, `#!/usr/bin/env nash import nashlib/all import klb/aws/all vpcTags = ( (Name klb-vpc-example) (Env testing) ) igwTags = ( (Name klb-igw-example) (Env testing) ) routeTblTags = ( (Name klb-rtbl-example) (Env testing) ) appSubnetTags = ( (Name klb-app-subnet-example) (Env testing) ) dbSubnetTags = ( (Name klb-db-subnet-example) (Env testing) ) sgTags = ( (Name klb-sg-example) (Env testing) ) fn print_resource(name, id) { printf "Created %s: %s%s%s\n" $name $NASH_GREEN $id $NASH_RESET } fn create_prod() { vpcid <= aws_vpc_create("10.0.0.1/16", $vpcTags) appnet <= aws_subnet_create($vpcid, "10.0.1.0/24", $appSubnetTags) dbnet <= aws_subnet_create($vpcid, "10.0.2.0/24", $dbSubnetTags) igwid <= aws_igw_create($igwTags) tblid <= aws_routetbl_create($vpcid, $routeTblTags) aws_igw_attach($igwid, $vpcid) aws_route2igw($tblid, "0.0.0.0/0", $igwid) grpid <= aws_secgroup_create("klb-default-sg", "sg description", $vpcid, $sgTags) print_resource("VPC", $vpcid) print_resource("app subnet", $appnet) print_resource("db subnet", $dbnet) print_resource("Internet Gateway", $igwid) print_resource("Routing table", $tblid) print_resource("Security group", $grpid) } create_prod()`, }, } testFmtTable(testTable, t) } func TestFmtPipes(t *testing.T) { testTable := []fmtTestTable{ { `echo hello | grep "he" > test`, `echo hello | grep "he" > test`, }, { `(echo hello | sed "s/he/wo/g" >[1] /tmp/test >[2] /dev/null)`, `( echo hello | sed "s/he/wo/g" >[1] /tmp/test >[2] /dev/null )`, }, { `choice <= ( -find $dir+"/" -maxdepth 1 | sed "s#.*/##" | sort | uniq | -fzf --exact -q "^"+$query -1 -0 --inline-info --header "select file: " )`, `choice <= ( -find $dir+"/" -maxdepth 1 | sed "s#.*/##" | sort | uniq | -fzf --exact -q "^"+$query -1 -0 --inline-info --header "select file: " )`, }, } testFmtTable(testTable, t) } ================================================ FILE: parser/parse_regression_test.go ================================================ package parser import ( "testing" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/token" ) func init() { ast.DebugCmp = true } func TestParseIssue22(t *testing.T) { expected := ast.NewTree("issue 22") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) fn := ast.NewFnDeclNode(token.NewFileInfo(1, 3), "gocd") fn.AddArg(ast.NewFnArgNode(token.NewFileInfo(1, 8), "path", false)) fnTree := ast.NewTree("fn") fnBlock := ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl := ast.NewIfNode(token.NewFileInfo(2, 1)) ifDecl.SetLvalue(ast.NewVarExpr(token.NewFileInfo(2, 4), "$path")) ifDecl.SetOp("==") ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(2, 13), "", true)) ifTree := ast.NewTree("if") ifBlock := ast.NewBlockNode(token.NewFileInfo(2, 1)) cdNode := ast.NewCommandNode(token.NewFileInfo(3, 2), "cd", false) arg := ast.NewVarExpr(token.NewFileInfo(3, 5), "$GOPATH") cdNode.AddArg(arg) ifBlock.Push(cdNode) ifTree.Root = ifBlock ifDecl.SetIfTree(ifTree) elseTree := ast.NewTree("else") elseBlock := ast.NewBlockNode(token.NewFileInfo(4, 9)) args := make([]ast.Expr, 3) args[0] = ast.NewVarExpr(token.NewFileInfo(5, 5), "$GOPATH") args[1] = ast.NewStringExpr(token.NewFileInfo(5, 12), "/src/", true) args[2] = ast.NewVarExpr(token.NewFileInfo(5, 20), "$path") cdNodeElse := ast.NewCommandNode(token.NewFileInfo(5, 2), "cd", false) carg := ast.NewConcatExpr(token.NewFileInfo(5, 5), args) cdNodeElse.AddArg(carg) elseBlock.Push(cdNodeElse) elseTree.Root = elseBlock ifDecl.SetElseTree(elseTree) fnBlock.Push(ifDecl) fnTree.Root = fnBlock fn.SetTree(fnTree) ln.Push(fn) expected.Root = ln parserTest("issue 22", `fn gocd(path) { if $path == "" { cd $GOPATH } else { cd $GOPATH+"/src/"+$path } }`, expected, t, true) } func TestParseIssue38(t *testing.T) { expected := ast.NewTree("parse issue38") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) fnInv := ast.NewFnInvNode(token.NewFileInfo(1, 0), "cd") args := make([]ast.Expr, 3) args[0] = ast.NewVarExpr(token.NewFileInfo(1, 3), "$GOPATH") args[1] = ast.NewStringExpr(token.NewFileInfo(1, 12), "/src/", true) args[2] = ast.NewVarExpr(token.NewFileInfo(1, 19), "$path") arg := ast.NewConcatExpr(token.NewFileInfo(1, 3), args) fnInv.AddArg(arg) ln.Push(fnInv) expected.Root = ln parserTest("parse issue38", `cd($GOPATH+"/src/"+$path)`, expected, t, true) } func TestParseIssue43(t *testing.T) { content := `fn gpull() { branch <= git rev-parse --abbrev-ref HEAD | xargs echo -n git pull origin $branch refreshPrompt() }` expected := ast.NewTree("parse issue 41") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) fnDecl := ast.NewFnDeclNode(token.NewFileInfo(1, 3), "gpull") fnTree := ast.NewTree("fn") fnBlock := ast.NewBlockNode(token.NewFileInfo(1, 0)) gitRevParse := ast.NewCommandNode(token.NewFileInfo(2, 11), "git", false) gitRevParse.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 15), "rev-parse", true)) gitRevParse.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 25), "--abbrev-ref", false)) gitRevParse.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 38), "HEAD", false)) branchAssign, err := ast.NewExecAssignNode(token.NewFileInfo(2, 1), []*ast.NameNode{ ast.NewNameNode(token.NewFileInfo(2, 1), "branch", nil, )}, gitRevParse) if err != nil { t.Error(err) return } xargs := ast.NewCommandNode(token.NewFileInfo(2, 45), "xargs", false) xargs.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 51), "echo", false)) xargs.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 56), "-n", false)) pipe := ast.NewPipeNode(token.NewFileInfo(2, 43), false) pipe.AddCmd(gitRevParse) pipe.AddCmd(xargs) branchAssign.SetCommand(pipe) fnBlock.Push(branchAssign) gitPull := ast.NewCommandNode(token.NewFileInfo(1, 0), "git", false) gitPull.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 0), "pull", false)) gitPull.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 0), "origin", false)) gitPull.AddArg(ast.NewVarExpr(token.NewFileInfo(1, 0), "$branch")) fnBlock.Push(gitPull) fnInv := ast.NewFnInvNode(token.NewFileInfo(1, 0), "refreshPrompt") fnBlock.Push(fnInv) fnTree.Root = fnBlock fnDecl.SetTree(fnTree) ln.Push(fnDecl) expected.Root = ln parserTest("parse issue 41", content, expected, t, true) } func TestParseIssue68(t *testing.T) { expected := ast.NewTree("parse issue #68") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) catCmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "cat", false) catArg := ast.NewStringExpr(token.NewFileInfo(1, 4), "PKGBUILD", false) catCmd.AddArg(catArg) sedCmd := ast.NewCommandNode(token.NewFileInfo(1, 15), "sed", false) sedArg := ast.NewStringExpr(token.NewFileInfo(1, 20), `s#\$pkgdir#/home/i4k/alt#g`, true) sedCmd.AddArg(sedArg) sedRedir := ast.NewRedirectNode(token.NewFileInfo(1, 49)) sedRedirArg := ast.NewStringExpr(token.NewFileInfo(1, 51), "PKGBUILD2", false) sedRedir.SetLocation(sedRedirArg) sedCmd.AddRedirect(sedRedir) pipe := ast.NewPipeNode(token.NewFileInfo(1, 13), false) pipe.AddCmd(catCmd) pipe.AddCmd(sedCmd) ln.Push(pipe) expected.Root = ln parserTest("parse issue #68", `cat PKGBUILD | sed "s#\\$pkgdir#/home/i4k/alt#g" > PKGBUILD2`, expected, t, false) } func TestParseIssue69(t *testing.T) { expected := ast.NewTree("parse-issue-69") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) parts := make([]ast.Expr, 2) parts[0] = ast.NewVarExpr(token.NewFileInfo(1, 5), "$a") parts[1] = ast.NewStringExpr(token.NewFileInfo(1, 9), "b", true) concat := ast.NewConcatExpr(token.NewFileInfo(1, 5), parts) listValues := make([]ast.Expr, 1) listValues[0] = concat list := ast.NewListExpr(token.NewFileInfo(1, 4), listValues) assign := ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "a", nil), list, ) ln.Push(assign) expected.Root = ln parserTest("parse-issue-69", `a = ($a+"b")`, expected, t, true) } func TestParseImportIssue94(t *testing.T) { expected := ast.NewTree("test import") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) importStmt := ast.NewImportNode(token.NewFileInfo(1, 0), ast.NewStringExpr(token.NewFileInfo(1, 7), "common", false)) ln.Push(importStmt) expected.Root = ln parserTest("test import", "import common", expected, t, true) } func TestParseIssue108(t *testing.T) { // keywords cannot be used as command arguments expected := ast.NewTree("parse issue #108") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) catCmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "cat", false) catArg := ast.NewStringExpr(token.NewFileInfo(1, 4), "spec.ebnf", false) catCmd.AddArg(catArg) grepCmd := ast.NewCommandNode(token.NewFileInfo(1, 16), "grep", false) grepArg := ast.NewStringExpr(token.NewFileInfo(1, 21), `-i`, false) grepArg2 := ast.NewStringExpr(token.NewFileInfo(1, 24), "rfork", false) grepCmd.AddArg(grepArg) grepCmd.AddArg(grepArg2) pipe := ast.NewPipeNode(token.NewFileInfo(1, 14), false) pipe.AddCmd(catCmd) pipe.AddCmd(grepCmd) ln.Push(pipe) expected.Root = ln parserTest("parse issue #108", `cat spec.ebnf | grep -i rfork`, expected, t, false) } func TestParseIssue123(t *testing.T) { parser := NewParser("invalid cmd assignment", `IFS <= ("\n")`) _, err := parser.Parse() if err == nil { t.Errorf("Must fail...") return } expected := "invalid cmd assignment:1:9: Unexpected token STRING. Expecting IDENT or ARG" if err.Error() != expected { t.Fatalf("Error string differs. Expecting '%s' but got '%s'", expected, err.Error()) } } ================================================ FILE: parser/parse_test.go ================================================ package parser import ( "strings" "testing" "github.com/madlambda/nash/ast" "github.com/madlambda/nash/token" ) func parserTest(name, content string, expected *ast.Tree, t *testing.T, enableReverse bool) *ast.Tree { parser := NewParser(name, content) tr, err := parser.Parse() if err != nil { t.Error(err) t.Logf("Failed syntax: '%s'", content) return nil } if tr == nil { t.Errorf("Failed to parse") return nil } if !expected.IsEqual(tr) { t.Errorf("Expected: %s\n\nResult: %s\n", expected, tr) t.Logf("Failed syntax: '%s'", content) return tr } if !enableReverse { return tr } // Test if the reverse of tree is the content again... *hard* trcontent := strings.TrimSpace(tr.String()) content = strings.TrimSpace(content) if content != trcontent { t.Errorf(`Failed to reverse the tree. Expected: '%s' But got: '%s' `, content, trcontent) } return tr } func parserTestFail(t *testing.T, execStr string) { parser := NewParser("", execStr) tr, err := parser.Parse() if err == nil { t.Errorf("Parsing '%s' must fail", execStr) return } if tr != nil { t.Error("tr must be nil") return } } func TestParseSimple(t *testing.T) { expected := ast.NewTree("parser simple") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "echo", false) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 6), "hello world", true)) ln.Push(cmd) expected.Root = ln parserTest("parser simple", `echo "hello world"`, expected, t, true) cmd1 := ast.NewCommandNode(token.NewFileInfo(1, 0), "cat", false) arg1 := ast.NewStringExpr(token.NewFileInfo(1, 4), "/etc/resolv.conf", false) arg2 := ast.NewStringExpr(token.NewFileInfo(1, 21), "/etc/hosts", false) cmd1.AddArg(arg1) cmd1.AddArg(arg2) ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ln.Push(cmd1) expected.Root = ln parserTest("parser simple", `cat /etc/resolv.conf /etc/hosts`, expected, t, true) } func TestParseReverseGetSame(t *testing.T) { parser := NewParser("reverse simple", "echo \"hello world\"") tr, err := parser.Parse() if err != nil { t.Error(err) return } if tr.String() != "echo \"hello world\"" { t.Errorf("Failed to reverse tree: %s", tr.String()) return } } func TestParsePipe(t *testing.T) { expected := ast.NewTree("parser pipe") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) first := ast.NewCommandNode(token.NewFileInfo(1, 0), "echo", false) first.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 6), "hello world", true)) second := ast.NewCommandNode(token.NewFileInfo(1, 21), "awk", false) second.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 26), "{print $1}", true)) pipe := ast.NewPipeNode(token.NewFileInfo(1, 19), false) pipe.AddCmd(first) pipe.AddCmd(second) ln.Push(pipe) expected.Root = ln parserTest("parser pipe", `echo "hello world" | awk "{print $1}"`, expected, t, true) } func TestBasicSetEnvAssignment(t *testing.T) { expected := ast.NewTree("simple set assignment") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) set, err := ast.NewSetenvNode(token.NewFileInfo(1, 0), "test", nil) if err != nil { t.Fatal(err) } ln.Push(set) expected.Root = ln parserTest("simple set assignment", `setenv test`, expected, t, true) // setenv with assignment expected = ast.NewTree("setenv with simple assignment") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) assign := ast.NewSingleAssignNode(token.NewFileInfo(1, 7), ast.NewNameNode(token.NewFileInfo(1, 7), "test", nil), ast.NewStringExpr(token.NewFileInfo(1, 15), "hello", true)) set, err = ast.NewSetenvNode(token.NewFileInfo(1, 0), "test", assign) if err != nil { t.Fatal(err) } ln.Push(set) expected.Root = ln parserTest("setenv with simple assignment", `setenv test = "hello"`, expected, t, true) expected = ast.NewTree("setenv with simple cmd assignment") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 15), "ls", false) cmdAssign, err := ast.NewExecAssignNode( token.NewFileInfo(1, 7), []*ast.NameNode{ast.NewNameNode(token.NewFileInfo(1, 7), "test", nil)}, cmd, ) if err != nil { t.Fatal(err) } set, err = ast.NewSetenvNode(token.NewFileInfo(1, 0), "test", cmdAssign) if err != nil { t.Fatal(err) } ln.Push(set) expected.Root = ln parserTest("simple assignment", `setenv test <= ls`, expected, t, true) } func TestBasicAssignment(t *testing.T) { expected := ast.NewTree("simple assignment") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) assign := ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "test", nil), ast.NewStringExpr(token.NewFileInfo(1, 8), "hello", true)) ln.Push(assign) expected.Root = ln parserTest("simple assignment", `test = "hello"`, expected, t, true) // test concatenation of strings and variables ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) concats := make([]ast.Expr, 2, 2) concats[0] = ast.NewStringExpr(token.NewFileInfo(1, 8), "hello", true) concats[1] = ast.NewVarExpr(token.NewFileInfo(1, 15), "$var") arg1 := ast.NewConcatExpr(token.NewFileInfo(1, 8), concats) assign = ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "test", nil), arg1, ) ln.Push(assign) expected.Root = ln parserTest("test", `test = "hello"+$var`, expected, t, true) for _, test := range []string{ "test=hello", "test = hello", "test = 1", "test = false", "test = -1", `test = "1", "2"`, } { parserTestFail(t, test) } } func TestVarAssignment(t *testing.T) { expected := ast.NewTree("var assignment") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) varAssign := ast.NewVarAssignDecl(token.NewFileInfo(1, 0), ast.NewSingleAssignNode(token.NewFileInfo(1, 4), ast.NewNameNode(token.NewFileInfo(1, 4), "test", nil), ast.NewStringExpr(token.NewFileInfo(1, 12), "hello", true)), ) ln.Push(varAssign) expected.Root = ln parserTest("var assignment", `var test = "hello"`, expected, t, true) for _, test := range []string{ "var test=hello", "var", "var test", "var test = false", "var test = -1", `var test = "1", "2"`, } { parserTestFail(t, test) } } func TestParseMultipleAssign(t *testing.T) { one := ast.NewNameNode(token.NewFileInfo(1, 0), "one", nil) two := ast.NewNameNode(token.NewFileInfo(1, 5), "two", nil) value1 := ast.NewStringExpr(token.NewFileInfo(1, 12), "1", true) value2 := ast.NewStringExpr(token.NewFileInfo(1, 17), "2", true) assign := ast.NewAssignNode(token.NewFileInfo(1, 0), []*ast.NameNode{one, two}, []ast.Expr{value1, value2}, ) expected := ast.NewTree("tuple assignment") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ln.Push(assign) expected.Root = ln parserTest("tuple assignment", `one, two = "1", "2"`, expected, t, true) lvalue1 := ast.NewListExpr(token.NewFileInfo(1, 11), []ast.Expr{}) value2 = ast.NewStringExpr(token.NewFileInfo(1, 16), "2", true) assign = ast.NewAssignNode(token.NewFileInfo(1, 0), []*ast.NameNode{one, two}, []ast.Expr{lvalue1, value2}, ) ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ln.Push(assign) expected.Root = ln parserTest("tuple assignment", `one, two = (), "2"`, expected, t, true) } func TestParseMultipleExecAssignment(t *testing.T) { expected := ast.NewTree("multiple cmd assignment") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 16), "ls", false) assign, err := ast.NewExecAssignNode(token.NewFileInfo(1, 0), []*ast.NameNode{ ast.NewNameNode(token.NewFileInfo(1, 0), "test", nil), ast.NewNameNode(token.NewFileInfo(1, 6), "status", nil), }, cmd, ) if err != nil { t.Error(err) return } ln.Push(assign) expected.Root = ln parserTest("multiple cmd assignment", `test, status <= ls`, expected, t, true) expected = ast.NewTree("multiple cmd assignment") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd = ast.NewCommandNode(token.NewFileInfo(1, 19), "ls", false) assign, err = ast.NewExecAssignNode(token.NewFileInfo(1, 0), []*ast.NameNode{ ast.NewNameNode(token.NewFileInfo(1, 0), "test", ast.NewIntExpr(token.NewFileInfo(1, 5), 0)), ast.NewNameNode(token.NewFileInfo(1, 9), "status", nil), }, cmd, ) if err != nil { t.Error(err) return } ln.Push(assign) expected.Root = ln parserTest("multiple cmd assignment", `test[0], status <= ls`, expected, t, true) } func TestParseInvalidIndexing(t *testing.T) { // test indexed assignment parser := NewParser("invalid", `test[a] = "a"`) _, err := parser.Parse() if err == nil { t.Error("Parse must fail") return } else if err.Error() != "invalid:1:5: Expected number or variable in index. Found ARG" { t.Error("Invalid err msg") return } parser = NewParser("invalid", `test[] = "a"`) _, err = parser.Parse() if err == nil { t.Error("Parse must fail") return } else if err.Error() != "invalid:1:5: Expected number or variable in index. Found ]" { t.Error("Invalid err msg") return } parser = NewParser("invalid", `test[10.0] = "a"`) _, err = parser.Parse() if err == nil { t.Error("Parse must fail") return } else if err.Error() != "invalid:1:5: Expected number or variable in index. Found ARG" { t.Error("Invalid err msg") return } } func TestParseListAssignment(t *testing.T) { expected := ast.NewTree("list assignment") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) values := make([]ast.Expr, 0, 4) values = append(values, ast.NewStringExpr(token.NewFileInfo(2, 1), "plan9", false), ast.NewStringExpr(token.NewFileInfo(3, 1), "from", false), ast.NewStringExpr(token.NewFileInfo(4, 1), "bell", false), ast.NewStringExpr(token.NewFileInfo(5, 1), "labs", false), ) elem := ast.NewListExpr(token.NewFileInfo(1, 7), values) assign := ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "test", nil), elem, ) ln.Push(assign) expected.Root = ln parserTest("list assignment", `test = ( plan9 from bell labs )`, expected, t, false) } func TestParseListOfListsAssignment(t *testing.T) { expected := ast.NewTree("list assignment") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) plan9 := make([]ast.Expr, 0, 4) plan9 = append(plan9, ast.NewStringExpr(token.NewFileInfo(2, 2), "plan9", false), ast.NewStringExpr(token.NewFileInfo(2, 8), "from", false), ast.NewStringExpr(token.NewFileInfo(2, 13), "bell", false), ast.NewStringExpr(token.NewFileInfo(2, 18), "labs", false), ) elem1 := ast.NewListExpr(token.NewFileInfo(2, 1), plan9) linux := make([]ast.Expr, 0, 2) linux = append(linux, ast.NewStringExpr(token.NewFileInfo(3, 2), "linux", false)) linux = append(linux, ast.NewStringExpr(token.NewFileInfo(3, 8), "kernel", false)) elem2 := ast.NewListExpr(token.NewFileInfo(3, 1), linux) values := make([]ast.Expr, 2) values[0] = elem1 values[1] = elem2 elem := ast.NewListExpr(token.NewFileInfo(1, 7), values) assign := ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "test", nil), elem, ) ln.Push(assign) expected.Root = ln parserTest("list assignment", `test = ( (plan9 from bell labs) (linux kernel) )`, expected, t, false) } func TestParseCmdAssignment(t *testing.T) { expected := ast.NewTree("simple cmd assignment") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 8), "ls", false) assign, err := ast.NewExecAssignNode(token.NewFileInfo(1, 0), []*ast.NameNode{ast.NewNameNode(token.NewFileInfo(1, 0), "test", nil)}, cmd, ) if err != nil { t.Error(err) return } ln.Push(assign) expected.Root = ln parserTest("simple assignment", `test <= ls`, expected, t, true) } func TestParseInvalidEmpty(t *testing.T) { parser := NewParser("invalid", ";") _, err := parser.Parse() if err == nil { t.Error("Parse must fail") return } } func TestParsePathCommand(t *testing.T) { expected := ast.NewTree("parser simple") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "/bin/echo", false) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 11), "hello world", true)) ln.Push(cmd) expected.Root = ln parserTest("parser simple", `/bin/echo "hello world"`, expected, t, true) } func TestParseWithShebang(t *testing.T) { expected := ast.NewTree("parser shebang") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmt := ast.NewCommentNode(token.NewFileInfo(1, 0), "#!/bin/nash") cmd := ast.NewCommandNode(token.NewFileInfo(3, 0), "echo", false) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(3, 5), "bleh", false)) ln.Push(cmt) ln.Push(cmd) expected.Root = ln parserTest("parser shebang", `#!/bin/nash echo bleh `, expected, t, true) } func TestParseEmptyFile(t *testing.T) { expected := ast.NewTree("empty file") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) expected.Root = ln parserTest("empty file", "", expected, t, true) } func TestParseSingleCommand(t *testing.T) { expected := ast.NewTree("single command") expected.Root = ast.NewBlockNode(token.NewFileInfo(1, 0)) expected.Root.Push(ast.NewCommandNode(token.NewFileInfo(1, 0), "bleh", false)) parserTest("single command", `bleh`, expected, t, true) } func TestParseRedirectSimple(t *testing.T) { expected := ast.NewTree("redirect") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "cmd", false) redir := ast.NewRedirectNode(token.NewFileInfo(1, 4)) redir.SetMap(2, ast.RedirMapSupress) cmd.AddRedirect(redir) ln.Push(cmd) expected.Root = ln parserTest("simple redirect", `cmd >[2=]`, expected, t, true) expected = ast.NewTree("redirect2") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd = ast.NewCommandNode(token.NewFileInfo(1, 0), "cmd", false) redir = ast.NewRedirectNode(token.NewFileInfo(1, 4)) redir.SetMap(2, 1) cmd.AddRedirect(redir) ln.Push(cmd) expected.Root = ln parserTest("simple redirect", `cmd >[2=1]`, expected, t, true) } func TestParseRedirectWithLocation(t *testing.T) { expected := ast.NewTree("redirect with location") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "cmd", false) redir := ast.NewRedirectNode(token.NewFileInfo(1, 4)) redir.SetMap(2, ast.RedirMapNoValue) redir.SetLocation(ast.NewStringExpr(token.NewFileInfo(1, 9), "/var/log/service.log", false)) cmd.AddRedirect(redir) ln.Push(cmd) expected.Root = ln parserTest("simple redirect", `cmd >[2] /var/log/service.log`, expected, t, true) } func TestParseRedirectMultiples(t *testing.T) { expected := ast.NewTree("redirect multiples") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "cmd", false) redir1 := ast.NewRedirectNode(token.NewFileInfo(1, 4)) redir2 := ast.NewRedirectNode(token.NewFileInfo(1, 11)) redir1.SetMap(1, 2) redir2.SetMap(2, ast.RedirMapSupress) cmd.AddRedirect(redir1) cmd.AddRedirect(redir2) ln.Push(cmd) expected.Root = ln parserTest("multiple redirects", `cmd >[1=2] >[2=]`, expected, t, true) } func TestParseCommandWithStringsEqualsNot(t *testing.T) { expected := ast.NewTree("strings works as expected") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd1 := ast.NewCommandNode(token.NewFileInfo(1, 0), "echo", false) cmd2 := ast.NewCommandNode(token.NewFileInfo(2, 0), "echo", false) cmd1.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 5), "hello", false)) cmd2.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 6), "hello", true)) ln.Push(cmd1) ln.Push(cmd2) expected.Root = ln parserTest("strings works as expected", `echo hello echo "hello" `, expected, t, true) } func TestParseCommandSeparatedBySemicolon(t *testing.T) { expected := ast.NewTree("semicolon") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd1 := ast.NewCommandNode(token.NewFileInfo(1, 0), "echo", false) cmd2 := ast.NewCommandNode(token.NewFileInfo(1, 11), "echo", false) cmd1.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 5), "hello", false)) cmd2.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 16), "world", false)) ln.Push(cmd1) ln.Push(cmd2) expected.Root = ln parserTest("strings works as expected", `echo hello;echo world`, expected, t, false) } func TestParseStringNotFinished(t *testing.T) { parser := NewParser("string not finished", `echo "hello world`) tr, err := parser.Parse() if err == nil { t.Error("Error: should fail") return } if tr != nil { t.Errorf("Failed to parse") return } } func TestParseCd(t *testing.T) { expected := ast.NewTree("test cd") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cd := ast.NewCommandNode(token.NewFileInfo(1, 0), "cd", false) arg := ast.NewStringExpr(token.NewFileInfo(1, 3), "/tmp", false) cd.AddArg(arg) ln.Push(cd) expected.Root = ln parserTest("test cd", "cd /tmp", expected, t, true) // test cd into home expected = ast.NewTree("test cd into home") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) cd = ast.NewCommandNode(token.NewFileInfo(1, 0), "cd", false) ln.Push(cd) expected.Root = ln parserTest("test cd into home", "cd", expected, t, true) // test cd .. expected = ast.NewTree("test cd ..") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) cd = ast.NewCommandNode(token.NewFileInfo(1, 0), "cd", false) cd.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 3), "..", false)) ln.Push(cd) expected.Root = ln parserTest("test cd ..", "cd ..", expected, t, true) expected = ast.NewTree("cd into HOME by setenv") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) assign := ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "HOME", nil), ast.NewStringExpr(token.NewFileInfo(1, 8), "/", true), ) set, err := ast.NewSetenvNode(token.NewFileInfo(3, 0), "HOME", nil) if err != nil { t.Fatal(err) } cd = ast.NewCommandNode(token.NewFileInfo(5, 0), "cd", false) pwd := ast.NewCommandNode(token.NewFileInfo(6, 0), "pwd", false) ln.Push(assign) ln.Push(set) ln.Push(cd) ln.Push(pwd) expected.Root = ln parserTest("test cd into HOME by setenv", `HOME = "/" setenv HOME cd pwd`, expected, t, true) // Test cd into custom variable expected = ast.NewTree("cd into variable value") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) arg = ast.NewStringExpr(token.NewFileInfo(1, 10), "/home/i4k/gopath", true) assign = ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "GOPATH", nil), arg, ) cd = ast.NewCommandNode(token.NewFileInfo(3, 0), "cd", false) arg2 := ast.NewVarExpr(token.NewFileInfo(3, 3), "$GOPATH") cd.AddArg(arg2) ln.Push(assign) ln.Push(cd) expected.Root = ln parserTest("test cd into variable value", `GOPATH = "/home/i4k/gopath" cd $GOPATH`, expected, t, true) // Test cd into custom variable expected = ast.NewTree("cd into variable value with concat") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) arg = ast.NewStringExpr(token.NewFileInfo(1, 10), "/home/i4k/gopath", true) assign = ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "GOPATH", nil), arg, ) concat := make([]ast.Expr, 0, 2) concat = append(concat, ast.NewVarExpr(token.NewFileInfo(3, 3), "$GOPATH")) concat = append(concat, ast.NewStringExpr(token.NewFileInfo(3, 12), "/src/github.com", true)) cd = ast.NewCommandNode(token.NewFileInfo(3, 0), "cd", false) carg := ast.NewConcatExpr(token.NewFileInfo(3, 3), concat) cd.AddArg(carg) ln.Push(assign) ln.Push(cd) expected.Root = ln parserTest("test cd into variable value", `GOPATH = "/home/i4k/gopath" cd $GOPATH+"/src/github.com"`, expected, t, true) } func TestParseConcatOfIndexedVar(t *testing.T) { expected := ast.NewTree("concat indexed var") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) arg1 := ast.NewStringExpr(token.NewFileInfo(1, 4), "ec2", false) arg2 := ast.NewStringExpr(token.NewFileInfo(1, 8), "create-tags", false) arg3 := ast.NewStringExpr(token.NewFileInfo(1, 20), "--resources", false) arg4 := ast.NewVarExpr(token.NewFileInfo(1, 32), "$resource") arg5 := ast.NewStringExpr(token.NewFileInfo(1, 42), "--tags", false) c1 := ast.NewStringExpr(token.NewFileInfo(1, 50), "Key=", true) c2 := ast.NewIndexExpr(token.NewFileInfo(1, 56), ast.NewVarExpr(token.NewFileInfo(1, 56), "$tag"), ast.NewIntExpr(token.NewFileInfo(1, 61), 0)) c3 := ast.NewStringExpr(token.NewFileInfo(1, 65), ",Value=", true) c4 := ast.NewIndexExpr(token.NewFileInfo(1, 74), ast.NewVarExpr(token.NewFileInfo(1, 74), "$tag"), ast.NewIntExpr(token.NewFileInfo(1, 79), 1)) cvalues := make([]ast.Expr, 4) cvalues[0] = c1 cvalues[1] = c2 cvalues[2] = c3 cvalues[3] = c4 arg6 := ast.NewConcatExpr(token.NewFileInfo(1, 50), cvalues) cmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "aws", false) cmd.AddArg(arg1) cmd.AddArg(arg2) cmd.AddArg(arg3) cmd.AddArg(arg4) cmd.AddArg(arg5) cmd.AddArg(arg6) ln.Push(cmd) expected.Root = ln parserTest("concat indexed var", `aws ec2 create-tags --resources $resource --tags "Key="+$tag[0]+",Value="+$tag[1]`, expected, t, true) } func TestParseRfork(t *testing.T) { expected := ast.NewTree("test rfork") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd1 := ast.NewRforkNode(token.NewFileInfo(1, 0)) f1 := ast.NewStringExpr(token.NewFileInfo(1, 6), "u", false) cmd1.SetFlags(f1) ln.Push(cmd1) expected.Root = ln parserTest("test rfork", "rfork u", expected, t, true) } func TestParseRforkWithBlock(t *testing.T) { expected := ast.NewTree("rfork with block") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) rfork := ast.NewRforkNode(token.NewFileInfo(1, 0)) arg := ast.NewStringExpr(token.NewFileInfo(1, 6), "u", false) rfork.SetFlags(arg) insideFork := ast.NewCommandNode(token.NewFileInfo(2, 1), "mount", false) insideFork.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 7), "-t", false)) insideFork.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 10), "proc", false)) insideFork.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 15), "proc", false)) insideFork.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 20), "/proc", false)) bln := ast.NewBlockNode(token.NewFileInfo(1, 8)) bln.Push(insideFork) subtree := ast.NewTree("rfork") subtree.Root = bln rfork.SetTree(subtree) ln.Push(rfork) expected.Root = ln parserTest("rfork with block", `rfork u { mount -t proc proc /proc } `, expected, t, true) } func TestUnpairedRforkBlocks(t *testing.T) { parser := NewParser("unpaired", "rfork u {") _, err := parser.Parse() if err == nil { t.Errorf("Should fail because of unpaired open/close blocks") return } } func TestParseImport(t *testing.T) { expected := ast.NewTree("test import") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) importStmt := ast.NewImportNode(token.NewFileInfo(1, 0), ast.NewStringExpr(token.NewFileInfo(1, 7), "env.sh", false)) ln.Push(importStmt) expected.Root = ln parserTest("test import", "import env.sh", expected, t, true) expected = ast.NewTree("test import with quotes") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) importStmt = ast.NewImportNode(token.NewFileInfo(1, 0), ast.NewStringExpr(token.NewFileInfo(1, 8), "env.sh", true)) ln.Push(importStmt) expected.Root = ln parserTest("test import", `import "env.sh"`, expected, t, true) } func TestParseIf(t *testing.T) { expected := ast.NewTree("test if") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl := ast.NewIfNode(token.NewFileInfo(1, 0)) ifDecl.SetLvalue(ast.NewStringExpr(token.NewFileInfo(1, 4), "test", true)) ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(1, 14), "other", true)) ifDecl.SetOp("==") subBlock := ast.NewBlockNode(token.NewFileInfo(1, 21)) cmd := ast.NewCommandNode(token.NewFileInfo(2, 1), "pwd", false) subBlock.Push(cmd) ifTree := ast.NewTree("if block") ifTree.Root = subBlock ifDecl.SetIfTree(ifTree) ln.Push(ifDecl) expected.Root = ln parserTest("test if", `if "test" == "other" { pwd }`, expected, t, true) expected = ast.NewTree("test if") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl = ast.NewIfNode(token.NewFileInfo(1, 0)) ifDecl.SetLvalue(ast.NewStringExpr(token.NewFileInfo(1, 4), "", true)) ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(1, 10), "other", true)) ifDecl.SetOp("!=") subBlock = ast.NewBlockNode(token.NewFileInfo(1, 17)) cmd = ast.NewCommandNode(token.NewFileInfo(2, 1), "pwd", false) subBlock.Push(cmd) ifTree = ast.NewTree("if block") ifTree.Root = subBlock ifDecl.SetIfTree(ifTree) ln.Push(ifDecl) expected.Root = ln parserTest("test if", `if "" != "other" { pwd }`, expected, t, true) } func TestParseFuncall(t *testing.T) { expected := ast.NewTree("fn inv") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) aFn := ast.NewFnInvNode(token.NewFileInfo(1, 0), "a") ln.Push(aFn) expected.Root = ln parserTest("test basic fn inv", `a()`, expected, t, true) expected = ast.NewTree("fn inv") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) aFn = ast.NewFnInvNode(token.NewFileInfo(1, 0), "a") bFn := ast.NewFnInvNode(token.NewFileInfo(1, 2), "b") aFn.AddArg(bFn) ln.Push(aFn) expected.Root = ln parserTest("test fn composition", `a(b())`, expected, t, true) expected = ast.NewTree("fn inv") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) aFn = ast.NewFnInvNode(token.NewFileInfo(1, 0), "a") bFn = ast.NewFnInvNode(token.NewFileInfo(1, 2), "b") b2Fn := ast.NewFnInvNode(token.NewFileInfo(1, 7), "b") aFn.AddArg(bFn) aFn.AddArg(b2Fn) ln.Push(aFn) expected.Root = ln parserTest("test fn composition", `a(b(), b())`, expected, t, true) expected = ast.NewTree("fn inv") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) aFn = ast.NewFnInvNode(token.NewFileInfo(1, 0), "a") bFn = ast.NewFnInvNode(token.NewFileInfo(1, 2), "b") b2Fn = ast.NewFnInvNode(token.NewFileInfo(1, 4), "b") bFn.AddArg(b2Fn) aFn.AddArg(bFn) ln.Push(aFn) expected.Root = ln parserTest("test fn composition", `a(b(b()))`, expected, t, true) expected = ast.NewTree("fn inv list") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) aFn = ast.NewFnInvNode(token.NewFileInfo(1, 0), "a") lExpr := ast.NewListExpr(token.NewFileInfo(1, 2), []ast.Expr{ ast.NewStringExpr(token.NewFileInfo(1, 4), "1", true), ast.NewStringExpr(token.NewFileInfo(1, 8), "2", true), ast.NewStringExpr(token.NewFileInfo(1, 12), "3", true), }) aFn.AddArg(lExpr) ln.Push(aFn) expected.Root = ln parserTest("test fn list arg", `a(("1" "2" "3"))`, expected, t, true) // test valid funcall syntaxes (do not verify AST) for _, tc := range []string{ `func()`, `func(())`, // empty list `func($a)`, `_($a, $b)`, `func($a())`, `func($a(), $b())`, `func($a($b($c())))`, `func($a("a"))`, `__((((()))))`, // perfect fit for a nash obfuscating code contest `_( () )`, `_( (), (), (), (), (), )`, `_((() () () () ()))`, `deploy((bomb shell))`, // unquoted list elements are still supported :-( `func("a", ())`, `_($a+$b)`, `_($a+"")`, `_(""+$a)`, `func((()()))`, } { parser := NewParser("test", tc) _, err := parser.Parse() if err != nil { t.Fatal(err) } } } func TestParseFuncallInvalid(t *testing.T) { for _, tc := range []string{ `test(()`, `_())`, `func(a)`, `func("a", a)`, `func(_(((((()))))`, `func(()+())`, `func("1"+("2" "3"))`, `func(()())`, } { parser := NewParser("test", tc) _, err := parser.Parse() if err == nil { t.Fatalf("Syntax '%s' must fail...", tc) } } } func TestParseIfFnInv(t *testing.T) { expected := ast.NewTree("test if") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl := ast.NewIfNode(token.NewFileInfo(1, 0)) ifDecl.SetLvalue(ast.NewFnInvNode(token.NewFileInfo(1, 3), "test")) ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(1, 14), "other", true)) ifDecl.SetOp("==") subBlock := ast.NewBlockNode(token.NewFileInfo(1, 21)) cmd := ast.NewCommandNode(token.NewFileInfo(2, 1), "pwd", false) subBlock.Push(cmd) ifTree := ast.NewTree("if block") ifTree.Root = subBlock ifDecl.SetIfTree(ifTree) ln.Push(ifDecl) expected.Root = ln parserTest("test if", `if test() == "other" { pwd }`, expected, t, true) expected = ast.NewTree("test if") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl = ast.NewIfNode(token.NewFileInfo(1, 0)) fnInv := ast.NewFnInvNode(token.NewFileInfo(1, 3), "test") fnInv.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 9), "bleh", true)) ifDecl.SetLvalue(fnInv) ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(1, 20), "other", true)) ifDecl.SetOp("!=") subBlock = ast.NewBlockNode(token.NewFileInfo(1, 27)) cmd = ast.NewCommandNode(token.NewFileInfo(2, 1), "pwd", false) subBlock.Push(cmd) ifTree = ast.NewTree("if block") ifTree.Root = subBlock ifDecl.SetIfTree(ifTree) ln.Push(ifDecl) expected.Root = ln parserTest("test if", `if test("bleh") != "other" { pwd }`, expected, t, true) } func TestParseIfLvariable(t *testing.T) { expected := ast.NewTree("test if with variable") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl := ast.NewIfNode(token.NewFileInfo(1, 0)) ifDecl.SetLvalue(ast.NewVarExpr(token.NewFileInfo(1, 3), "$test")) ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(1, 13), "other", true)) ifDecl.SetOp("==") subBlock := ast.NewBlockNode(token.NewFileInfo(1, 20)) cmd := ast.NewCommandNode(token.NewFileInfo(2, 1), "pwd", false) subBlock.Push(cmd) ifTree := ast.NewTree("if block") ifTree.Root = subBlock ifDecl.SetIfTree(ifTree) ln.Push(ifDecl) expected.Root = ln parserTest("test if", `if $test == "other" { pwd }`, expected, t, true) } func TestParseIfRvariable(t *testing.T) { expected := ast.NewTree("test if with variable") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl := ast.NewIfNode(token.NewFileInfo(1, 0)) ifDecl.SetLvalue(ast.NewVarExpr(token.NewFileInfo(1, 3), "$test")) ifDecl.SetRvalue(ast.NewVarExpr(token.NewFileInfo(1, 12), "$other")) ifDecl.SetOp("==") subBlock := ast.NewBlockNode(token.NewFileInfo(1, 19)) cmd := ast.NewCommandNode(token.NewFileInfo(2, 1), "pwd", false) subBlock.Push(cmd) ifTree := ast.NewTree("if block") ifTree.Root = subBlock ifDecl.SetIfTree(ifTree) ln.Push(ifDecl) expected.Root = ln parserTest("test if", `if $test == $other { pwd }`, expected, t, true) } func TestParseIfElse(t *testing.T) { expected := ast.NewTree("test if else with variable") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl := ast.NewIfNode(token.NewFileInfo(1, 0)) ifDecl.SetLvalue(ast.NewVarExpr(token.NewFileInfo(1, 3), "$test")) ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(1, 13), "other", true)) ifDecl.SetOp("==") subBlock := ast.NewBlockNode(token.NewFileInfo(1, 20)) cmd := ast.NewCommandNode(token.NewFileInfo(2, 1), "pwd", false) subBlock.Push(cmd) ifTree := ast.NewTree("if block") ifTree.Root = subBlock ifDecl.SetIfTree(ifTree) elseBlock := ast.NewBlockNode(token.NewFileInfo(3, 7)) exitCmd := ast.NewCommandNode(token.NewFileInfo(4, 1), "exit", false) elseBlock.Push(exitCmd) elseTree := ast.NewTree("else block") elseTree.Root = elseBlock ifDecl.SetElseTree(elseTree) ln.Push(ifDecl) expected.Root = ln parserTest("test if", `if $test == "other" { pwd } else { exit }`, expected, t, true) } func TestParseIfElseIf(t *testing.T) { expected := ast.NewTree("test if else with variable") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl := ast.NewIfNode(token.NewFileInfo(1, 0)) ifDecl.SetLvalue(ast.NewVarExpr(token.NewFileInfo(1, 3), "$test")) ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(1, 13), "other", true)) ifDecl.SetOp("==") subBlock := ast.NewBlockNode(token.NewFileInfo(1, 20)) cmd := ast.NewCommandNode(token.NewFileInfo(2, 1), "pwd", false) subBlock.Push(cmd) ifTree := ast.NewTree("if block") ifTree.Root = subBlock ifDecl.SetIfTree(ifTree) elseIfDecl := ast.NewIfNode(token.NewFileInfo(3, 7)) elseIfDecl.SetLvalue(ast.NewVarExpr(token.NewFileInfo(3, 10), "$test")) elseIfDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(3, 20), "others", true)) elseIfDecl.SetOp("==") elseIfBlock := ast.NewBlockNode(token.NewFileInfo(3, 28)) elseifCmd := ast.NewCommandNode(token.NewFileInfo(4, 1), "ls", false) elseIfBlock.Push(elseifCmd) elseIfTree := ast.NewTree("if block") elseIfTree.Root = elseIfBlock elseIfDecl.SetIfTree(elseIfTree) elseBlock := ast.NewBlockNode(token.NewFileInfo(5, 7)) exitCmd := ast.NewCommandNode(token.NewFileInfo(6, 1), "exit", false) elseBlock.Push(exitCmd) elseTree := ast.NewTree("else block") elseTree.Root = elseBlock elseIfDecl.SetElseTree(elseTree) elseBlock2 := ast.NewBlockNode(token.NewFileInfo(3, 7)) elseBlock2.Push(elseIfDecl) elseTree2 := ast.NewTree("first else tree") elseTree2.Root = elseBlock2 ifDecl.SetElseTree(elseTree2) ln.Push(ifDecl) expected.Root = ln parserTest("test if", `if $test == "other" { pwd } else if $test == "others" { ls } else { exit }`, expected, t, true) } func TestParseFnBasic(t *testing.T) { // root expected := ast.NewTree("fn") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) // fn fn := ast.NewFnDeclNode(token.NewFileInfo(1, 3), "build") tree := ast.NewTree("fn body") lnBody := ast.NewBlockNode(token.NewFileInfo(1, 0)) tree.Root = lnBody fn.SetTree(tree) // root ln.Push(fn) expected.Root = ln parserTest("fn", `fn build() { }`, expected, t, true) // root expected = ast.NewTree("fn") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) // fn fn = ast.NewFnDeclNode(token.NewFileInfo(1, 3), "build") tree = ast.NewTree("fn body") lnBody = ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "ls", false) lnBody.Push(cmd) tree.Root = lnBody fn.SetTree(tree) // root ln.Push(fn) expected.Root = ln parserTest("fn", `fn build() { ls }`, expected, t, true) // root expected = ast.NewTree("fn") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) // fn fn = ast.NewFnDeclNode(token.NewFileInfo(1, 3), "build") fn.AddArg(ast.NewFnArgNode(token.NewFileInfo(1, 9), "image", false)) tree = ast.NewTree("fn body") lnBody = ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd = ast.NewCommandNode(token.NewFileInfo(1, 0), "ls", false) lnBody.Push(cmd) tree.Root = lnBody fn.SetTree(tree) // root ln.Push(fn) expected.Root = ln parserTest("fn", `fn build(image) { ls }`, expected, t, true) // root expected = ast.NewTree("fn") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) // fn fn = ast.NewFnDeclNode(token.NewFileInfo(1, 3), "build") fn.AddArg(ast.NewFnArgNode(token.NewFileInfo(1, 9), "image", false)) fn.AddArg(ast.NewFnArgNode(token.NewFileInfo(1, 16), "debug", false)) tree = ast.NewTree("fn body") lnBody = ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd = ast.NewCommandNode(token.NewFileInfo(1, 0), "ls", false) lnBody.Push(cmd) tree.Root = lnBody fn.SetTree(tree) // root ln.Push(fn) expected.Root = ln parserTest("fn", `fn build(image, debug) { ls }`, expected, t, true) } func TestParseInlineFnDecl(t *testing.T) { expected := ast.NewTree("fn") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) fn := ast.NewFnDeclNode(token.NewFileInfo(1, 3), "cd") tree := ast.NewTree("fn body") lnBody := ast.NewBlockNode(token.NewFileInfo(1, 0)) echo := ast.NewCommandNode(token.NewFileInfo(1, 11), "echo", false) echo.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 16), "hello", true)) lnBody.Push(echo) tree.Root = lnBody fn.SetTree(tree) // root ln.Push(fn) expected.Root = ln parserTest("inline fn", `fn cd() { echo "hello" }`, expected, t, false) test := ast.NewCommandNode(token.NewFileInfo(1, 26), "test", false) test.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 32), "-d", false)) test.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 35), "/etc", false)) pipe := ast.NewPipeNode(token.NewFileInfo(1, 11), false) pipe.AddCmd(echo) pipe.AddCmd(test) lnBody = ast.NewBlockNode(token.NewFileInfo(1, 0)) lnBody.Push(pipe) tree.Root = lnBody fn.SetTree(tree) ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ln.Push(fn) expected.Root = ln parserTest("inline fn", `fn cd() { echo "hello" | test -d /etc }`, expected, t, false) } func TestParseBindFn(t *testing.T) { expected := ast.NewTree("bindfn") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) bindFn := ast.NewBindFnNode(token.NewFileInfo(1, 0), "cd", "cd2") ln.Push(bindFn) expected.Root = ln parserTest("bindfn", `bindfn cd cd2`, expected, t, true) } func TestParseRedirectionVariable(t *testing.T) { expected := ast.NewTree("redirection var") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 0), "cmd", false) redir := ast.NewRedirectNode(token.NewFileInfo(1, 4)) redirArg := ast.NewVarExpr(token.NewFileInfo(1, 6), "$outFname") redir.SetLocation(redirArg) cmd.AddRedirect(redir) ln.Push(cmd) expected.Root = ln parserTest("redir var", `cmd > $outFname`, expected, t, true) } func TestParseReturn(t *testing.T) { expected := ast.NewTree("return") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ret := ast.NewReturnNode(token.NewFileInfo(1, 0)) ln.Push(ret) expected.Root = ln parserTest("return", `return`, expected, t, true) expected = ast.NewTree("return list") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ret = ast.NewReturnNode(token.NewFileInfo(1, 0)) listvalues := make([]ast.Expr, 2) listvalues[0] = ast.NewStringExpr(token.NewFileInfo(1, 9), "val1", true) listvalues[1] = ast.NewStringExpr(token.NewFileInfo(1, 16), "val2", true) retReturn := ast.NewListExpr(token.NewFileInfo(1, 7), listvalues) ret.Returns = []ast.Expr{retReturn} ln.Push(ret) expected.Root = ln parserTest("return", `return ("val1" "val2")`, expected, t, true) expected = ast.NewTree("return variable") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ret = ast.NewReturnNode(token.NewFileInfo(1, 0)) ret.Returns = []ast.Expr{ast.NewVarExpr(token.NewFileInfo(1, 7), "$var")} ln.Push(ret) expected.Root = ln parserTest("return", `return $var`, expected, t, true) expected = ast.NewTree("return string") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ret = ast.NewReturnNode(token.NewFileInfo(1, 0)) ret.Returns = []ast.Expr{ast.NewStringExpr(token.NewFileInfo(1, 8), "value", true)} ln.Push(ret) expected.Root = ln parserTest("return", `return "value"`, expected, t, true) expected = ast.NewTree("return funcall") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ret = ast.NewReturnNode(token.NewFileInfo(1, 0)) aFn := ast.NewFnInvNode(token.NewFileInfo(1, 7), "a") ret.Returns = []ast.Expr{aFn} ln.Push(ret) expected.Root = ln parserTest("return", `return a()`, expected, t, true) expected = ast.NewTree("return multiple values") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ret = ast.NewReturnNode(token.NewFileInfo(1, 0)) a1 := ast.NewStringExpr(token.NewFileInfo(1, 8), "1", true) a2 := ast.NewStringExpr(token.NewFileInfo(1, 13), "2", true) a3 := ast.NewStringExpr(token.NewFileInfo(1, 18), "3", true) ret.Returns = []ast.Expr{a1, a2, a3} ln.Push(ret) expected.Root = ln parserTest("return", `return "1", "2", "3"`, expected, t, true) } func TestParseIfInvalid(t *testing.T) { parser := NewParser("if invalid", `if a == b { pwd }`) _, err := parser.Parse() if err == nil { t.Error("Must fail. Only quoted strings and variables on if clauses.") return } } func TestParseFor(t *testing.T) { expected := ast.NewTree("for") forStmt := ast.NewForNode(token.NewFileInfo(1, 0)) forTree := ast.NewTree("for block") forBlock := ast.NewBlockNode(token.NewFileInfo(1, 0)) forTree.Root = forBlock forStmt.SetTree(forTree) ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) ln.Push(forStmt) expected.Root = ln parserTest("for", `for { }`, expected, t, true) forStmt.SetIdentifier("f") forStmt.SetInExpr(ast.NewVarExpr(token.NewFileInfo(1, 9), "$files")) parserTest("for", `for f in $files { }`, expected, t, true) forStmt.SetIdentifier("f") fnInv := ast.NewFnInvNode(token.NewFileInfo(1, 9), "getfiles") fnArg := ast.NewStringExpr(token.NewFileInfo(1, 19), "/", true) fnInv.AddArg(fnArg) forStmt.SetInExpr(fnInv) parserTest("for", `for f in getfiles("/") { }`, expected, t, true) forStmt.SetIdentifier("f") value1 := ast.NewStringExpr(token.NewFileInfo(1, 10), "1", false) value2 := ast.NewStringExpr(token.NewFileInfo(1, 12), "2", false) value3 := ast.NewStringExpr(token.NewFileInfo(1, 14), "3", false) value4 := ast.NewStringExpr(token.NewFileInfo(1, 16), "4", false) value5 := ast.NewStringExpr(token.NewFileInfo(1, 18), "5", false) list := ast.NewListExpr(token.NewFileInfo(1, 9), []ast.Expr{ value1, value2, value3, value4, value5, }) forStmt.SetInExpr(list) parserTest("for", `for f in (1 2 3 4 5) { }`, expected, t, true) } func TestParseVariableIndexing(t *testing.T) { expected := ast.NewTree("variable indexing") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) indexedVar := ast.NewIndexExpr( token.NewFileInfo(1, 7), ast.NewVarExpr(token.NewFileInfo(1, 7), "$values"), ast.NewIntExpr(token.NewFileInfo(1, 15), 0), ) assignment := ast.NewSingleAssignNode(token.NewFileInfo(1, 0), ast.NewNameNode(token.NewFileInfo(1, 0), "test", nil), indexedVar, ) ln.Push(assignment) expected.Root = ln parserTest("variable indexing", `test = $values[0]`, expected, t, true) ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) ifDecl := ast.NewIfNode(token.NewFileInfo(1, 0)) lvalue := ast.NewVarExpr(token.NewFileInfo(1, 3), "$values") indexedVar = ast.NewIndexExpr(token.NewFileInfo(1, 3), lvalue, ast.NewIntExpr(token.NewFileInfo(1, 11), 0)) ifDecl.SetLvalue(indexedVar) ifDecl.SetOp("==") ifDecl.SetRvalue(ast.NewStringExpr(token.NewFileInfo(1, 18), "1", true)) ifBlock := ast.NewTree("if") lnBody := ast.NewBlockNode(token.NewFileInfo(1, 21)) ifBlock.Root = lnBody ifDecl.SetIfTree(ifBlock) ln.Push(ifDecl) expected.Root = ln parserTest("variable indexing", `if $values[0] == "1" { }`, expected, t, true) } func TestParseMultilineCmdExec(t *testing.T) { expected := ast.NewTree("parser simple") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 1), "echo", true) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 7), "hello world", true)) ln.Push(cmd) expected.Root = ln parserTest("parser simple", `(echo "hello world")`, expected, t, true) expected = ast.NewTree("parser aws cmd") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd = ast.NewCommandNode(token.NewFileInfo(2, 1), "aws", true) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 5), "ec2", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 9), "run-instances", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(3, 3), "--image-id", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(3, 14), "ami-xxxxxxxx", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(4, 3), "--count", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(4, 11), "1", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(5, 3), "--instance-type", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(5, 19), "t1.micro", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(6, 3), "--key-name", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(6, 14), "MyKeyPair", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(7, 3), "--security-groups", false)) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(7, 21), "my-sg", false)) ln.Push(cmd) expected.Root = ln parserTest("parser simple", `( aws ec2 run-instances --image-id ami-xxxxxxxx --count 1 --instance-type t1.micro --key-name MyKeyPair --security-groups my-sg )`, expected, t, true) } func TestParseMultilineCmdAssign(t *testing.T) { expected := ast.NewTree("parser simple assign") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) cmd := ast.NewCommandNode(token.NewFileInfo(1, 10), "echo", true) cmd.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 16), "hello world", true)) assign, err := ast.NewExecAssignNode(token.NewFileInfo(1, 0), []*ast.NameNode{ast.NewNameNode(token.NewFileInfo(1, 0), "hello", nil)}, cmd, ) if err != nil { t.Error(err) return } ln.Push(assign) expected.Root = ln parserTest("parser simple", `hello <= (echo "hello world")`, expected, t, true) } func TestMultiPipe(t *testing.T) { expected := ast.NewTree("parser pipe") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) first := ast.NewCommandNode(token.NewFileInfo(1, 1), "echo", false) first.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 7), "hello world", true)) second := ast.NewCommandNode(token.NewFileInfo(1, 22), "awk", false) second.AddArg(ast.NewStringExpr(token.NewFileInfo(1, 27), "{print $1}", true)) pipe := ast.NewPipeNode(token.NewFileInfo(1, 20), true) pipe.AddCmd(first) pipe.AddCmd(second) ln.Push(pipe) expected.Root = ln parserTest("parser pipe", `(echo "hello world" | awk "{print $1}")`, expected, t, true) // get longer stringify expected = ast.NewTree("parser pipe") ln = ast.NewBlockNode(token.NewFileInfo(1, 0)) first = ast.NewCommandNode(token.NewFileInfo(2, 1), "echo", false) first.AddArg(ast.NewStringExpr(token.NewFileInfo(2, 7), "hello world", true)) second = ast.NewCommandNode(token.NewFileInfo(3, 1), "awk", false) second.AddArg(ast.NewStringExpr(token.NewFileInfo(3, 6), "{print AAAAAAAAAAAAAAAAAAAAAA}", true)) pipe = ast.NewPipeNode(token.NewFileInfo(2, 20), true) pipe.AddCmd(first) pipe.AddCmd(second) ln.Push(pipe) expected.Root = ln parserTest("parser pipe", `( echo "hello world" | awk "{print AAAAAAAAAAAAAAAAAAAAAA}" )`, expected, t, true) } func TestFnVariadic(t *testing.T) { // root expected := ast.NewTree("variadic") ln := ast.NewBlockNode(token.NewFileInfo(1, 0)) // fn fn := ast.NewFnDeclNode(token.NewFileInfo(1, 3), "println") fn.AddArg(ast.NewFnArgNode(token.NewFileInfo(1, 11), "fmt", false)) fn.AddArg(ast.NewFnArgNode(token.NewFileInfo(1, 16), "arg", true)) tree := ast.NewTree("fn body") lnBody := ast.NewBlockNode(token.NewFileInfo(1, 0)) print := ast.NewFnInvNode(token.NewFileInfo(2, 2), "print") print.AddArg(ast.NewConcatExpr(token.NewFileInfo(1, 7), []ast.Expr{ ast.NewVarExpr(token.NewFileInfo(2, 7), "$fmt"), ast.NewStringExpr(token.NewFileInfo(2, 12), "\n", true), })) print.AddArg(ast.NewVarVariadicExpr(token.NewFileInfo(2, 12), "$arg", true)) lnBody.Push(print) tree.Root = lnBody fn.SetTree(tree) // root ln.Push(fn) expected.Root = ln parserTest("fn", `fn println(fmt, arg...) { print($fmt+"\n", $arg...) }`, expected, t, true) } func TestParseValidDotdotdot(t *testing.T) { for _, tc := range []string{ // things that should not break "ls ...", "go get ./...", `echo "..."`, `strangecmd... -h`, `bad_designed...fail -f`, } { parser := NewParser("", tc) _, err := parser.Parse() if err != nil { t.Fatalf("Code: '%s' failed: %s", tc, err.Error()) } } } func TestParseInvalidDotdotdot(t *testing.T) { for _, tc := range []string{ "...", `if ... == "" {}`, `if $var... == "" {}`, `a = $var...`, `a, b, c = ("a" "b" "c")...`, // please, no // `fn println(arg..., fmt) {}`, // Not sure if must fail at parsing... } { parser := NewParser("", tc) _, err := parser.Parse() if err == nil { t.Fatalf("Syntax '%s' must fail", tc) } } } func TestFunctionPipes(t *testing.T) { parser := NewParser("invalid pipe with functions", `echo "some thing" | replace(" ", "|")`) _, err := parser.Parse() if err == nil { t.Error("Must fail. Function must be bind'ed to command name to use in pipe.") return } } ================================================ FILE: proposal/1-scope-management.md ================================================ # Proposal: Proper scope management This has already been implemented but these docs remain here as some sort of rationale. In the end we implemented proposal one, requiring new variables to be declared with the **var** keyword. ## Abstract Currently on nash there is no way to properly work with closures because scope management is very limited. Lets elaborate on the problem by implementing a list object by instantiating a set of functions that manipulates the same data. ```sh fn list() { l = () fn add(val) { l <= append($l, $val) } fn get(i) { return $l[$i] } fn string() { print("list: [%s]\n", $l) } return $add, $get, $string } ``` The idea is to hide all list data behind these 3 functions that will manipulate the same data. The problem is that today this is not possible, using this code: ```sh add, get, string <= list() $add("1") $add("2") $string() v <= $get("0") echo $v ``` Will result in: ``` list: [] /tmp/test.sh:27:5: /tmp/test.sh:11:23: Index out of bounds. len($l) == 0, but given 0 ``` As you can see, even when we call the **add** function the list remains empty, why is that ? The problem is on the add function: ```sh fn add(val) { l <= append($l, $val) } ``` When we reference the **l** variable it uses the reference on the outer scope (the empty list), but there is no way to express syntactically that we want to change the list on the outer scope instead of creating a new variable **l** (shadowing the outer **l**). That is why the **get** and **print** functions are always referencing an outer list **l** that is empty, a new one is created each time the add function is called. In this document we navigate the solution space for this problem. ## Proposal I - Create new variables explicitly On this proposal new variable creation requires an explicit syntax construction. We could add a new keyword `var` that will be used to declare and initialize variables in the local scope, like this: ```js var i = "0" ``` While the current syntax: ```js i = "0" ``` Will be assigning a new value to an already existent variable **i**. The assignment will first look for the target variable in the local scope and then in the parent, traversing the entire stack, until it's found and then updated, otherwise (in case the variable is not found) the interpreter must abort with error. ```sh var count = "0" # declare local variable fn inc() { # update outer variable count, _ <= expr $count "+" 1 } inc() print($count) # outputs: 1 ``` Below is how this proposal solves the list example: ```sh fn list() { # initialize an "l" variable in this scope var l = () fn add(val) { # use the "l" variable from parent scope # find first in the this scope if not found # then find variable in the parent scope l <= append($l, $val) } fn get(i) { # use the "l" variable from parent scope return $l[$i] } fn string() { # use the "l" variable from parent scope print("list: [%s]\n", $l) } fn not_clear() { # force initialize a new "l" variable in this scope # because this the "l" list in the parent scope is not cleared var l = () } return $add, $get, $string } ``` Syntactically, the `var` statement is an extension of the assignment and exec-assignment statements, and then it should support multiple declarations in a single statement also. Eg.: ```sh var i, j = "0", "1" var body, err <= curl -f $url var name, surname, err <= getAuthor() ``` Using var always creates new variables, shadowing previous ones, for example: ```sh var a, b = "0", "1" # works fine, variables didn't existed before var a, b, c = "4", "5", "6" # works! too, creating new a, b, c ``` On a dynamic typed language there is very little difference between creating a new var or just reassigning it since variables are just references that store no type information at all. For example, what is the difference between this: ``` var a = "1" a = () ``` And this ? ``` var a = "1" var a = () ``` The behavior will be exactly the same, there is no semantic error on reassigning the same variable to a value with a different type, so reassigning on redeclaring has no difference at all (although it makes sense for statically typed languages). Statements are evaluated in order, so this: ``` a = () var a = "1" ``` Is **NOT** the same as this: ``` var a = "1" var a = () ``` This is easier to understand when using closures, let's go back to our list implementation, we had something like this: ``` var l = () fn add(val) { # use the "l" variable from parent scope # find first in the this scope if not found # then find variable in the parent scope l <= append($l, $val) } ``` If we write this: ``` var l = () fn add(val) { # creates new var var l = () # manipulates new l var l <= append($l, $val) } ``` The **add** function will not manipulate the **l** variable from the outer scope, and our list implementation will not work properly. But writing this: ``` var l = () fn add(val) { # manipulates outer l var l <= append($l, $val) # creates new var that is useless var l = () } ``` Will work, since we assigned a new value to the outer **l** before creating a new **l** var. The approach described here is very similar to how variables are handled in [Lua](https://www.lua.org/), with the exception that Lua uses the **local** keyword, instead of var. Also, Lua allows global variables to be created by default, on Nash we prefer to avoid global stuff and produce an error when assigning new values to variables that do not exist. Summarizing, on this proposal creating new variables is explicit and referencing existent variables on outer scopes is implicit. ## Proposal II - Manipulate outer scope explicitly This proposal adds a new `outer` keyword that permits the update of variables in the outer scope. The default and implicit behavior of variable assignments is to always create a new variable. Considering our list example: ```sh fn list() { # initialize an "l" variable in this scope l = () fn add(val) { # use the "l" variable from the parent outer l <= append($l, $val) } fn get(i) { # use the "l" variable from the parent outer l return $l[$i] } fn string() { # use the "l" variable from the parent outer l print("list: [%s]\n", $l) } return $add, $get, $string } ``` The `outer` keyword has the same meaning that Python's `global` keyword. Different from Python global, outer must appear on all assignments, like this: ```sh fn list() { # initialize an "l" variable in this scope l = () fn doubleadd(val) { outer l <= append($l, $val) outer l <= append($l, $val) } return $doubleadd } ``` This would be buggy and only add once: ```sh fn list() { # initialize an "l" variable in this scope l = () fn doubleadd(val) { outer l <= append($l, $val) l <= append($l, $val) } return $doubleadd } ``` Trying to elaborate more on possible combinations when using the **outer** keyword we get at some hard questions, like what does outer means on this case: ``` fn list() { # initialize an "l" variable in this scope l = () fn doubleadd(val) { l <= append($l, $val) outer l <= append($l, $val) } return $doubleadd } ``` Will outer just handle the reference on its own scope or will it jump its own scope and manipulate the outer variable ? The name outer implies that it will manipulate the outer scope, bypassing its own current scope, but how do you read the outer variable ? We would need to support something like: ``` fn list() { # initialize an "l" variable in this scope l = () fn add(val) { l <= "whatever" outer l <= append(outer $l, $val) } return $doubleadd } ``` It is like with outer we are bypassing the lexical semantics of the code, the order of declarations is not relevant anymore since you have a form of "goto" to jump the current scope. ## Comparing both approaches As everything in life, the design space for how to handle scope management is full of tradeoffs. Making outer scope management explicit makes declaring new variables easier, since you have to type less to create new vars. But managing scope using closures gets more cumbersome, consider this nested closures with the **outer** keyword: ```sh fn list() { l = () fn add(val) { # use the "l" variable from the parent outer l <= append($l, $val) fn addagain() { outer l <= append($l, $val) } return $addagain } return $add } ``` And this one with **var** : ```sh fn list() { var l = () fn add(val) { # use the "l" variable from the parent l <= append($l, $val) fn addagain() { l <= append($l, $val) } return $addagain } return $add } ``` The **var** option requires more writing for the common case of declaring new variables (specially on the interactive shell this is pretty annoying), but makes closures pretty natural to write, you just manipulate the variables that exists lexically on your scope, like you would do inside a **if** or **for** block. Thinking about cognition, it seems easier to write buggy code by forgetting to add an **outer** on the code than forgetting to add a **var** and by mistake manipulate an variable outside the scope. The decision to break if the variable does not exist also enhances the **var** option as less buggy since no new variable will be created if you forget the **var**, but lexically reachable variables will be manipulated (this is ameliorated by the fact that we don't have global variables). If we go for **outer** it seems that we are going to write less, but some code, involving closures, will be harder to read (and write). Since code is usually read more than it is written it seems like a sensible choice to optimize for readability and understandability than just save a few keystrokes. But any statements made about cognition are really hard to be considered as a global truth, since all human beings are biased which makes identification of common patterns of cognition really hard. But if software design has any kind of goal, must be this =). ================================================ FILE: proposal/2-concurrency.md ================================================ # Proposal: Concurrency on Nash There has been some discussion on how to provide concurrency to nash. There is a [discussion here](https://github.com/madlambda/nash/issues/224) on how concurrency could be added as a set of built-in functions. As we progressed discussing it seemed desirable to have a concurrency that enforced no sharing between concurrent functions. It eliminates races and forces all communication to happen explicitly, and the performance overhead would not be a problem to a high level language as nash. ## Lightweight Processes This idea is inspired on Erlang concurrency model. Since Nash does not aspire to do everything that Erlang does (like distributed programming) so this is not a copy, we just take some things as inspiration. Why call this a process ? On the [Erlang docs](http://erlang.org/doc/getting_started/conc_prog.html) there is a interesting definition of process: ``` the term "process" is usually used when the threads of execution share no data with each other and the term "thread" when they share data in some way. Threads of execution in Erlang share no data, that is why they are called processes ``` In this context the process word is used to mean a concurrent thread of execution that does not share any data. The only means of communication are through message passing. Since these processes are lightweight creating a lot of them will be cheap (at least must cheaper than OS processes). Instead of using channel instances in this model you send messages to processes (actor model), it works pretty much like a networking model using UDP datagrams. The idea is to leverage this as a syntactic construction of the language to make it as explicit and easy as possible to use. This idea introduces 4 new concepts, 3 built-in functions and one new keyword. The keyword **spawn** is used to spawn a function as a new process. The function **send** is used to send messages to a process. The function **receive** is used to receive messages from a process. The function **self** returns the pid of the process calling it. An example of a simple ping/pong: ``` pid <= spawn fn () { ping, senderpid <= receive() echo $ping send($senderpid, "pong") }() send($pid, "ping", self()) pong <= receive() echo $pong ``` Spawned functions can also receive parameters (always deep copies): ``` pid <= spawn fn (answerpid) { send($answerpid, "pong") }(self()) pong <= receive() echo $pong ``` A simple fan-out/fan-in implementation (N jobs <-> N processes): ``` jobs = ("1" "2" "3" "4" "5") for job in $jobs { spawn fn (job, answerpid) { import io io_println("job[%s] done", $job) send($answerpid, format("result [%s]", $job)) }($job, self()) } for job in $jobs { result <= receive() echo $result } ``` All output (stdout and stderr) of processes go to their parent until the root (main) process, so printing inside a child process will print on the stdout of the main process. ### Advanced Fan-out Fan-in Here is an example of a more elaborated fan-out/fan-in. On this case we have much more jobs to execute than workers, so it requires more coordination than the previous example. For brevity this example does not handle timeouts. Lets suppose an script that tries different passwords on a host: ``` var passwords_feed <= spawn fn() { fn sendpassword(password) { var worker <= receive() if !send($worker, $password) { sendpassword($password) } } for password in generate_passwords() { sendpassword($password) } } fn login(output, passwords_feed, done) { for send($passwords_feed, self()) { var password = receive() var result <= login "someuser" $password send($output, $result) } send($done, "done") } fn outputhandler() { for { var result = receive() if $result == "0" { echo "success" } } } var workers = 10 var feed <= spawn passwords_feed() var outputhandler <= spawn outputhandler() for i in range(0, $workers) { spawn login($outputhandler, $feed, self()) } for i in range(0, $workers) { msg <= receive() if $msg != "done" { echo "dafuck ?" } } ``` ### Error Handling Error handling on this concurrency model is very similar to how we do it on a distributed system. If a remote service fails and just dies and you are using UDP you will never be informed of it, the behavior will be to timeout the request and try again (possibly to another service instance through a load balancer). To implement this idea we can add a timeout to the receive an add a new parameter, a boolean, indicating if there is a message or if a timeout has occurred. Example: ``` msg, ok <= receive(timeout) if !ok { echo "oops timeout" } ``` The timeout can be omitted if you wish to just wait forever. For send operations we need to add just one boolean return value indicating if the process pid exists and the message has been delivered: ``` if !send($pid, $msg) { echo "oops message cant be sent" } ``` Since the processes are always local there is no need for a more detailed error message (the message would always be the same), the error will always involve a pid that has no owner (the process never existed or already exited). We could add a more specific error message if we decide that the process message queue can get too big and we start to drop messages. The error would help to differentiate from a dead process or a overloaded process. An error indicating a overloaded process could help to implement back pressure logic (try again later). But if we are sticking with local concurrency only this may be unnecessary complexity. You can avoid this by always sending N messages and waiting for N responses before sending more messages. ### TODO Spawned functions should have access to imported modules ? (seems like no, but some usages of this may seem odd) If send is never blocking, what if process queue gets too big ? just go on until memory exhausts ? Should send be synchronous how we are going to differentiate between a timeout or a invalid pid error ? On the other hand synchronous send solves the queueing problem. ## Extend rfork Converging to a no shared state between concurrent functions initiated the idea of using the current rfork built-in as a means to express concurrency on Nash. This would already be possible today, the idea is just to make it even easier, specially the communication between different concurrent processes. This idea enables an even greater amount of isolation between concurrent processes since rfork enables different namespaces isolation (besides memory), but it has the obvious fallback of not being very lightweight. Since the idea of nash is to write simple scripts this does not seem to be a problem. If it is on the future we can create lightweight concurrent processes (green threads) that works orthogonally with rfork. The prototype for the new rfork would be something like this: ```sh chan <= rfork [ns_param1, ns_param2] (chan) { //some code } ``` The code on the rfork block does not have access to the lexical outer scope but it receives as a parameter a channel instance. This channel instance can be used by the forked processes and by the creator of the process to communicate. We could use built-in functions: ```sh chan <= rfork [ns_param1, ns_param2] (chan) { cwrite($chan, "hi") } a <= cread($chan) ``` Or some syntactic extension: ```sh chan <= rfork [ns_param1, ns_param2] (chan) { $chan <- "hi" } a <= <-$chan ``` Since this channel is meant only to be used to communicate with the created process, it will be closed when the process exit: ```sh chan <= rfork [ns_param1, ns_param2] (chan) { } # returns empty string when channel is closed <-$chan ``` Fan out and fan in should be pretty trivial: ```sh chan1 <= rfork [ns_param1, ns_param2] (chan) { } chan2 <= rfork [ns_param1, ns_param2] (chan) { } # waiting for both to finish <-$chan1 <-$chan2 ``` ================================================ FILE: scanner/examples_test.go ================================================ package scanner_test import ( "fmt" "github.com/madlambda/nash/scanner" ) func Example() { lex := scanner.Lex("-input-", `echo "hello world"`) for tok := range lex.Tokens { fmt.Println(tok) } // Output: // IDENT // STRING // ; // EOF } ================================================ FILE: scanner/lex.go ================================================ // Package scanner is the lexical parser. package scanner import ( "fmt" "strings" "unicode" "unicode/utf8" "github.com/madlambda/nash/token" ) type ( Token struct { typ token.Token token.FileInfo val string } stateFn func(*Lexer) stateFn // Lexer holds the state of the scanner Lexer struct { name string // identify the source, used only for error reports input string // the string being scanned start int // start position of current token width int // width of last rune read Tokens chan Token // channel of scanned tokens // file positions pos int // file offset line int // current line position lineStart int // line of the symbol's start prevColumn int // previous column value column int // current column position columnStart int // column of the symbol's start openParens int addSemicolon bool } ) const ( eof = -1 ) func (i Token) Type() token.Token { return i.typ } func (i Token) Value() string { return i.val } func (i Token) String() string { switch i.typ { case token.Illegal: return "ERROR: " + i.val case token.EOF: return "EOF" } if len(i.typ.String()) > 10 { return fmt.Sprintf("%s...", i.typ.String()[0:10]) } return fmt.Sprintf("%s", i.typ) } // run lexes the input by executing state functions until the state is nil func (l *Lexer) run() { l.line, l.lineStart, l.column, l.columnStart = 1, 1, 0, 0 for state := lexStart; state != nil; { state = state(l) } l.emit(token.EOF) close(l.Tokens) // No more tokens will be delivered } func (l *Lexer) emitVal(t token.Token, val string, line, column int) { l.Tokens <- Token{ FileInfo: token.NewFileInfo(line, column), typ: t, val: val, } l.start = l.pos l.lineStart = l.line l.columnStart = l.column } func (l *Lexer) emit(t token.Token) { l.Tokens <- Token{ FileInfo: token.NewFileInfo(l.lineStart, l.columnStart), typ: t, val: l.input[l.start:l.pos], } l.start = l.pos l.lineStart = l.line l.columnStart = l.column } // peek returns but does not consume the next rune from input func (l *Lexer) peek() rune { rune := l.next() l.backup() return rune } // next consumes the next rune from input func (l *Lexer) next() rune { var r rune if l.pos >= len(l.input) { l.width = 0 return eof } r, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) l.pos += l.width l.prevColumn = l.column if r == '\n' { l.line++ l.column = 0 } else { l.column++ } return r } // ignore skips over the pending input before this point func (l *Lexer) ignore() { l.start = l.pos l.lineStart = l.line l.columnStart = l.column } // backup steps back one rune func (l *Lexer) backup() { l.pos -= l.width r, _ := utf8.DecodeRuneInString(l.input[l.pos:]) l.column = l.prevColumn if r == '\n' { l.line-- } } // acceptRun consumes a run of runes from the valid setup func (l *Lexer) acceptRun(valid string) { for strings.IndexRune(valid, l.next()) >= 0 { } l.backup() } // errorf emit an error token func (l *Lexer) errorf(format string, args ...interface{}) stateFn { fname := l.name if fname == "" { fname = "" } errMsg := fmt.Sprintf(format, args...) arguments := make([]interface{}, 0, len(args)+2) arguments = append(arguments, fname, l.line, l.column, errMsg) l.Tokens <- Token{ FileInfo: token.NewFileInfo(l.line, l.column), typ: token.Illegal, val: fmt.Sprintf("%s:%d:%d: %s", arguments...), } l.start = len(l.input) l.lineStart = l.line l.columnStart = l.column l.pos = l.start return nil // finish the state machine } func Lex(name, input string) *Lexer { l := &Lexer{ name: name, input: input, Tokens: make(chan Token), } go l.run() // concurrently run state machine return l } func lexStart(l *Lexer) stateFn { r := l.next() switch { case r == eof: if l.addSemicolon { l.emitVal(token.Semicolon, ";", l.line, l.column) } l.addSemicolon = false return nil case '0' <= r && r <= '9': digits := "0123456789" l.acceptRun(digits) next := l.peek() // >[2=] // cmd[2] if next == '=' || next == ']' || (!isIdentifier(l.peek()) && !isArgument(l.peek())) { l.emit(token.Number) } else if isIdentifier(l.peek()) { absorbIdentifier(l) if isArgument(l.peek()) { absorbArgument(l) l.emit(token.Arg) } else { l.emit(token.Ident) } } else if isArgument(l.peek()) { absorbArgument(l) l.emit(token.Arg) } return lexStart case r == ';': l.emit(token.Semicolon) return lexStart case isSpace(r): return lexSpace case isEndOfLine(r): l.ignore() if l.addSemicolon && l.openParens == 0 { l.emitVal(token.Semicolon, ";", l.line, l.column) } l.addSemicolon = false return lexStart case r == '"': l.ignore() return lexQuote case r == '#': return lexComment case r == '+': l.emit(token.Plus) return lexStart case r == '>': l.emit(token.Gt) return lexStart case r == '|': l.emit(token.Pipe) return lexStart case r == '$': r = l.next() if !isIdentifier(r) { return l.errorf("Expected identifier, but found %q", r) } absorbIdentifier(l) next := l.peek() if next != eof && !isSpace(next) && !isEndOfLine(next) && next != ';' && next != ')' && next != ',' && next != '+' && next != '[' && next != ']' && next != '(' && next != '.' { l.errorf("Unrecognized character in action: %#U", next) return nil } l.emit(token.Variable) return lexStart case r == '=': if l.peek() == '=' { l.next() l.emit(token.Equal) } else { l.emit(token.Assign) } return lexStart case r == '!': if l.peek() == '=' { l.next() l.emit(token.NotEqual) } else { l.emit(token.Arg) } return lexStart case r == '<': if l.peek() == '=' { l.next() l.emit(token.AssignCmd) } else { l.emit(token.Lt) } return lexStart case r == '{': l.addSemicolon = false l.emit(token.LBrace) return lexStart case r == '}': l.emit(token.RBrace) l.addSemicolon = false return lexStart case r == '[': l.emit(token.LBrack) return lexStart case r == ']': l.emit(token.RBrack) return lexStart case r == '(': l.openParens++ l.emit(token.LParen) l.addSemicolon = false return lexStart case r == ')': l.openParens-- l.emit(token.RParen) l.addSemicolon = true return lexStart case r == ',': l.emit(token.Comma) return lexStart case r == '.': dotLine, dotColumn := l.line, l.column next := l.peek() if next == '.' { l.next() next = l.peek() if next == '.' { l.next() l.emitVal(token.Dotdotdot, "...", dotLine, dotColumn) return lexStart } } absorbArgument(l) l.emit(token.Arg) if next == eof && l.openParens > 0 { l.addSemicolon = false } else { l.addSemicolon = true } return lexStart case isIdentifier(r): // nash literals are lowercase absorbIdentifier(l) next := l.peek() if isEndOfLine(next) || isSpace(next) || next == '=' || next == '(' || next == ')' || next == ',' || next == '[' || next == eof { lit := scanIdentifier(l) if len(lit) > 1 && r >= 'a' && r <= 'z' { l.emit(token.Lookup(lit)) } else { l.emit(token.Ident) } } else if next == '.' { // because of shell idiosyncrasies I've to replicate // almost same dotdotdot lex here... ident := l.input[l.start:l.pos] identLine, identCol := l.lineStart, l.columnStart dotLine, dotColumn := l.line, l.column l.next() next = l.peek() if next == '.' { l.next() next = l.peek() if next == '.' { l.next() l.emitVal(token.Ident, ident, identLine, identCol) l.emitVal(token.Dotdotdot, "...", dotLine, dotColumn) return lexStart } } absorbArgument(l) l.emit(token.Arg) } else { absorbArgument(l) l.emit(token.Arg) } if next == eof && l.openParens > 0 { l.addSemicolon = false } else { l.addSemicolon = true } return lexStart case isArgument(r): absorbArgument(l) l.emit(token.Arg) l.addSemicolon = true return lexStart } return l.errorf("Unrecognized character in action: %#U", r) } func absorbIdentifier(l *Lexer) { for { r := l.next() if isIdentifier(r) { continue // absorb } break } l.backup() // pos is now ahead of the alphanum } func absorbArgument(l *Lexer) { for { r := l.next() if isArgument(r) { continue // absorb } break } l.backup() // pos is now ahead of the alphanum } func scanIdentifier(l *Lexer) string { absorbIdentifier(l) return l.input[l.start:l.pos] } func lexQuote(l *Lexer) stateFn { var data []rune data = make([]rune, 0, 256) for { r := l.next() if r != '"' && r != eof { if r == '\\' { r = l.next() switch r { case 'n': data = append(data, '\n') case 't': data = append(data, '\t') case '\\': data = append(data, '\\') case '"': data = append(data, '"') case 'x', 'u', 'U': return l.errorf("Escape types 'x', 'u' and 'U' aren't implemented yet") case '0', '1', '2', '3', '4', '5', '6', '7': x := r - '0' for i := 2; i > 0; i-- { r = l.next() if r >= '0' && r <= '7' { x = x*8 + r - '0' continue } return l.errorf("non-octal character in escape sequence: %c", r) } if x > 255 { return l.errorf("octal escape value > 255: %d", x) } data = append(data, x) } } else { data = append(data, r) } continue } if r == eof { return l.errorf("Quoted string not finished: %s", l.input[l.start:]) } l.emitVal(token.String, string(data), l.lineStart, l.columnStart) l.ignore() // ignores last quote break } return lexStart } func lexComment(l *Lexer) stateFn { for { r := l.next() if isEndOfLine(r) { l.backup() l.emit(token.Comment) break } if r == eof { l.backup() l.emit(token.Comment) break } } return lexStart } func lexSpace(l *Lexer) stateFn { ignoreSpaces(l) return lexStart } func ignoreSpaces(l *Lexer) { for { r := l.next() if !isSpace(r) { break } } l.backup() l.ignore() } // isSpace reports whether r is a space character. func isSpace(r rune) bool { return r == ' ' || r == '\t' } func isArgument(r rune) bool { isId := isAlpha(r) return isId || (r != eof && !isEndOfLine(r) && !isSpace(r) && r != '$' && r != '{' && r != '}' && r != '(' && r != ']' && r != '[' && r != ')' && r != '>' && r != '"' && r != ',' && r != ';' && r != '|') } func isIdentifier(r rune) bool { return isAlpha(r) || r == '_' } // isIdentifier reports whether r is a valid identifier func isAlpha(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) } // isEndOfLine reports whether r is an end-of-line character. func isEndOfLine(r rune) bool { return r == '\r' || r == '\n' } ================================================ FILE: scanner/lex_regression_test.go ================================================ package scanner import ( "testing" "github.com/madlambda/nash/token" ) func TestLexerIssue34(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "cat"}, {typ: token.Arg, val: "/etc/passwd"}, {typ: token.Gt, val: ">"}, {typ: token.Arg, val: "/dev/null"}, {typ: token.Ident, val: "echo"}, {typ: token.String, val: "hello world"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test-issue-34", `cat /etc/passwd > /dev/null echo "hello world"`, expected, t) } func TestLexerIssue21(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.Variable, val: "$outFname"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test redirection variable", `cmd > $outFname`, expected, t) } func TestLexerIssue22(t *testing.T) { expected := []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "gocd"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "path"}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.If, val: "if"}, {typ: token.Variable, val: "$path"}, {typ: token.Equal, val: "=="}, {typ: token.String, val: ""}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "cd"}, {typ: token.Variable, val: "$GOPATH"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.Else, val: "else"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "cd"}, {typ: token.Variable, val: "$GOPATH"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: "/src/"}, {typ: token.Plus, val: "+"}, {typ: token.Variable, val: "$path"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test issue 22", `fn gocd(path) { if $path == "" { cd $GOPATH } else { cd $GOPATH + "/src/" + $path } }`, expected, t) } func TestLexerIssue19(t *testing.T) { line := `version = "4.5.6" canonName <= echo -n $version | sed "s/\\.//g"` expected := []Token{ {typ: token.Ident, val: "version"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "4.5.6"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "canonName"}, {typ: token.AssignCmd, val: "<="}, {typ: token.Ident, val: "echo"}, {typ: token.Arg, val: "-n"}, {typ: token.Variable, val: "$version"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "sed"}, {typ: token.String, val: "s/\\.//g"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}} testTable("test issue 19", line, expected, t) } func TestLexerIssue38(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "cd"}, {typ: token.LParen, val: "("}, {typ: token.Variable, val: "$GOPATH"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: "/src/"}, {typ: token.Plus, val: "+"}, {typ: token.Variable, val: "$path"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test issue38", `cd($GOPATH + "/src/" + $path)`, expected, t) } func TestLexerIssue43(t *testing.T) { expected := []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "gpull"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "branch"}, {typ: token.AssignCmd, val: "<="}, {typ: token.Ident, val: "git"}, {typ: token.Arg, val: "rev-parse"}, {typ: token.Arg, val: "--abbrev-ref"}, {typ: token.Ident, val: "HEAD"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "xargs"}, {typ: token.Ident, val: "echo"}, {typ: token.Arg, val: "-n"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "git"}, {typ: token.Ident, val: "pull"}, {typ: token.Ident, val: "origin"}, {typ: token.Variable, val: "$branch"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "refreshPrompt"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test issue #41", `fn gpull() { branch <= git rev-parse --abbrev-ref HEAD | xargs echo -n git pull origin $branch refreshPrompt() }`, expected, t) } func TestLexerIssue68(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "cat"}, {typ: token.Ident, val: "PKGBUILD"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "sed"}, {typ: token.String, val: "s#\\\\$pkgdir#/home/i4k/alt#g"}, {typ: token.Gt, val: ">"}, {typ: token.Ident, val: "PKGBUILD2"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test issue #68", `cat PKGBUILD | sed "s#\\\\$pkgdir#/home/i4k/alt#g" > PKGBUILD2`, expected, t) } func TestLexerIssue85(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "a"}, {typ: token.AssignCmd, val: "<="}, {typ: token.Arg, val: "-echo"}, {typ: token.Ident, val: "hello"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test issue 85", `a <= -echo hello`, expected, t) } func TestLexerIssue69(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "a"}, {typ: token.Assign, val: "="}, {typ: token.LParen, val: "("}, {typ: token.Variable, val: "$a"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: "b"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test69", `a = ($a + "b")`, expected, t) } func TestLexerIssue127(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "rm"}, {typ: token.Arg, val: "-rf"}, {typ: token.Illegal, val: "test127:1:12: Unrecognized character in action: U+002F '/'"}, {typ: token.EOF}, } testTable("test127", `rm -rf $HOME/.vim`, expected, t) } ================================================ FILE: scanner/lex_test.go ================================================ package scanner import ( "fmt" "strconv" "github.com/madlambda/nash/token" ) import "testing" func testTable(name, content string, expected []Token, t *testing.T) { l := Lex(name, content) if l == nil { t.Errorf("Failed to initialize lexer") return } if l.Tokens == nil { t.Errorf("Failed to initialize lexer") return } result := make([]Token, 0, 1024) for i := range l.Tokens { result = append(result, i) } if len(result) != len(expected) { t.Errorf("Failed to parse commands, length differs %d != %d", len(result), len(expected)) fmt.Printf("Parsing content: %s\n", content) for _, res := range result { fmt.Printf("parsed: %+v\n", res) } fmt.Printf("\n") for _, exp := range expected { fmt.Printf("expect: %+v\n", exp) } return } for i := 0; i < len(expected); i++ { if expected[i].typ != result[i].typ { t.Errorf("'%s (%s)' != '%s (%s)'", expected[i].typ, expected[i].val, result[i].typ, result[i].val) return } if expected[i].val != result[i].val { t.Errorf("Parsing '%s':\n\terror: '%s' != '%s'", content, expected[i].val, result[i].val) return } } } func TestLexerCommandStringArgs(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "echo"}, {typ: token.Ident, val: "hello"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "echo"}, {typ: token.String, val: "hello"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test args", `echo hello echo "hello"`, expected, t) } func TestLexerTokenToString(t *testing.T) { it := Token{ typ: token.EOF, } if it.String() != "EOF" { t.Errorf("Wrong eof string: %s", it.String()) } it = Token{ typ: token.Illegal, val: "some error", } if it.String() != "ERROR: some error" { t.Errorf("wrong error string: %s", it.String()) } it = Token{ typ: token.Ident, val: "echo", } if it.String() != "IDENT" { t.Errorf("wrong command name: %s", it.String()) } it = Token{ typ: token.Ident, val: "echoooooooooooooooooooooooo", } // test if long names are truncated if it.String() != "IDENT" { t.Errorf("wrong command name: %s", it.String()) } } func TestLexerShebangOnly(t *testing.T) { expected := []Token{ { typ: token.Comment, val: "#!/bin/nash", }, { typ: token.EOF, }, } testTable("testShebangonly", "#!/bin/nash\n", expected, t) } func TestLexerSimpleSetEnvAssignment(t *testing.T) { expected := []Token{ {typ: token.SetEnv, val: "setenv"}, {typ: token.Ident, val: "name"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSet", `setenv name`, expected, t) } func TestLexerSimpleAssignment(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "test"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "value"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testAssignment", `test="value"`, expected, t) testTable("testAssignment spacy", `test = "value"`, expected, t) testTable("testAssignment spacy", `test ="value"`, expected, t) testTable("testAssignment spacy", `test= "value"`, expected, t) testTable("testAssignment spacy", `test ="value"`, expected, t) testTable("testAssignment spacy", `test ="value"`, expected, t) testTable("testAssignment spacy", `test = "value"`, expected, t) expected = []Token{ {typ: token.Ident, val: "test"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "value"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "other"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "other"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test multiple separate assignments", ` test="value" other="other"`, expected, t) expected = []Token{ {typ: token.Ident, val: "test"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "value"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "other"}, {typ: token.Assign, val: "="}, {typ: token.Variable, val: "$test"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "echo"}, {typ: token.Variable, val: "$other"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test multiple separate assignments", ` test="value" other=$test echo $other`, expected, t) expected = []Token{ {typ: token.Ident, val: "STALI_SRC"}, {typ: token.Assign, val: "="}, {typ: token.Variable, val: "$PWD"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: "/src"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test underscore", `STALI_SRC = $PWD + "/src"`, expected, t) expected = []Token{ {typ: token.Ident, val: "PROMPT"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "("}, {typ: token.Plus, val: "+"}, {typ: token.Variable, val: "$path"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: ")"}, {typ: token.Plus, val: "+"}, {typ: token.Variable, val: "$PROMPT"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test concat with parenthesis", `PROMPT = "("+$path+")"+$PROMPT`, expected, t) expected = []Token{ {typ: token.Ident, val: "a"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "0"}, {typ: token.RBrack, val: "]"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "test"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test index assignment", `a[0] = "test"`, expected, t) } func TestLexerListAssignment(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "test"}, {typ: token.Assign, val: "="}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "plan9"}, {typ: token.Ident, val: "from"}, {typ: token.Ident, val: "bell"}, {typ: token.Ident, val: "labs"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testListAssignment", "test=( plan9 from bell labs )", expected, t) testTable("testListAssignment no space", "test=(plan9 from bell labs)", expected, t) testTable("testListAssignment multiline", `test = ( plan9 from bell labs )`, expected, t) expected = []Token{ {typ: token.Ident, val: "test"}, {typ: token.Assign, val: "="}, {typ: token.LParen, val: "("}, {typ: token.String, val: "plan9"}, {typ: token.Ident, val: "from"}, {typ: token.String, val: "bell"}, {typ: token.Ident, val: "labs"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testListAssignment mixed args", `test=( "plan9" from "bell" labs )`, expected, t) testTable("testListAssignment mixed args", `test=("plan9" from "bell" labs)`, expected, t) testTable("testListAssignment mixed args", `test = ( "plan9" from "bell" labs )`, expected, t) expected = []Token{ {typ: token.Ident, val: "test"}, {typ: token.Assign, val: "="}, {typ: token.LParen, val: "("}, {typ: token.Variable, val: "$plan9"}, {typ: token.Ident, val: "from"}, {typ: token.Variable, val: "$bell"}, {typ: token.Ident, val: "labs"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testListAssignment mixed args", `test=( $plan9 from $bell labs )`, expected, t) testTable("testListAssignment mixed args", `test=($plan9 from $bell labs)`, expected, t) testTable("testListAssignment mixed args", `test = ( $plan9 from $bell labs )`, expected, t) } func TestLexerListOfLists(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "l"}, {typ: token.Assign, val: "="}, {typ: token.LParen, val: "("}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testlistoflists", `l = (())`, expected, t) testTable("testlistoflists", `l = ( () )`, expected, t) expected = []Token{ {typ: token.Ident, val: "l"}, {typ: token.Assign, val: "="}, {typ: token.LParen, val: "("}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "plan9"}, {typ: token.Ident, val: "from"}, {typ: token.Ident, val: "bell"}, {typ: token.Ident, val: "labs"}, {typ: token.RParen, val: ")"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "linux"}, {typ: token.RParen, val: ")"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testlistoflists", `l = ((plan9 from bell labs) (linux))`, expected, t) testTable("testlistoflists", `l = ( (plan9 from bell labs) (linux) )`, expected, t) } func TestLexerInvalidAssignments(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "test"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "value"}, {typ: token.Ident, val: "other"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testInvalidAssignments", `test="value" other`, expected, t) } func TestLexerSimpleCommand(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "echo"}, {typ: token.String, val: "hello world"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `echo "hello world"`, expected, t) expected = []Token{ {typ: token.Ident, val: "echo"}, {typ: token.Arg, val: "rootfs-x86_64"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `echo rootfs-x86_64`, expected, t) expected = []Token{ {typ: token.Ident, val: "git"}, {typ: token.Ident, val: "clone"}, {typ: token.Arg, val: "--depth=1"}, {typ: token.Arg, val: "http://git.sta.li/toolchain"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `git clone --depth=1 http://git.sta.li/toolchain`, expected, t) expected = []Token{ {typ: token.Ident, val: "ls"}, {typ: token.Variable, val: "$GOPATH"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `ls $GOPATH`, expected, t) expected = []Token{ {typ: token.Ident, val: "ls"}, {typ: token.Variable, val: "$GOPATH"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: "/src/github.com"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `ls $GOPATH+"/src/github.com"`, expected, t) expected = []Token{ {typ: token.Ident, val: "ls"}, {typ: token.String, val: "/src/github.com"}, {typ: token.Plus, val: "+"}, {typ: token.Variable, val: "$GOPATH"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `ls "/src/github.com"+$GOPATH`, expected, t) expected = []Token{ {typ: token.Ident, val: "ls"}, {typ: token.String, val: "/home/user"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: "/.gvm/pkgsets/global/src"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `ls "/home/user" + "/.gvm/pkgsets/global/src"`, expected, t) expected = []Token{ {typ: token.Arg, val: "./rkt"}, {typ: token.Semicolon, val: ";"}, // automatic semicolon insertion {typ: token.EOF}, } testTable("test local command", "./rkt", expected, t) } func TestLexerPipe(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "ls"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "wc"}, {typ: token.Arg, val: "-l"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testPipe", `ls | wc -l`, expected, t) expected = []Token{ {typ: token.Ident, val: "ls"}, {typ: token.Arg, val: "-l"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "wc"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "awk"}, {typ: token.String, val: "{print $1}"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testPipe", `ls -l | wc | awk "{print $1}"`, expected, t) expected = []Token{ {typ: token.Ident, val: "go"}, {typ: token.Ident, val: "tool"}, {typ: token.Ident, val: "vet"}, {typ: token.Arg, val: "-h"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.Assign, val: "="}, {typ: token.Number, val: "1"}, {typ: token.RBrack, val: "]"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "grep"}, {typ: token.Ident, val: "log"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testPipe with redirection", `go tool vet -h >[2=1] | grep log`, expected, t) expected = []Token{ {typ: token.Ident, val: "go"}, {typ: token.Ident, val: "tool"}, {typ: token.Ident, val: "vet"}, {typ: token.Arg, val: "-h"}, {typ: token.Gt, val: ">"}, {typ: token.Arg, val: "out.log"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "grep"}, {typ: token.Ident, val: "log"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testPipe with redirection", `go tool vet -h > out.log | grep log`, expected, t) } func TestPipeFunctions(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "echo"}, {typ: token.String, val: "some thing"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "replace"}, {typ: token.LParen, val: "("}, {typ: token.String, val: " "}, {typ: token.Comma, val: ","}, {typ: token.String, val: "|"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test pipe with function", `echo "some thing" | replace(" ", "|")`, expected, t) } func TestLexerUnquoteArg(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "echo"}, {typ: token.Ident, val: "hello"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `echo hello`, expected, t) expected = []Token{ {typ: token.Ident, val: "echo"}, {typ: token.Arg, val: "hello-world"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `echo hello-world`, expected, t) expected = []Token{ {typ: token.Ident, val: "echo"}, {typ: token.Comment, val: "#hello-world"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testSimpleCommand", `echo #hello-world`, expected, t) } func TestLexerDashedCommand(t *testing.T) { expected := []Token{ {typ: token.Arg, val: "google-chrome"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testDashedCommand", `google-chrome`, expected, t) } func TestLexerPathCommand(t *testing.T) { expected := []Token{ {typ: token.Arg, val: "/bin/echo"}, {typ: token.String, val: "hello world"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testPathCommand", `/bin/echo "hello world"`, expected, t) } func TestLexerInvalidBlock(t *testing.T) { expected := []Token{ {typ: token.LBrace, val: "{"}, {typ: token.EOF}, } testTable("testInvalidBlock", "{", expected, t) } func TestLexerQuotedStringNotFinished(t *testing.T) { expected := []Token{ { typ: token.Ident, val: "echo", }, { typ: token.Illegal, val: "testQuotedstringnotfinished:1:17: Quoted string not finished: hello world", }, { typ: token.EOF, }, } testTable("testQuotedstringnotfinished", "echo \"hello world", expected, t) } func TestLexerVariousCommands(t *testing.T) { content := ` echo "hello world" mount -t proc proc /proc ` expected := []Token{ {typ: token.Ident, val: "echo"}, {typ: token.String, val: "hello world"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "mount"}, {typ: token.Arg, val: "-t"}, {typ: token.Ident, val: "proc"}, {typ: token.Ident, val: "proc"}, {typ: token.Arg, val: "/proc"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testVariouscommands", content, expected, t) } func TestLexerRfork(t *testing.T) { expected := []Token{ {typ: token.Rfork, val: "rfork"}, {typ: token.Ident, val: "u"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testRfork", "rfork u\n", expected, t) expected = []Token{ {typ: token.Rfork, val: "rfork"}, {typ: token.Ident, val: "usnm"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "echo"}, {typ: token.String, val: "inside namespace :)"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("testRforkWithBlock", ` rfork usnm { echo "inside namespace :)" } `, expected, t) } func TestLexerSomethingIdontcareanymore(t *testing.T) { // maybe oneliner rfork isnt a good idea expected := []Token{ {typ: token.Rfork, val: "rfork"}, {typ: token.Ident, val: "u"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "ls"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test whatever", "rfork u { ls }", expected, t) } func TestLexerBuiltinCd(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "cd"}, {typ: token.String, val: "some place"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testBuiltinCd", `cd "some place"`, expected, t) expected = []Token{ {typ: token.Ident, val: "cd"}, {typ: token.Arg, val: "/proc"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testBuiltinCdNoQuote", `cd /proc`, expected, t) expected = []Token{ {typ: token.Ident, val: "cd"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testBuiltincd home", `cd`, expected, t) expected = []Token{ {typ: token.Ident, val: "HOME"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "/"}, {typ: token.Semicolon, val: ";"}, {typ: token.SetEnv, val: "setenv"}, {typ: token.Ident, val: "HOME"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "cd"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "pwd"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("testBuiltin cd bug", ` HOME="/" setenv HOME cd pwd `, expected, t) expected = []Token{ {typ: token.Ident, val: "cd"}, {typ: token.Variable, val: "$GOPATH"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test builtin cd into variable", `cd $GOPATH`, expected, t) expected = []Token{ {typ: token.Ident, val: "cd"}, {typ: token.Variable, val: "$GOPATH"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: "/src/github.com"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test cd with concat", `cd $GOPATH+"/src/github.com"`, expected, t) } func TestLexerRedirectSimple(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.Arg, val: "file.out"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test simple redirect", "cmd > file.out", expected, t) expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.String, val: "tcp://localhost:8888"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test simple redirect", `cmd > "tcp://localhost:8888"`, expected, t) expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.String, val: "udp://localhost:8888"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test simple redirect", `cmd > "udp://localhost:8888"`, expected, t) expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.String, val: "unix:///tmp/sock.txt"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test simple redirect", `cmd > "unix:///tmp/sock.txt"`, expected, t) } func TestLexerRedirectMap(t *testing.T) { // Suppress stderr output expected := []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.Assign, val: "="}, {typ: token.RBrack, val: "]"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test suppress stderr", "cmd >[2=]", expected, t) // points stderr to stdout expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.Assign, val: "="}, {typ: token.Number, val: "1"}, {typ: token.RBrack, val: "]"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test stderr=stdout", "cmd >[2=1]", expected, t) } func TestLexerRedirectMapToLocation(t *testing.T) { // Suppress stderr output expected := []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.Assign, val: "="}, {typ: token.RBrack, val: "]"}, {typ: token.Arg, val: "file.out"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test suppress stderr", "cmd >[2=] file.out", expected, t) // points stderr to stdout expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.Assign, val: "="}, {typ: token.Number, val: "1"}, {typ: token.RBrack, val: "]"}, {typ: token.Arg, val: "file.out"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test stderr=stdout", "cmd >[2=1] file.out", expected, t) expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.RBrack, val: "]"}, {typ: token.Arg, val: "/var/log/service.log"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test suppress stderr", `cmd >[2] /var/log/service.log`, expected, t) } func TestLexerRedirectMultipleMaps(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "1"}, {typ: token.RBrack, val: "]"}, {typ: token.Arg, val: "file.out"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.RBrack, val: "]"}, {typ: token.Arg, val: "file.err"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test suppress stderr", `cmd >[1] file.out >[2] file.err`, expected, t) expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.Assign, val: "="}, {typ: token.Number, val: "1"}, {typ: token.RBrack, val: "]"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "1"}, {typ: token.RBrack, val: "]"}, {typ: token.Arg, val: "/var/log/service.log"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test suppress stderr", `cmd >[2=1] >[1] /var/log/service.log`, expected, t) expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "1"}, {typ: token.Assign, val: "="}, {typ: token.Number, val: "2"}, {typ: token.RBrack, val: "]"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "2"}, {typ: token.Assign, val: "="}, {typ: token.RBrack, val: "]"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test suppress stderr", `cmd >[1=2] >[2=]`, expected, t) } func TestLexerImport(t *testing.T) { expected := []Token{ {typ: token.Import, val: "import"}, {typ: token.Arg, val: "env.sh"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test import", `import env.sh`, expected, t) } func TestLexerSimpleIf(t *testing.T) { expected := []Token{ {typ: token.If, val: "if"}, {typ: token.String, val: "test"}, {typ: token.Equal, val: "=="}, {typ: token.String, val: "other"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "rm"}, {typ: token.Arg, val: "-rf"}, {typ: token.Arg, val: "/"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test simple if", `if "test" == "other" { rm -rf / }`, expected, t) expected = []Token{ {typ: token.If, val: "if"}, {typ: token.String, val: "test"}, {typ: token.NotEqual, val: "!="}, {typ: token.Variable, val: "$test"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "rm"}, {typ: token.Arg, val: "-rf"}, {typ: token.Arg, val: "/"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test simple if", `if "test" != $test { rm -rf / }`, expected, t) testTable("test simple if", ` if "test" != $test { rm -rf / }`, expected, t) } func TestLexerIfWithConcat(t *testing.T) { expected := []Token{ {typ: token.If, val: "if"}, {typ: token.Variable, val: "$test"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: "001"}, {typ: token.NotEqual, val: "!="}, {typ: token.String, val: "value001"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "rm"}, {typ: token.Arg, val: "-rf"}, {typ: token.Arg, val: "/"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test if concat", `if $test + "001" != "value001" { rm -rf / }`, expected, t) } func TestLexerIfWithFuncInvocation(t *testing.T) { expected := []Token{ {typ: token.If, val: "if"}, {typ: token.Ident, val: "test"}, {typ: token.LParen, val: "("}, {typ: token.String, val: "some val"}, {typ: token.RParen, val: ")"}, {typ: token.NotEqual, val: "!="}, {typ: token.String, val: "value001"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "rm"}, {typ: token.Arg, val: "-rf"}, {typ: token.Arg, val: "/"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test if concat", `if test("some val") != "value001" { rm -rf / }`, expected, t) } func TestLexerIfElse(t *testing.T) { expected := []Token{ {typ: token.If, val: "if"}, {typ: token.String, val: "test"}, {typ: token.Equal, val: "=="}, {typ: token.String, val: "other"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "rm"}, {typ: token.Arg, val: "-rf"}, {typ: token.Arg, val: "/"}, {typ: token.RBrace, val: "}"}, {typ: token.Else, val: "else"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "pwd"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test simple if", `if "test" == "other" { rm -rf / } else { pwd }`, expected, t) } func TestLexerIfElseIf(t *testing.T) { expected := []Token{ {typ: token.If, val: "if"}, {typ: token.String, val: "test"}, {typ: token.Equal, val: "=="}, {typ: token.String, val: "other"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "rm"}, {typ: token.Arg, val: "-rf"}, {typ: token.Arg, val: "/"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.Else, val: "else"}, {typ: token.If, val: "if"}, {typ: token.String, val: "test"}, {typ: token.Equal, val: "=="}, {typ: token.Variable, val: "$var"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "pwd"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.Else, val: "else"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "exit"}, {typ: token.Number, val: "1"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test simple if", ` if "test" == "other" { rm -rf / } else if "test" == $var { pwd } else { exit 1 }`, expected, t) } func TestLexerFnBasic(t *testing.T) { expected := []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "build"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test empty fn", `fn build() {}`, expected, t) // lambda expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test empty fn", `fn () {}`, expected, t) // IIFE expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.RBrace, val: "}"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test empty fn", `fn () {}()`, expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "build"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "image"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "debug"}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test empty fn with args", `fn build(image, debug) {}`, expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "build"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "image"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "debug"}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "ls"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "tar"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test empty fn with args and body", `fn build(image, debug) { ls tar }`, expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "cd"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "path"}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "cd"}, {typ: token.Variable, val: "$path"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "PROMPT"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "("}, {typ: token.Plus, val: "+"}, {typ: token.Variable, val: "$path"}, {typ: token.Plus, val: "+"}, {typ: token.String, val: ")"}, {typ: token.Plus, val: "+"}, {typ: token.Variable, val: "$PROMPT"}, {typ: token.Semicolon, val: ";"}, {typ: token.SetEnv, val: "setenv"}, {typ: token.Ident, val: "PROMPT"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test cd fn with PROMPT update", `fn cd(path) { cd $path PROMPT="(" + $path + ")"+$PROMPT setenv PROMPT }`, expected, t) } func TestLexerFuncall(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "build"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test fn invocation", `build()`, expected, t) expected = []Token{ {typ: token.Ident, val: "build"}, {typ: token.LParen, val: "("}, {typ: token.String, val: "ubuntu"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test fn invocation", `build("ubuntu")`, expected, t) expected = []Token{ {typ: token.Ident, val: "build"}, {typ: token.LParen, val: "("}, {typ: token.String, val: "ubuntu"}, {typ: token.Comma, val: ","}, {typ: token.Variable, val: "$debug"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test fn invocation", `build("ubuntu", $debug)`, expected, t) expected = []Token{ {typ: token.Ident, val: "build"}, {typ: token.LParen, val: "("}, {typ: token.Variable, val: "$debug"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test fn invocation", `build($debug)`, expected, t) expected = []Token{ {typ: token.Ident, val: "a"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "b"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test fn composition", `a(b())`, expected, t) expected = []Token{ {typ: token.Ident, val: "ids_luns"}, {typ: token.AssignCmd, val: "<="}, {typ: token.Ident, val: "append"}, {typ: token.LParen, val: "("}, {typ: token.Variable, val: "$ids_luns"}, {typ: token.Comma, val: ","}, {typ: token.LParen, val: "("}, {typ: token.Variable, val: "$id"}, {typ: token.Variable, val: "$lun"}, {typ: token.RParen, val: ")"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test katcipis bad mood", `ids_luns <= append($ids_luns, ($id $lun))`, expected, t) } func TestLexerAssignCmdOut(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "ipaddr"}, {typ: token.AssignCmd, val: "<="}, {typ: token.Ident, val: "someprogram"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test assignCmdOut", `ipaddr <= someprogram`, expected, t) } func TestLexerMultipleAssignCmdOut(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "ret"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "status"}, {typ: token.AssignCmd, val: "<="}, {typ: token.Ident, val: "someprogram"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test multiple return assignCmdOut", `ret, status <= someprogram`, expected, t) expected = []Token{ {typ: token.Ident, val: "ret"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "_"}, {typ: token.AssignCmd, val: "<="}, {typ: token.Ident, val: "someprogram"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test multiple return assignCmdOut", `ret, _ <= someprogram`, expected, t) expected = []Token{ {typ: token.Ident, val: "ret"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "obj"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "err"}, {typ: token.AssignCmd, val: "<="}, {typ: token.Ident, val: "somefn"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test multiple return assignCmdOut", `ret, obj, err <= somefn()`, expected, t) } func TestMultipleAssignments(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "a"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "b"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "1"}, {typ: token.Comma, val: ","}, {typ: token.String, val: "2"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test multiple assign", `a, b = "1", "2"`, expected, t) } func TestLexerBindFn(t *testing.T) { expected := []Token{ {typ: token.BindFn, val: "bindfn"}, {typ: token.Ident, val: "cd"}, {typ: token.Ident, val: "cd2"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test bindfn", `bindfn cd cd2`, expected, t) } func TestLexerRedirectionNetwork(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "echo"}, {typ: token.String, val: "hello world"}, {typ: token.Gt, val: ">"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "1"}, {typ: token.RBrack, val: "]"}, {typ: token.String, val: "tcp://localhost:6667"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test redirection network", `echo "hello world" >[1] "tcp://localhost:6667"`, expected, t) } func TestLexerDump(t *testing.T) { expected := []Token{ {typ: token.Dump, val: "dump"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test dump", `dump`, expected, t) expected = []Token{ {typ: token.Dump, val: "dump"}, {typ: token.Ident, val: "out"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test dump", `dump out`, expected, t) expected = []Token{ {typ: token.Dump, val: "dump"}, {typ: token.Variable, val: "$out"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test dump", `dump $out`, expected, t) } func TestLexerReturn(t *testing.T) { expected := []Token{ {typ: token.Return, val: "return"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test return", "return", expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "test"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Return, val: "return"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test return", "fn test() { return }", expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "test"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Return, val: "return"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test return", `fn test() { return }`, expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "test"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Return, val: "return"}, {typ: token.String, val: "some value"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test return", `fn test() { return "some value"}`, expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "test"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Return, val: "return"}, {typ: token.String, val: "some value"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test return", `fn test() { return "some value" }`, expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "test"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "value"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "some value"}, {typ: token.Semicolon, val: ";"}, {typ: token.Return, val: "return"}, {typ: token.Variable, val: "$value"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test return", `fn test() { value = "some value" return $value }`, expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "test"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Return, val: "return"}, {typ: token.LParen, val: "("}, {typ: token.String, val: "test"}, {typ: token.String, val: "test2"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test return", `fn test() { return ("test" "test2") }`, expected, t) expected = []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "test"}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Return, val: "return"}, {typ: token.Variable, val: "$PWD"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test return", `fn test() { return $PWD }`, expected, t) } func TestLexerFor(t *testing.T) { expected := []Token{ {typ: token.For, val: "for"}, {typ: token.LBrace, val: "{"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test inf loop", `for {}`, expected, t) expected = []Token{ {typ: token.For, val: "for"}, {typ: token.Ident, val: "f"}, {typ: token.Ident, val: "in"}, {typ: token.Variable, val: "$files"}, {typ: token.LBrace, val: "{"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test inf loop", `for f in $files {}`, expected, t) expected = []Token{ {typ: token.For, val: "for"}, {typ: token.Ident, val: "f"}, {typ: token.Ident, val: "in"}, {typ: token.Ident, val: "getfiles"}, {typ: token.LParen, val: "("}, {typ: token.String, val: "/"}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test inf loop", `for f in getfiles("/") {}`, expected, t) expected = []Token{ {typ: token.For, val: "for"}, {typ: token.Ident, val: "f"}, {typ: token.Ident, val: "in"}, {typ: token.LParen, val: "("}, {typ: token.Number, val: "1"}, {typ: token.Number, val: "2"}, {typ: token.Number, val: "3"}, {typ: token.Number, val: "4"}, {typ: token.Number, val: "5"}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test inf loop", `for f in (1 2 3 4 5) {}`, expected, t) } func TestLexerFnAsFirstClass(t *testing.T) { expected := []Token{ {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "printer"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "val"}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "echo"}, {typ: token.Arg, val: "-n"}, {typ: token.Variable, val: "$val"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.Fn, val: "fn"}, {typ: token.Ident, val: "success"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "print"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "val"}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Variable, val: "$print"}, {typ: token.LParen, val: "("}, {typ: token.String, val: "[SUCCESS] "}, {typ: token.Plus, val: "+"}, {typ: token.Variable, val: "$val"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.Ident, val: "success"}, {typ: token.LParen, val: "("}, {typ: token.Variable, val: "$printer"}, {typ: token.Comma, val: ","}, {typ: token.String, val: "Command executed!"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test fn as first class", ` fn printer(val) { echo -n $val } fn success(print, val) { $print("[SUCCESS] " + $val) } success($printer, "Command executed!") `, expected, t) } func TestLexerListIndexing(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Assign, val: "="}, {typ: token.Variable, val: "$commands"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "0"}, {typ: token.RBrack, val: "]"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } for i := 0; i < 1000; i++ { expected[4] = Token{ typ: token.Number, val: strconv.Itoa(i), } testTable("test variable indexing", `cmd = $commands[`+strconv.Itoa(i)+`]`, expected, t) } expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Assign, val: "="}, {typ: token.Variable, val: "$commands"}, {typ: token.LBrack, val: "["}, {typ: token.Arg, val: "a"}, {typ: token.RBrack, val: "]"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test invalid number", `cmd = $commands[a]`, expected, t) expected = []Token{ {typ: token.Ident, val: "cmd"}, {typ: token.Assign, val: "="}, {typ: token.Variable, val: "$commands"}, {typ: token.LBrack, val: "["}, {typ: token.RBrack, val: "]"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test invalid number", `cmd = $commands[]`, expected, t) expected = []Token{ {typ: token.Ident, val: "echo"}, {typ: token.Ident, val: "test"}, {typ: token.Variable, val: "$names"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "666"}, {typ: token.RBrack, val: "]"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test variable index on commands", `echo test $names[666]`, expected, t) expected = []Token{ {typ: token.If, val: "if"}, {typ: token.Variable, val: "$crazies"}, {typ: token.LBrack, val: "["}, {typ: token.Number, val: "0"}, {typ: token.RBrack, val: "]"}, {typ: token.Equal, val: "=="}, {typ: token.String, val: "patito"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "echo"}, {typ: token.String, val: ":D"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test if with indexing", `if $crazies[0] == "patito" { echo ":D" }`, expected, t) } func TestLexerMultilineCmdExecution(t *testing.T) { expected := []Token{ {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("multiline", `()`, expected, t) expected = []Token{ {typ: token.LParen, val: "("}, {typ: token.Ident, val: "echo"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("multiline", `(echo)`, expected, t) expected = []Token{ {typ: token.LParen, val: "("}, {typ: token.Ident, val: "echo"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "BBBBB"}, {typ: token.Ident, val: "BBBBB"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test multiline cmd execution", `(echo AAAAA AAAAA AAAAA AAAAA AAAAA AAAAA BBBBB BBBBB)`, expected, t) } func TestLexerMultilineCmdAssign(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "some"}, {typ: token.AssignCmd, val: "<="}, {typ: token.LParen, val: "("}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("multiline", `some <= ()`, expected, t) expected = []Token{ {typ: token.Ident, val: "some"}, {typ: token.AssignCmd, val: "<="}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "echo"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("multiline", `some <= (echo)`, expected, t) expected = []Token{ {typ: token.Ident, val: "some"}, {typ: token.AssignCmd, val: "<="}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "echo"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "AAAAA"}, {typ: token.Ident, val: "BBBBB"}, {typ: token.Ident, val: "BBBBB"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("multiline", `some <= (echo AAAAA AAAAA AAAAA AAAAA AAAAA AAAAA BBBBB BBBBB)`, expected, t) testTable("multiline", `some <= ( echo AAAAA AAAAA AAAAA AAAAA AAAAA AAAAA BBBBB BBBBB )`, expected, t) } func TestLexerCommandDelimiter(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "echo"}, {typ: token.String, val: "hello"}, {typ: token.Semicolon, val: ";"}, {typ: token.Ident, val: "echo"}, {typ: token.String, val: "world"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("semicolons to separate commands", `echo "hello"; echo "world"`, expected, t) } func TestLexerLongAssignment(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "grpid"}, {typ: token.AssignCmd, val: "<="}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "aws"}, {typ: token.Ident, val: "ec2"}, {typ: token.Arg, val: "create-security-group"}, {typ: token.Arg, val: "--group-name"}, {typ: token.Variable, val: "$name"}, {typ: token.Arg, val: "--description"}, {typ: token.Variable, val: "$desc"}, {typ: token.Variable, val: "$vpcarg"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "jq"}, {typ: token.String, val: ".GroupId"}, {typ: token.Pipe, val: "|"}, {typ: token.Ident, val: "xargs"}, {typ: token.Ident, val: "echo"}, {typ: token.Arg, val: "-n"}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test xxx", `grpid <= ( aws ec2 create-security-group --group-name $name --description $desc $vpcarg | jq ".GroupId" | xargs echo -n)`, expected, t) } func TestLexerVarArgs(t *testing.T) { expected := []Token{ {typ: token.Ident, val: "println"}, {typ: token.LParen, val: "("}, {typ: token.Ident, val: "fmt"}, {typ: token.Comma, val: ","}, {typ: token.Ident, val: "args"}, {typ: token.Dotdotdot, val: "..."}, {typ: token.RParen, val: ")"}, {typ: token.LBrace, val: "{"}, {typ: token.Ident, val: "print"}, {typ: token.LParen, val: "("}, {typ: token.Variable, val: "$fmt"}, {typ: token.Comma, val: ","}, {typ: token.Variable, val: "$args"}, {typ: token.Dotdotdot, val: "..."}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.RBrace, val: "}"}, {typ: token.EOF}, } testTable("test var args", `println(fmt, args...) { print($fmt, $args...) }`, expected, t) testTable("test var args", `println(fmt, args ...) { print($fmt, $args...) }`, expected, t) expected = []Token{ {typ: token.Ident, val: "print"}, {typ: token.LParen, val: "("}, {typ: token.String, val: "%s:%s:%s"}, {typ: token.Comma, val: ","}, {typ: token.LParen, val: "("}, {typ: token.String, val: "a"}, {typ: token.String, val: "b"}, {typ: token.String, val: "c"}, {typ: token.RParen, val: ")"}, {typ: token.Dotdotdot, val: "..."}, {typ: token.RParen, val: ")"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test literal expansion", `print("%s:%s:%s", ("a" "b" "c")...)`, expected, t) testTable("test literal expansion", `print("%s:%s:%s", ("a" "b" "c") ...)`, expected, t) } func TestLexerVar(t *testing.T) { expected := []Token{ {typ: token.Var, val: "var"}, {typ: token.Ident, val: "a"}, {typ: token.Assign, val: "="}, {typ: token.String, val: "hello world"}, {typ: token.Semicolon, val: ";"}, {typ: token.EOF}, } testTable("test simple var decl", `var a = "hello world"`, expected, t) } ================================================ FILE: sh/obj.go ================================================ package sh import "fmt" //go:generate stringer -type=objType const ( StringType objType = iota + 1 FnType ListType ) type ( objType int Obj interface { Type() objType String() string } ListObj struct { objType list []Obj } FnObj struct { objType fn FnDef } StrObj struct { objType runes []rune } Collection interface { Len() int Get(index int) (Obj, error) } WriteableCollection interface { Set(index int, val Obj) error } ) func NewCollection(o Obj) (Collection, error) { sizer, ok := o.(Collection) if !ok { return nil, fmt.Errorf( "SizeError: trying to get size from type %s which is not a collection", o.Type(), ) } return sizer, nil } func NewWriteableCollection(o Obj) (WriteableCollection, error) { indexer, ok := o.(WriteableCollection) if !ok { return nil, fmt.Errorf( "IndexError: trying to use a non write/indexable type %s to write on index: ", o.Type(), ) } return indexer, nil } func (o objType) Type() objType { return o } func NewStrObj(val string) *StrObj { return &StrObj{ runes: []rune(val), objType: StringType, } } func (o *StrObj) Str() string { return string(o.runes) } func (o *StrObj) String() string { return o.Str() } func (o *StrObj) Get(index int) (Obj, error) { if index >= o.Len() { return nil, fmt.Errorf( "IndexError: Index[%d] out of range, string size[%d]", index, o.Len(), ) } return NewStrObj(string(o.runes[index])), nil } func (o *StrObj) Len() int { return len(o.runes) } func NewFnObj(val FnDef) *FnObj { return &FnObj{ fn: val, objType: FnType, } } func (o *FnObj) Fn() FnDef { return o.fn } func (o *FnObj) String() string { return fmt.Sprintf("", o.fn.Name()) } func NewListObj(val []Obj) *ListObj { return &ListObj{ list: val, objType: ListType, } } func (o *ListObj) Len() int { return len(o.list) } func (o *ListObj) Set(index int, value Obj) error { if index >= len(o.list) { return fmt.Errorf( "IndexError: Index[%d] out of range, list size[%d]", index, len(o.list), ) } o.list[index] = value return nil } func (o *ListObj) Get(index int) (Obj, error) { if index >= len(o.list) { return nil, fmt.Errorf( "IndexError: Index out of bounds, index[%d] but list size[%d]", index, len(o.list), ) } return o.list[index], nil } func (o *ListObj) List() []Obj { return o.list } func (o *ListObj) String() string { result := "" list := o.List() for i := 0; i < len(list); i++ { l := list[i] result += l.String() if i < len(list)-1 { result += " " } } return result } ================================================ FILE: sh/objtype_string.go ================================================ // Code generated by "stringer -type=objType"; DO NOT EDIT package sh import "fmt" const _objType_name = "StringTypeFnTypeListType" var _objType_index = [...]uint8{0, 10, 16, 24} func (i objType) String() string { i -= 1 if i < 0 || i >= objType(len(_objType_index)-1) { return fmt.Sprintf("objType(%d)", i+1) } return _objType_name[_objType_index[i]:_objType_index[i+1]] } ================================================ FILE: sh/shell.go ================================================ package sh import "io" type ( Runner interface { Start() error Wait() error Results() []Obj SetArgs([]Obj) error SetEnviron([]string) SetStdin(io.Reader) SetStdout(io.Writer) SetStderr(io.Writer) StdoutPipe() (io.ReadCloser, error) Stdin() io.Reader Stdout() io.Writer Stderr() io.Writer } FnArg struct { Name string IsVariadic bool } Fn interface { Name() string ArgNames() []FnArg Runner String() string } FnDef interface { Name() string ArgNames() []FnArg Build() Fn } ) func NewFnArg(name string, isVariadic bool) FnArg { return FnArg{ Name: name, IsVariadic: isVariadic, } } ================================================ FILE: spec.ebnf ================================================ /* Nash program */ program = { statement } . /* Statement */ statement = varDecl | command | fnInv | builtin | comment . /* Variable declaration */ varDecl = assignValue | assignCmdOut . assignValue = identifierList "=" varSpecList . identifierList = identifier [ "," identifierList ] . varSpecList = varSpec [ "," varSpecList ] . varSpec = ( list | string ) . string = stringLit | ( stringConcat { stringConcat } ) . assignCmdOut = identifier "<=" ( command | fnInv ) . /* Command */ command = ( [ "(" ] cmdpart [ ")" ] | pipe ) . cmdpart = [ "-" ] ( cmdname | abscmd ) { argument } { redirect } . cmdname = identifier . abscmd = filename . argument = ( unicode_char { unicode_char } ) | stringLit . pipe = [ "(" ] cmdpart "|" cmdpart [ { "|" cmdpart } ] [ ")" ] . redirect = ( ">" ( filename | uri | variable ) | ">" "[" unicode_digit "]" ( filename | uri | variable ) | ">" "[" unicode_digit "=" ( unicode_digit | identifier ) "]" | ">" "[" unicode_digit "=" "]" ) . /* Builtin */ builtin = importDecl | rforkDecl | ifDecl | forDecl | setenvDecl | fnDecl | bindfn | dump . /* Import statement */ importDecl = "import" ( filename | stringLit ) . /* Rfork scope */ rforkDecl = "rfork" rforkFlags "{" program "}" . rforkFlags = { identifier } . /* If-else-if */ ifDecl = "if" ( variable | string ) comparison ( variable | string ) "{" program "}" [ "else" "{" program "}" ] [ "else" ifDecl ] . /* For loop */ forDecl = "for" [ identifier "in" ( list | variable | fnInv) ] "{" program "}" . /* Function declaration */ fnDecl = "fn" identifier "(" fnArgs ")" "{" program [ returnDecl ] "}" . fnArgs = { fnArg [ "," ] } . fnArg = identifier [ "..." ] . /* return declaration */ returnDecl = "return" [ ( variable | stringLit | list | fnInv ) ] . /* Function invocation */ fnInv = ( variable | identifier ) "(" fnArgValues ")" . fnArgValues = { fnArgValue [ "," ] } . fnArgValue = [ stringLit | stringConcat | list | (variable [ "..." ]) | (list [ "..." ]) fnInv ] . /* Function binding */ bindfn = "bindfn" identifier identifier . /* dump shell state */ dump = "dump" [ filename ] . /* Set environment variable */ setenvDecl = "setenv" ( identifier | varDecl ) . /* Comment */ comment = "#" { unicode_char } . /* Lists */ list = "(" { argument } ")" . letter = unicode_letter | "_" . filename = { [ "/" ] { unicode_letter } } . ipaddr = unicode_digit { unicode_digit } "." unicode_digit { unicode_digit } "." unicode_digit { unicode_digit } "." unicode_digit { unicode_digit } "." . port = unicode_digit { unicode_digit } . networkaddr = ipaddr ":" port . location = filename | networkaddr . schema = "file" | "tcp" | "udp" | "unix" . uri = schema "://" location . identifier = letter { letter | unicode_digit } . variable = "$" identifier . comparison = "==" | "!=" . stringLit = "\"" { unicode_char | newline } "\"" . stringConcat = ( stringLit | variable ) "+" (stringLit | variable ) . /* terminals */ newline = /* the Unicode code point U+000A */ . unicode_char = /* an arbitrary Unicode code point except newline */ . unicode_letter = /* a Unicode code point classified as "Letter" */ . unicode_digit = /* a Unicode code point classified as "Number, decimal digit" */ . ================================================ FILE: spec_test.go ================================================ package nash import ( "bytes" "io/ioutil" "path/filepath" "testing" "github.com/madlambda/nash/tests" "golang.org/x/exp/ebnf" ) func TestSpecificationIsSane(t *testing.T) { filename := filepath.Join(tests.Projectpath, "spec.ebnf") content, err := ioutil.ReadFile(filename) if err != nil { t.Error(err) return } buf := bytes.NewBuffer(content) grammar, err := ebnf.Parse(filename, buf) if err != nil { t.Error(err) return } err = ebnf.Verify(grammar, "program") if err != nil { t.Error(err) return } } ================================================ FILE: stdbin/mkdir/main.go ================================================ package main import ( "os" "fmt" ) func mkdirs(dirnames []string) error { for _, d := range dirnames { if err := os.MkdirAll(d, 0644); err != nil { return err } } return nil } func main() { if len(os.Args) < 2 { fmt.Fprintf(os.Stderr, "usage: %s ...\n", os.Args[0]) os.Exit(1) } err := mkdirs(os.Args[1:]) if err != nil { fmt.Fprintf(os.Stderr, "error: %s\n", err) os.Exit(1) } } ================================================ FILE: stdbin/mkdir/mkdir_test.go ================================================ package main import ( "testing" "path" "path/filepath" "io/ioutil" "os" ) type testcase struct { dirs []string } func testMkdir(t *testing.T, tc testcase) { tmpDir, err := ioutil.TempDir("", "nash-mkdir") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpDir) var dirs []string for _, p := range tc.dirs { dirs = append(dirs, filepath.FromSlash(path.Join(tmpDir, p))) } err = mkdirs(dirs) if err != nil { t.Fatal(err) } for _, d := range dirs { if s, err := os.Stat(d); err != nil { t.Fatal(err) } else if s.Mode()&os.ModeDir != os.ModeDir { t.Fatalf("Invalid directory mode: %v", s.Mode()) } } } func TestMkdir(t *testing.T) { for _, tc := range []testcase{ { dirs: []string{}, }, { dirs: []string{ "1", "2", "3", "4", "5", "some", "thing", "_random_", "_", }, }, { dirs: []string{"a", "a"}, // directory already exists, silently works }, } { tc := tc testMkdir(t, tc) } } ================================================ FILE: stdbin/pwd/main.go ================================================ package main import ( "fmt" "os" ) func main() { wd, err := os.Getwd() if err != nil { fmt.Fprintf(os.Stderr, "error: %s", err) os.Exit(1) } fmt.Println(wd) } ================================================ FILE: stdbin/strings/main.go ================================================ package main import ( "flag" "fmt" "os" ) func main() { const defaultMinTextSize = 4 var minTextSize uint flag.UintVar( &minTextSize, "s", defaultMinTextSize, "the minimum size in runes to characterize as a text", ) scanner := Do(os.Stdin, minTextSize) for scanner.Scan() { fmt.Println(scanner.Text()) } if scanner.Err() != nil { fmt.Fprintf(os.Stderr, "error: %s", scanner.Err()) os.Exit(1) } } ================================================ FILE: stdbin/strings/strings.go ================================================ package main import ( "bufio" "fmt" "io" "unicode/utf8" ) func Do(input io.Reader, minTextSize uint) *bufio.Scanner { outputReader, outputWriter := io.Pipe() go searchstrings(input, minTextSize, outputWriter) scanner := bufio.NewScanner(outputReader) const maxBufferSize = 1024 * 1024 b := make([]byte, maxBufferSize) scanner.Buffer(b, maxBufferSize) return scanner } func searchstrings(input io.Reader, minTextSize uint, output *io.PipeWriter) { newline := byte('\n') searcher := wordSearcher{minTextSize: minTextSize} write := func(data []byte) error { data = append(data, newline) n, err := output.Write(data) if n != len(data) { return fmt.Errorf( "expected to write[%d] wrote[%d]", len(data), n, ) } return err } handleIOError := func(err error) bool { if err != nil { var finalwriteerr error if text, ok := searcher.flushBuffer(); ok { finalwriteerr = write(text) } if err == io.EOF { if finalwriteerr == nil { output.Close() } else { output.CloseWithError(fmt.Errorf( "error[%s] writing last data", finalwriteerr, )) } } else { output.CloseWithError(err) } return true } return false } data := make([]byte, 1) for { // WHY: Don't see the point of checking N when reading a single byte n, err := input.Read(data) if n <= 0 { if handleIOError(err) { return } // WHY: // Implementations of Read are discouraged from // returning a zero byte count with a nil error, // except when len(p) == 0. // Callers should treat a return of 0 and nil as // indicating that nothing happened; in particular it // does not indicate EOF. continue } if text, ok := searcher.next(data[0]); ok { err = write(text) } if handleIOError(err) { return } } } type byteType int type wordSearcher struct { buffer []byte possibleRune []byte waitingForRune bool minTextSize uint } const ( binaryType byteType = iota asciiType runeStartType ) func (w *wordSearcher) next(b byte) ([]byte, bool) { if w.waitingForRune { return w.nextRune(b) } return w.nextASCII(b) } func (w *wordSearcher) nextRune(b byte) ([]byte, bool) { const maxUTFSize = 4 if b == 0 { w.resetRuneSearch() return w.flushBuffer() } if word := string([]byte{b}); utf8.ValidString(word) { w.resetRuneSearch() data, ok := w.flushBuffer() w.writeOnBuffer(b) return data, ok } if utf8.RuneStart(b) { w.resetRuneSearch() data, ok := w.flushBuffer() w.startRuneSearch(b) return data, ok } w.writeOnPossibleRune(b) if utf8.ValidString(string(w.possibleRune)) { w.writeOnBuffer(w.possibleRune...) w.resetRuneSearch() return nil, false } if len(w.possibleRune) == maxUTFSize { w.resetRuneSearch() return w.flushBuffer() } return nil, false } func (w *wordSearcher) resetRuneSearch() { w.waitingForRune = false w.possibleRune = nil } func (w *wordSearcher) nextASCII(b byte) ([]byte, bool) { switch bytetype(b) { case binaryType: { return w.flushBuffer() } case asciiType: { w.writeOnBuffer(b) } case runeStartType: { w.startRuneSearch(b) } } return nil, false } func (w *wordSearcher) startRuneSearch(b byte) { w.waitingForRune = true w.writeOnPossibleRune(b) } func (w *wordSearcher) writeOnBuffer(b ...byte) { w.buffer = append(w.buffer, b...) } func (w *wordSearcher) writeOnPossibleRune(b byte) { w.possibleRune = append(w.possibleRune, b) } func (w *wordSearcher) bufferLenInRunes() uint { return uint(len([]rune(string(w.buffer)))) } func (w *wordSearcher) flushBuffer() ([]byte, bool) { if len(w.buffer) == 0 { return nil, false } if w.bufferLenInRunes() < w.minTextSize { w.buffer = nil return nil, false } b := w.buffer w.buffer = nil return b, true } func bytetype(b byte) byteType { if b == 0 { return binaryType } if word := string([]byte{b}); utf8.ValidString(word) { return asciiType } if utf8.RuneStart(b) { return runeStartType } return binaryType } ================================================ FILE: stdbin/strings/strings_test.go ================================================ package main_test import ( "bufio" "bytes" "errors" "fmt" "io" "testing" strings "github.com/madlambda/nash/stdbin/strings" ) func TestStrings(t *testing.T) { type testcase struct { name string input func([]byte) []byte output []string minWordSize uint } tcases := []testcase{ { name: "UTF-8With2Bytes", minWordSize: 1, input: func(bin []byte) []byte { return append([]byte("λ"), bin...) }, output: []string{"λ"}, }, { name: "UTF-8With3Bytes", minWordSize: 1, input: func(bin []byte) []byte { return append([]byte("€"), bin...) }, output: []string{"€"}, }, { name: "UTF-8With4Bytes", minWordSize: 1, input: func(bin []byte) []byte { return append([]byte("𐍈"), bin...) }, output: []string{"𐍈"}, }, { name: "NonASCIIWordHasOneLessCharThanMin", minWordSize: 2, input: func(bin []byte) []byte { return append([]byte("λ"), bin...) }, output: []string{}, }, { name: "NonASCIIWordHasMinWordSize", minWordSize: 2, input: func(bin []byte) []byte { return append([]byte("λλ"), bin...) }, output: []string{"λλ"}, }, { name: "WordHasOneLessCharThanMin", minWordSize: 2, input: func(bin []byte) []byte { return append([]byte("k"), bin...) }, output: []string{}, }, { name: "WordHasMinWordSize", minWordSize: 2, input: func(bin []byte) []byte { return append([]byte("kz"), bin...) }, output: []string{"kz"}, }, { name: "WordHasOneMoreCharThanMinWordSize", minWordSize: 2, input: func(bin []byte) []byte { return append([]byte("ktz"), bin...) }, output: []string{"ktz"}, }, { name: "StartingWithOneChar", minWordSize: 1, input: func(bin []byte) []byte { return append([]byte("k"), bin...) }, output: []string{"k"}, }, { name: "EndWithOneChar", minWordSize: 1, input: func(bin []byte) []byte { return append(bin, []byte("k")...) }, output: []string{"k"}, }, { name: "OneCharInTheMiddle", minWordSize: 1, input: func(bin []byte) []byte { t := append(bin, []byte("k")...) t = append(t, bin...) return t }, output: []string{"k"}, }, { name: "StartingWithText", minWordSize: 1, input: func(bin []byte) []byte { expected := "textOnBeggining" return append([]byte(expected), bin...) }, output: []string{"textOnBeggining"}, }, { name: "TextOnMiddle", minWordSize: 1, input: func(bin []byte) []byte { expected := "textOnMiddle" return append(bin, append([]byte(expected), bin...)...) }, output: []string{"textOnMiddle"}, }, { name: "NonASCIITextOnMiddle", minWordSize: 1, input: func(bin []byte) []byte { expected := "λλλ" return append(bin, append([]byte(expected), bin...)...) }, output: []string{"λλλ"}, }, { name: "ASCIIAndNonASCII", minWordSize: 1, input: func(bin []byte) []byte { expected := "(define (λ (x) (+ x a)))" return append(bin, append([]byte(expected), bin...)...) }, output: []string{"(define (λ (x) (+ x a)))"}, }, { name: "TextOnEnd", minWordSize: 1, input: func(bin []byte) []byte { expected := "textOnEnd" return append(bin, append([]byte(expected), bin...)...) }, output: []string{"textOnEnd"}, }, { name: "JustText", minWordSize: 1, input: func(bin []byte) []byte { return []byte("justtext") }, output: []string{"justtext"}, }, { name: "JustBinary", minWordSize: 1, input: func(bin []byte) []byte { return bin }, output: []string{}, }, { name: "TextSeparatedByBinary", minWordSize: 1, input: func(bin []byte) []byte { text := []byte("text") t := []byte{} t = append(t, bin...) t = append(t, text...) t = append(t, bin...) t = append(t, text...) return t }, output: []string{"text", "text"}, }, { name: "NonASCIITextSeparatedByBinary", minWordSize: 1, input: func(bin []byte) []byte { text := []byte("awesomeλ=)") t := []byte{} t = append(t, bin...) t = append(t, text...) t = append(t, bin...) t = append(t, text...) return t }, output: []string{"awesomeλ=)", "awesomeλ=)"}, }, { name: "WordsAreNotAccumulativeBetweenBinData", minWordSize: 2, input: func(bin []byte) []byte { t := append([]byte("k"), bin...) return append(t, byte('t')) }, output: []string{}, }, { name: "ASCIISeparatedByByteThatLooksLikeUTF", minWordSize: 1, input: func(bin []byte) []byte { return append([]byte{ 'n', runestart, 'k', }, bin...) }, output: []string{"n", "k"}, }, { name: "ASCIIAfterPossibleFirstByteOfUTF", minWordSize: 1, input: func(bin []byte) []byte { return append([]byte{ runestart, 'k', }, bin...) }, output: []string{"k"}, }, { name: "ASCIIAfterPossibleSecondByteOfUTF", minWordSize: 1, input: func(bin []byte) []byte { return append([]byte{ byte(0xE2), byte(0x82), 'k', }, bin...) }, output: []string{"k"}, }, { name: "ASCIIAfterPossibleThirdByteOfUTF", minWordSize: 1, input: func(bin []byte) []byte { return append([]byte{ byte(0xF0), byte(0x90), byte(0x8D), 'k', }, bin...) }, output: []string{"k"}, }, { name: "AfterFalseRuneStartRuneStartOnSecondByte", minWordSize: 1, input: func(bin []byte) []byte { i := []byte{byte(0xF0)} i = append(i, []byte("λ")...) return append(i, bin...) }, output: []string{"λ"}, }, { name: "AfterFalseRuneStartRuneStartOnThirdByte", minWordSize: 1, input: func(bin []byte) []byte { i := []byte{byte(0xF0), byte(0x90)} i = append(i, []byte("λ")...) return append(i, bin...) }, output: []string{"λ"}, }, { name: "AfterFalseRuneStartRuneStartOnFourthByte", minWordSize: 1, input: func(bin []byte) []byte { i := []byte{byte(0xF0), byte(0x90), byte(0x8D)} i = append(i, []byte("λ")...) return append(i, bin...) }, output: []string{"λ"}, }, { name: "ASCIIFakeRuneAndThemRune", minWordSize: 1, input: func(bin []byte) []byte { i := []byte{'v'} i = append(i, byte(0xF0)) i = append(i, []byte("λ")...) return append(i, bin...) }, output: []string{"v", "λ"}, }, { name: "ASCIISplittedByZero", minWordSize: 1, input: func([]byte) []byte { return []byte{'k', 0, 'n', 0, 'v'} }, output: []string{"k", "n", "v"}, }, { name: "RunesSplittedByZero", minWordSize: 1, input: func([]byte) []byte { i := []byte("λ") i = append(i, 0) i = append(i, []byte("λ")...) return i }, output: []string{"λ", "λ"}, }, { name: "ASCIIAndRunesSplittedByZero", minWordSize: 1, input: func([]byte) []byte { i := []byte("λ") i = append(i, 0) i = append(i, 's') i = append(i, 0) i = append(i, []byte("λ")...) return i }, output: []string{"λ", "s", "λ"}, }, } minBinChunkSize := 1 maxBinChunkSize := 128 for _, tcase := range tcases { for i := minBinChunkSize; i <= maxBinChunkSize; i++ { binsize := i testname := fmt.Sprintf("%s/binSize%d", tcase.name, binsize) t.Run(testname, func(t *testing.T) { bin := newBinary(uint(binsize)) input := tcase.input(bin) scanner := strings.Do(bytes.NewBuffer(input), tcase.minWordSize) lines := []string{} for scanner.Scan() { lines = append(lines, scanner.Text()) } if len(lines) != len(tcase.output) { t.Errorf("wanted size[%d] got size[%d]", len(tcase.output), len(lines)) t.Fatalf("wanted[%s] got[%s]", tcase.output, lines) } for i, want := range tcase.output { got := lines[i] if want != got { t.Errorf("unexpected line at[%d]", i) t.Errorf("wanted[%s] got[%s]", want, got) t.Errorf("wantedLines[%s] gotLines[%s]", tcase.output, lines) } } if scanner.Err() != nil { t.Fatalf("unexpected error[%s]", scanner.Err()) } }) } } } func TestStringsReadErrorOnFirstByte(t *testing.T) { var minWordSize uint = 1 scanner := strings.Do(newFakeReader(func(d []byte) (int, error) { return 0, errors.New("fake injected error") }), minWordSize) assertScannerFails(t, scanner, 0) } func TestStringsReadErrorOnSecondByte(t *testing.T) { var minWordSize uint = 1 sentFirstByte := false scanner := strings.Do(newFakeReader(func(d []byte) (int, error) { if sentFirstByte { return 0, errors.New("fake injected error") } d[0] = 'k' sentFirstByte = true return 1, nil }), minWordSize) assertScannerFails(t, scanner, 1) } func TestStringsReadErrorAfterValidUTF8StartingByte(t *testing.T) { var minWordSize uint = 1 sentFirstByte := false scanner := strings.Do(newFakeReader(func(d []byte) (int, error) { if sentFirstByte { return 0, errors.New("fake injected error") } sentFirstByte = true d[0] = runestart return 1, nil }), minWordSize) assertScannerFails(t, scanner, 0) } func TestStringsReadCanReturnEOFWithData(t *testing.T) { var minWordSize uint = 1 want := byte('k') scanner := strings.Do(newFakeReader(func(d []byte) (int, error) { if len(d) == 0 { t.Fatal("empty data on Read operation") } d[0] = want return 1, io.EOF }), minWordSize) if !scanner.Scan() { t.Fatal("unexpected Scan failure") } got := scanner.Text() if string(want) != got { t.Fatalf("want[%s] != got[%s]", string(want), got) } } const runestart byte = 0xC2 type FakeReader struct { read func([]byte) (int, error) } func (f *FakeReader) Read(d []byte) (int, error) { if f.read == nil { return 0, fmt.Errorf("FakeReader has no Read implementation") } return f.read(d) } func newFakeReader(read func([]byte) (int, error)) *FakeReader { return &FakeReader{read: read} } func assertScannerFails(t *testing.T, scanner *bufio.Scanner, expectedIter uint) { var iterations uint for scanner.Scan() { iterations += 1 } if iterations != expectedIter { t.Fatalf("expected[%d] Scan calls, got [%d]", expectedIter, iterations) } if scanner.Err() == nil { t.Fatal("expected failure on scanner, got none") } } func newBinary(size uint) []byte { // WHY: Starting with the most significant bit as 1 helps to test // UTF-8 corner cases. Don't change this without providing // testing for this. Not the best way to do this (not explicit) // but it is what we have for today =). bin := make([]byte, size) for i := 0; i < int(size); i++ { bin[i] = 0xFF } return bin } ================================================ FILE: stdbin/write/common_test.sh ================================================ # common test routines fn fatal(msg) { print($msg) exit("1") } fn assert(expected, got, desc) { if $expected != $got { fatal(format("%s: FAILED. Expected[%s] but got[%s]\n", $desc, $expected, $got)) } } ================================================ FILE: stdbin/write/fd.go ================================================ //+build !windows package main import ( "io" "os" ) func specialFile(path string) (io.WriteCloser, bool) { if path == "/dev/stdout" { return os.Stdout, true } else if path == "/dev/stderr" { return os.Stderr, true } return nil, false } ================================================ FILE: stdbin/write/fd_windows.go ================================================ package main import ( "io" "os" ) func specialFile(path string) (io.WriteCloser, bool) { if path == "CON" { // holycrap! return os.Stdout, true } return nil, false } ================================================ FILE: stdbin/write/main.go ================================================ package main import ( "bytes" "fmt" "io" "os" ) var banner = fmt.Sprintf("%s \n", os.Args[0]) func fatal(msg string) { fmt.Fprintf(os.Stderr, "%s", msg) os.Exit(1) } func main() { if len(os.Args) <= 1 || len(os.Args) > 3 { fatal(banner) } var ( fname = os.Args[1] in io.Reader ) if len(os.Args) == 2 { in = os.Stdin } else { in = bytes.NewBufferString(os.Args[2]) } err := write(fname, in) if err != nil { fatal(err.Error()) } } ================================================ FILE: stdbin/write/write.go ================================================ package main import ( "io" "os" "path/filepath" ) func toabs(path string) (string, error) { if !filepath.IsAbs(path) { wd, err := os.Getwd() if err != nil { return "", err } path = filepath.Join(wd, path) } return path, nil } func outfd(fname string) (io.WriteCloser, error) { fname, err := toabs(fname) if err != nil { return nil, err } var out io.WriteCloser out, ok := specialFile(fname) if !ok { f, err := os.OpenFile(fname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return nil, err } out = f } return out, nil } func write(fname string, in io.Reader) (err error) { out, err := outfd(fname) if err != nil { return err } defer func() { err2 := out.Close() if err == nil { err = err2 } }() _, err = io.Copy(out, in) return err } ================================================ FILE: stdbin/write/write_linux_test.sh ================================================ # linux tests of write command import "./common_test.sh" # this test uses only the write binary setenv PATH = "./stdbin/write" # (desc (out err status)) var tests = ( ("standard out" ("/dev/stdout" "hello world" "" "0")) ("standard err" ("/dev/stderr" "" "hello world" "0")) ) var outstr = "hello world" for test in $tests { var desc = $test[0] var tc = $test[1] print("testing %s\n", $desc) var device = $tc[0] var expectedOut = $tc[1] var expectedErr = $tc[2] var expectedSts = $tc[3] var out, err, status <= write $device $outstr assert($expectedSts, $status, "status code") assert($expectedOut, $out, "standard output") assert($expectedErr, $err, "standard error") } ================================================ FILE: stdbin/write/write_test.sh ================================================ import "./common_test.sh" setenv PATH = "./stdbin/write:/bin" # FIXME: we need our mktemp var nonExistentFile = "./here-be-dragons" fn clean() { _, _ <= rm -f $nonExistentFile } clean() var out, err, status <= write $nonExistentFile "hello" assert("", $out, "standard out isnt empty") assert("", $err, "standard err isnt empty") assert("0", $status, "status is not success") var content, status <= cat $nonExistentFile assert("0", $status, "status is not success") assert("hello", $content, "file content is wrong") # test append out, err, status <= write $nonExistentFile "1" assert("", $out, "standard out isnt empty") assert("", $err, "standard err isnt empty") assert("0", $status, "status is not success") content, status <= cat $nonExistentFile assert("0", $status, "status is not success") assert("hello1", $content, "file content is wrong") clean() ================================================ FILE: stdlib/io.sh ================================================ fn io_println(msg, args...) { print($msg + "\n", $args...) } ================================================ FILE: stdlib/io_example.sh ================================================ import io if len($ARGS) == "2" { io_println($ARGS[1]) exit("0") } io_println($ARGS[1], $ARGS[2]) ================================================ FILE: stdlib/io_test.sh ================================================ fn run_example(args...) { var got, status <= ./cmd/nash/nash ./stdlib/io_example.sh $args return $got, $status } fn assert_success(expected, got, status) { if $status != "0" { print("expected success, but got status code: %s\n", $status) exit("1") } if $got != $expected { print("expected [%s] got [%s]\n", $expected, $got) exit("1") } } fn test_println_format() { var got, status <= run_example("hello %s", "world") assert_success("hello world", $got, $status) } fn test_println() { var expected = "pazu" var got, status <= run_example($expected) assert_success($expected, $got, $status) } test_println_format() test_println() ================================================ FILE: stdlib/map.sh ================================================ fn map_new() { return () } fn map_get(map, key) { return map_get_default($map, $key, "") } fn map_iter(map, func) { for entry in $map { $func($entry[0], $entry[1]) } } fn map_get_default(map, key, default) { for entry in $map { if $entry[0] == $key { return $entry[1] } } return $default } fn map_add(map, key, val) { for entry in $map { if $entry[0] == $key { entry[1] = $val return $map } } var tuple = ($key $val) map <= append($map, $tuple) return $map } fn map_del(map, key) { var newmap = () for entry in $map { if $entry[0] != $key { var tuple = ($entry[0] $entry[1]) newmap <= append($newmap, $tuple) } } return $newmap } ================================================ FILE: stdlib/map_test.sh ================================================ import map fn expect(map, key, want) { var got <= map_get($map, $key) if $got != $want { echo "error: got["+$got+"] want["+$want+"]" exit("1") } } fn test_adding_keys() { var map <= map_new() map <= map_add($map, "key", "value") expect($map, "key", "value") map <= map_add($map, "key2", "value2") expect($map, "key2", "value2") map <= map_add($map, "key", "override") expect($map, "key", "override") } fn test_absent_key_will_have_empty_string_value() { var map <= map_new() expect($map, "absent", "") } fn test_absent_key_with_custom_default_value() { var map <= map_new() var want = "hi" var got <= map_get_default($map, "absent", $want) if $got != $want { echo "error: got["+$got+"] want["+$want+"]" exit("1") } } fn test_iterates_map() { var map <= map_new() map <= map_add($map, "key", "value") map <= map_add($map, "key2", "value2") var got <= map_new() fn iter(key, val) { got <= map_add($got, $key, $val) } map_iter($map, $iter) expect($map, "key", "value") expect($map, "key2", "value2") } fn test_removing_key() { var map <= map_new() map <= map_add($map, "key", "value") map <= map_add($map, "key2", "value2") expect($map, "key", "value") expect($map, "key2", "value2") map <= map_del($map, "key") expect($map, "key", "") expect($map, "key2", "value2") } fn test_removing_absent_key() { var map <= map_new() expect($map, "key", "") map <= map_del($map, "key") expect($map, "key", "") } test_adding_keys() test_absent_key_will_have_empty_string_value() test_absent_key_with_custom_default_value() test_iterates_map() test_removing_key() test_removing_absent_key() ================================================ FILE: testfiles/ex1.sh ================================================ #!/bin/cnt echo "hello world" ================================================ FILE: testfiles/fibonacci.sh ================================================ #!/usr/bin/env nash # Recursive fibonacci implementation to find the value # at index n in the sequence. # Some times: # λ> time ./testfiles/fibonacci.sh 1 # 1 # 0.00u 0.01s 0.01r ./testfiles/fibonacci.sh 1 # λ> time ./testfiles/fibonacci.sh 2 # 1 2 # 0.01u 0.01s 0.02r ./testfiles/fibonacci.sh 2 # λ> time ./testfiles/fibonacci.sh 3 # 1 2 3 # 0.01u 0.03s 0.03r ./testfiles/fibonacci.sh 3 # λ> time ./testfiles/fibonacci.sh 4 # 1 2 3 5 # 0.04u 0.04s 0.07r ./testfiles/fibonacci.sh 4 # λ> time ./testfiles/fibonacci.sh 5 # 1 2 3 5 8 # 0.09u 0.07s 0.13r ./testfiles/fibonacci.sh 5 # λ> time ./testfiles/fibonacci.sh 10 # 1 2 3 5 8 13 21 34 55 89 # 1.31u 1.18s 2.03r ./testfiles/fibonacci.sh 10 # λ> time ./testfiles/fibonacci.sh 15 # 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 # 15.01u 13.49s 22.55r ./testfiles/fibonacci.sh 15 # λ> time ./testfiles/fibonacci.sh 20 # 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 1597 2584 4181 6765 10946 # 169.27u 155.50s 265.19r ./testfiles/fibonacci.sh 20 # a is lower or equal than b? fn lte(a, b) { var _, st <= test $a -le $b return $st } fn fib(n) { if lte($n, "1") == "0" { return "1" } var a, _ <= expr $n - 1 var b, _ <= expr $n - 2 var _a <= fib($a) var _b <= fib($b) var ret, _ <= expr $_a "+" $_b return $ret } fn range(start, end) { var seq, _ <= seq $start $end var lst <= split($seq, "\n") return $lst } for i in range("1", $ARGS[1]) { print("%s ", fib($i)) } print("\n") ================================================ FILE: testfiles/sieve.sh ================================================ #!/usr/bin/env nash # Sieve of Erathostenes fn lt(a, b) { var _, st <= test $a -lt $b return $st } fn gt(a, b) { var _, st <= test $a -gt $b return $st } fn le(a, b) { var _, st <= test $a -le $b return $st } fn sqrt(n) { var v, _ <= expr $n * $n return $v } fn range(start, end) { var values, _ <= seq $start $end var list <= split($values, "\n") return $list } fn xrange(start, condfn) { var out = () if $condfn($start) == "0" { out = ($start) } else { return () } var next = $start for { next, _ <= expr $next "+" 1 if $condfn($next) == "0" { out <= append($out, $next) } else { return $out } } unreachable } fn sieve(n) { if lt($n, "2") == "0" { return () } if $n == "2" { return ("2") } var tries = ("0" "0") for i in range("2", $n) { tries <= append($tries, "1") } fn untilSqrtRoot(v) { return le(sqrt($v), $n) } for i in xrange("2", $untilSqrtRoot) { if $tries[$i] == "1" { for j in range("0", $n) { # arithmetic seems cryptic without integers =( var k, _ <= expr $i * $i "+" "(" $j * $i ")" if gt($k, $n) != "0" { tries[$k] = "0" } } } } var primes = () for i in range("2", $n) { if $tries[$i] == "1" { primes <= append($primes, $i) } } return $primes } for prime in sieve($ARGS[1]) { print("%s ", $prime) } print("\n") ================================================ FILE: tests/cfg.go ================================================ package tests import ( "os" "path/filepath" "runtime" "strings" ) var ( // Nashcmd is the nash's absolute binary path in source Nashcmd string // Projectpath is the path to nash source code Projectpath string // Testdir is the test assets directory Testdir string // Stdbindir is the stdbin directory Stdbindir string ) func init() { project := "github.com/madlambda/nash" wd, err := os.Getwd() if err != nil { panic("failed to get current directory") } pos := strings.Index(wd, project) + len(project) Projectpath = wd[:pos] Testdir = filepath.Join(Projectpath, "testfiles") Nashcmd = filepath.Join(Projectpath, "cmd", "nash", "nash") Stdbindir = filepath.Join(Projectpath, "stdbin") if runtime.GOOS == "windows" { Nashcmd += ".exe" } if _, err := os.Stat(Nashcmd); err != nil { panic("Please, run make build before running tests") } } ================================================ FILE: tests/doc.go ================================================ // Package tests contains all nash tests that are blackbox. // What would be blackbox ? These are tests that are targeted // directly on top of the language using only the shell API, // they are end to end in the sense that they will exercise // a lot of different packages on a single test. // // The objective of these tests is to have a compreensive set // of tests that are coupled only the language specification // and not to how the language is implemented. These allows // extremely aggressive refactorings to be made without // incurring in any changes on the tests. // // There are disadvantages but discussing integration VS unit // testing here is not the point (there are also unit tests). // // Here even tests that involves the script calling syscalls like // exit are allowed without interfering with the results of other tests package tests ================================================ FILE: tests/internal/assert/doc.go ================================================ // Package assert has some common assert functions package assert ================================================ FILE: tests/internal/assert/equal.go ================================================ package assert import ( "strings" "testing" ) func EqualStrings(t *testing.T, want string, got string) { // TODO: could use t.Helper here, but only on Go 1.9 if want != got { t.Fatalf("wanted[%s] but got[%s]", want, got) } } func ContainsString(t *testing.T, str string, sub string) { // TODO: could use t.Helper here, but only on Go 1.9 if !strings.Contains(str, sub) { t.Fatalf("[%s] is not a substring of [%s]", sub, str) } } ================================================ FILE: tests/internal/assert/error.go ================================================ package assert import "testing" func NoError(t *testing.T, err error, operation string) { if err != nil { t.Fatalf("error[%s] %s", err, operation) } } ================================================ FILE: tests/internal/sh/shell.go ================================================ // Package shell makes it easier to run nash scripts for test purposes package sh import ( "bytes" "io" "io/ioutil" "os" "os/exec" "testing" "github.com/madlambda/nash/tests/internal/assert" ) // Exec runs the script code and returns the result of it. func Exec( t *testing.T, nashpath string, scriptcode string, scriptargs ...string, ) (string, string, error) { scriptfile, err := ioutil.TempFile("", "testshell") assert.NoError(t, err, "creating tmp file") defer func() { err := scriptfile.Close() assert.NoError(t, err, "closing tmp file") err = os.Remove(scriptfile.Name()) assert.NoError(t, err, "deleting tmp file") }() _, err = io.Copy(scriptfile, bytes.NewBufferString(scriptcode)) assert.NoError(t, err, "writing script code to tmp file") scriptargs = append([]string{scriptfile.Name()}, scriptargs...) cmd := exec.Command(nashpath, scriptargs...) stdout := bytes.Buffer{} stderr := bytes.Buffer{} cmd.Stdout = &stdout cmd.Stderr = &stderr err = cmd.Run() return stdout.String(), stderr.String(), err } ================================================ FILE: tests/internal/tester/tester.go ================================================ // Package tester makes it easy to run multiple // script test cases. package tester import ( "testing" "github.com/madlambda/nash/tests/internal/assert" "github.com/madlambda/nash/tests/internal/sh" ) type TestCase struct { Name string ScriptCode string ExpectStdout string ExpectStderrToContain string Fails bool } func Run(t *testing.T, nashcmd string, cases ...TestCase) { for _, tcase := range cases { t.Run(tcase.Name, func(t *testing.T) { stdout, stderr, err := sh.Exec(t, nashcmd, tcase.ScriptCode) if !tcase.Fails { if err != nil { t.Fatalf( "error[%s] stdout[%s] stderr[%s]", err, stdout, stderr, ) } if stderr != "" { t.Fatalf( "unexpected stderr[%s], on success no stderr is expected", stderr, ) } } if tcase.ExpectStdout != "" { assert.EqualStrings(t, tcase.ExpectStdout, stdout) } if tcase.ExpectStderrToContain != "" { assert.ContainsString(t, stderr, tcase.ExpectStderrToContain) } }) } } ================================================ FILE: tests/listindex_test.go ================================================ package tests import ( "testing" "github.com/madlambda/nash/tests/internal/tester" ) func TestListIndexing(t *testing.T) { tester.Run(t, Nashcmd, tester.TestCase{ Name: "PositionalAccess", ScriptCode: ` var a = ("1" "2") echo $a[0] echo $a[1] `, ExpectStdout: "1\n2\n", }, tester.TestCase{ Name: "PositionalAssigment", ScriptCode: ` var a = ("1" "2") a[0] = "9" a[1] = "p" echo $a[0] + $a[1] `, ExpectStdout: "9p\n", }, tester.TestCase{ Name: "PositionalAccessWithVar", ScriptCode: ` var a = ("1" "2") var i = "0" echo $a[$i] i = "1" echo $a[$i] `, ExpectStdout: "1\n2\n", }, tester.TestCase{ Name: "Iteration", ScriptCode: ` var a = ("1" "2" "3") for x in $a { echo $x } `, ExpectStdout: "1\n2\n3\n", }, tester.TestCase{ Name: "IterateEmpty", ScriptCode: ` var a = () for x in $a { exit("1") } echo "ok" `, ExpectStdout: "ok\n", }, tester.TestCase{ Name: "IndexOutOfRangeFails", ScriptCode: ` var a = ("1" "2" "3") echo $a[3] `, Fails: true, ExpectStderrToContain: "IndexError", }, tester.TestCase{ Name: "IndexEmptyFails", ScriptCode: ` var a = () echo $a[0] `, Fails: true, ExpectStderrToContain: "IndexError", }, ) } ================================================ FILE: tests/stringindex_test.go ================================================ package tests import ( "testing" "github.com/madlambda/nash/tests/internal/tester" ) func TestStringIndexing(t *testing.T) { tester.Run(t, Nashcmd, tester.TestCase{ Name: "IterateEmpty", ScriptCode: ` var a = "" for x in $a { exit("1") } echo "ok" `, ExpectStdout: "ok\n", }, tester.TestCase{ Name: "IndexEmptyFails", ScriptCode: ` var a = "" echo $a[0] `, Fails: true, ExpectStderrToContain: "IndexError", }, tester.TestCase{ Name: "IsImmutable", ScriptCode: ` var a = "12" a[0] = "2" echo $a `, Fails: true, ExpectStderrToContain: "IndexError", }, ) } func TestStringIndexingASCII(t *testing.T) { tester.Run(t, Nashcmd, tester.TestCase{Name: "PositionalAccess", ScriptCode: ` var a = "12" echo $a[0] echo $a[1] `, ExpectStdout: "1\n2\n", }, tester.TestCase{ Name: "PositionalAccessReturnsString", ScriptCode: ` var a = "12" var x = $a[0] + $a[1] echo $x `, ExpectStdout: "12\n", }, tester.TestCase{ Name: "Len", ScriptCode: ` var a = "12" var l <= len($a) echo $l `, ExpectStdout: "2\n", }, tester.TestCase{ Name: "Iterate", ScriptCode: ` var a = "123" for x in $a { echo $x } `, ExpectStdout: "1\n2\n3\n", }, tester.TestCase{ Name: "IndexOutOfRangeFails", ScriptCode: ` var a = "123" echo $a[3] `, Fails: true, ExpectStderrToContain: "IndexError", }, ) } func TestStringIndexingNonASCII(t *testing.T) { tester.Run(t, Nashcmd, tester.TestCase{Name: "PositionalAccess", ScriptCode: ` var a = "⌘⌘" echo $a[0] echo $a[1] `, ExpectStdout: "⌘\n⌘\n", }, tester.TestCase{ Name: "Iterate", ScriptCode: ` var a = "⌘⌘" for x in $a { echo $x } `, ExpectStdout: "⌘\n⌘\n", }, tester.TestCase{ Name: "PositionalAccessReturnsString", ScriptCode: ` var a = "⌘⌘" var x = $a[0] + $a[1] echo $x `, ExpectStdout: "⌘⌘\n", }, tester.TestCase{ Name: "Len", ScriptCode: ` var a = "⌘⌘" var l <= len($a) echo $l `, ExpectStdout: "2\n", }, tester.TestCase{ Name: "IndexOutOfRangeFails", ScriptCode: ` var a = "⌘⌘" echo $a[2] `, Fails: true, ExpectStderrToContain: "IndexError", }, ) } ================================================ FILE: token/token.go ================================================ package token import "strconv" type ( Token int FileInfo struct { line, column int } ) const ( Illegal Token = iota + 1 // error ocurred EOF Comment literal_beg Ident String // "" Number // [0-9]+ Arg literal_end operator_beg Assign // = AssignCmd // <= Equal // == NotEqual // != Plus // + Minus // - Gt // > Lt // < Colon // , Semicolon // ; operator_end LBrace // { RBrace // } LParen // ( RParen // ) LBrack // [ RBrack // ] Pipe Comma Dotdotdot Variable keyword_beg Import SetEnv ShowEnv BindFn // "bindfn Dump // "dump" [ file ] Return If Else For Rfork Fn Var keyword_end ) var tokens = [...]string{ Illegal: "ILLEGAL", EOF: "EOF", Comment: "COMMENT", Ident: "IDENT", String: "STRING", Number: "NUMBER", Arg: "ARG", Assign: "=", AssignCmd: "<=", Equal: "==", NotEqual: "!=", Plus: "+", Minus: "-", Gt: ">", Lt: "<", Colon: ",", Semicolon: ";", LBrace: "{", RBrace: "}", LParen: "(", RParen: ")", LBrack: "[", RBrack: "]", Pipe: "|", Comma: ",", Dotdotdot: "...", Variable: "VARIABLE", Import: "import", SetEnv: "setenv", ShowEnv: "showenv", BindFn: "bindfn", Dump: "dump", Return: "return", If: "if", Else: "else", For: "for", Rfork: "rfork", Fn: "fn", Var: "var", } var keywords map[string]Token func init() { keywords = make(map[string]Token) for i := keyword_beg + 1; i < keyword_end; i++ { keywords[tokens[i]] = i } } func Lookup(ident string) Token { if tok, isKeyword := keywords[ident]; isKeyword { return tok } return Ident } func IsKeyword(t Token) bool { if t > keyword_beg && t < keyword_end { return true } return false } func NewFileInfo(l, c int) FileInfo { return FileInfo{l, c} } func (info FileInfo) Line() int { return info.line } func (info FileInfo) Column() int { return info.column } func (tok Token) String() string { s := "" if 0 < tok && tok < Token(len(tokens)) { s = tokens[tok] } if s == "" { s = "token(" + strconv.Itoa(int(tok)) + ")" } return s } ================================================ FILE: vendor/golang.org/x/exp/ebnf/ebnf.go ================================================ // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package ebnf is a library for EBNF grammars. The input is text ([]byte) // satisfying the following grammar (represented itself in EBNF): // // Production = name "=" [ Expression ] "." . // Expression = Alternative { "|" Alternative } . // Alternative = Term { Term } . // Term = name | token [ "…" token ] | Group | Option | Repetition . // Group = "(" Expression ")" . // Option = "[" Expression "]" . // Repetition = "{" Expression "}" . // // A name is a Go identifier, a token is a Go string, and comments // and white space follow the same rules as for the Go language. // Production names starting with an uppercase Unicode letter denote // non-terminal productions (i.e., productions which allow white-space // and comments between tokens); all other production names denote // lexical productions. // package ebnf // import "golang.org/x/exp/ebnf" import ( "errors" "fmt" "text/scanner" "unicode" "unicode/utf8" ) // ---------------------------------------------------------------------------- // Error handling type errorList []error func (list errorList) Err() error { if len(list) == 0 { return nil } return list } func (list errorList) Error() string { switch len(list) { case 0: return "no errors" case 1: return list[0].Error() } return fmt.Sprintf("%s (and %d more errors)", list[0], len(list)-1) } func newError(pos scanner.Position, msg string) error { return errors.New(fmt.Sprintf("%s: %s", pos, msg)) } // ---------------------------------------------------------------------------- // Internal representation type ( // An Expression node represents a production expression. Expression interface { // Pos is the position of the first character of the syntactic construct Pos() scanner.Position } // An Alternative node represents a non-empty list of alternative expressions. Alternative []Expression // x | y | z // A Sequence node represents a non-empty list of sequential expressions. Sequence []Expression // x y z // A Name node represents a production name. Name struct { StringPos scanner.Position String string } // A Token node represents a literal. Token struct { StringPos scanner.Position String string } // A List node represents a range of characters. Range struct { Begin, End *Token // begin ... end } // A Group node represents a grouped expression. Group struct { Lparen scanner.Position Body Expression // (body) } // An Option node represents an optional expression. Option struct { Lbrack scanner.Position Body Expression // [body] } // A Repetition node represents a repeated expression. Repetition struct { Lbrace scanner.Position Body Expression // {body} } // A Production node represents an EBNF production. Production struct { Name *Name Expr Expression } // A Bad node stands for pieces of source code that lead to a parse error. Bad struct { TokPos scanner.Position Error string // parser error message } // A Grammar is a set of EBNF productions. The map // is indexed by production name. // Grammar map[string]*Production ) func (x Alternative) Pos() scanner.Position { return x[0].Pos() } // the parser always generates non-empty Alternative func (x Sequence) Pos() scanner.Position { return x[0].Pos() } // the parser always generates non-empty Sequences func (x *Name) Pos() scanner.Position { return x.StringPos } func (x *Token) Pos() scanner.Position { return x.StringPos } func (x *Range) Pos() scanner.Position { return x.Begin.Pos() } func (x *Group) Pos() scanner.Position { return x.Lparen } func (x *Option) Pos() scanner.Position { return x.Lbrack } func (x *Repetition) Pos() scanner.Position { return x.Lbrace } func (x *Production) Pos() scanner.Position { return x.Name.Pos() } func (x *Bad) Pos() scanner.Position { return x.TokPos } // ---------------------------------------------------------------------------- // Grammar verification func isLexical(name string) bool { ch, _ := utf8.DecodeRuneInString(name) return !unicode.IsUpper(ch) } type verifier struct { errors errorList worklist []*Production reached Grammar // set of productions reached from (and including) the root production grammar Grammar } func (v *verifier) error(pos scanner.Position, msg string) { v.errors = append(v.errors, newError(pos, msg)) } func (v *verifier) push(prod *Production) { name := prod.Name.String if _, found := v.reached[name]; !found { v.worklist = append(v.worklist, prod) v.reached[name] = prod } } func (v *verifier) verifyChar(x *Token) rune { s := x.String if utf8.RuneCountInString(s) != 1 { v.error(x.Pos(), "single char expected, found "+s) return 0 } ch, _ := utf8.DecodeRuneInString(s) return ch } func (v *verifier) verifyExpr(expr Expression, lexical bool) { switch x := expr.(type) { case nil: // empty expression case Alternative: for _, e := range x { v.verifyExpr(e, lexical) } case Sequence: for _, e := range x { v.verifyExpr(e, lexical) } case *Name: // a production with this name must exist; // add it to the worklist if not yet processed if prod, found := v.grammar[x.String]; found { v.push(prod) } else { v.error(x.Pos(), "missing production "+x.String) } // within a lexical production references // to non-lexical productions are invalid if lexical && !isLexical(x.String) { v.error(x.Pos(), "reference to non-lexical production "+x.String) } case *Token: // nothing to do for now case *Range: i := v.verifyChar(x.Begin) j := v.verifyChar(x.End) if i >= j { v.error(x.Pos(), "decreasing character range") } case *Group: v.verifyExpr(x.Body, lexical) case *Option: v.verifyExpr(x.Body, lexical) case *Repetition: v.verifyExpr(x.Body, lexical) case *Bad: v.error(x.Pos(), x.Error) default: panic(fmt.Sprintf("internal error: unexpected type %T", expr)) } } func (v *verifier) verify(grammar Grammar, start string) { // find root production root, found := grammar[start] if !found { var noPos scanner.Position v.error(noPos, "no start production "+start) return } // initialize verifier v.worklist = v.worklist[0:0] v.reached = make(Grammar) v.grammar = grammar // work through the worklist v.push(root) for { n := len(v.worklist) - 1 if n < 0 { break } prod := v.worklist[n] v.worklist = v.worklist[0:n] v.verifyExpr(prod.Expr, isLexical(prod.Name.String)) } // check if all productions were reached if len(v.reached) < len(v.grammar) { for name, prod := range v.grammar { if _, found := v.reached[name]; !found { v.error(prod.Pos(), name+" is unreachable") } } } } // Verify checks that: // - all productions used are defined // - all productions defined are used when beginning at start // - lexical productions refer only to other lexical productions // // Position information is interpreted relative to the file set fset. // func Verify(grammar Grammar, start string) error { var v verifier v.verify(grammar, start) return v.errors.Err() } ================================================ FILE: vendor/golang.org/x/exp/ebnf/parser.go ================================================ // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package ebnf import ( "io" "strconv" "text/scanner" ) type parser struct { errors errorList scanner scanner.Scanner pos scanner.Position // token position tok rune // one token look-ahead lit string // token literal } func (p *parser) next() { p.tok = p.scanner.Scan() p.pos = p.scanner.Position p.lit = p.scanner.TokenText() } func (p *parser) error(pos scanner.Position, msg string) { p.errors = append(p.errors, newError(pos, msg)) } func (p *parser) errorExpected(pos scanner.Position, msg string) { msg = `expected "` + msg + `"` if pos.Offset == p.pos.Offset { // the error happened at the current position; // make the error message more specific msg += ", found " + scanner.TokenString(p.tok) if p.tok < 0 { msg += " " + p.lit } } p.error(pos, msg) } func (p *parser) expect(tok rune) scanner.Position { pos := p.pos if p.tok != tok { p.errorExpected(pos, scanner.TokenString(tok)) } p.next() // make progress in any case return pos } func (p *parser) parseIdentifier() *Name { pos := p.pos name := p.lit p.expect(scanner.Ident) return &Name{pos, name} } func (p *parser) parseToken() *Token { pos := p.pos value := "" if p.tok == scanner.String { value, _ = strconv.Unquote(p.lit) // Unquote may fail with an error, but only if the scanner found // an illegal string in the first place. In this case the error // has already been reported. p.next() } else { p.expect(scanner.String) } return &Token{pos, value} } // ParseTerm returns nil if no term was found. func (p *parser) parseTerm() (x Expression) { pos := p.pos switch p.tok { case scanner.Ident: x = p.parseIdentifier() case scanner.String: tok := p.parseToken() x = tok const ellipsis = '…' // U+2026, the horizontal ellipsis character if p.tok == ellipsis { p.next() x = &Range{tok, p.parseToken()} } case '(': p.next() x = &Group{pos, p.parseExpression()} p.expect(')') case '[': p.next() x = &Option{pos, p.parseExpression()} p.expect(']') case '{': p.next() x = &Repetition{pos, p.parseExpression()} p.expect('}') } return x } func (p *parser) parseSequence() Expression { var list Sequence for x := p.parseTerm(); x != nil; x = p.parseTerm() { list = append(list, x) } // no need for a sequence if list.Len() < 2 switch len(list) { case 0: p.errorExpected(p.pos, "term") return &Bad{p.pos, "term expected"} case 1: return list[0] } return list } func (p *parser) parseExpression() Expression { var list Alternative for { list = append(list, p.parseSequence()) if p.tok != '|' { break } p.next() } // len(list) > 0 // no need for an Alternative node if list.Len() < 2 if len(list) == 1 { return list[0] } return list } func (p *parser) parseProduction() *Production { name := p.parseIdentifier() p.expect('=') var expr Expression if p.tok != '.' { expr = p.parseExpression() } p.expect('.') return &Production{name, expr} } func (p *parser) parse(filename string, src io.Reader) Grammar { p.scanner.Init(src) p.scanner.Filename = filename p.next() // initializes pos, tok, lit grammar := make(Grammar) for p.tok != scanner.EOF { prod := p.parseProduction() name := prod.Name.String if _, found := grammar[name]; !found { grammar[name] = prod } else { p.error(prod.Pos(), name+" declared already") } } return grammar } // Parse parses a set of EBNF productions from source src. // It returns a set of productions. Errors are reported // for incorrect syntax and if a production is declared // more than once; the filename is used only for error // positions. // func Parse(filename string, src io.Reader) (Grammar, error) { var p parser grammar := p.parse(filename, src) return grammar, p.errors.Err() }