Repository: sandstorm-io/ekam Branch: master Commit: 755967bace08 Files: 92 Total size: 470.6 KB Directory structure: gitextract_jfckbr5d/ ├── .gitignore ├── CONTRIBUTING.md ├── CONTRIBUTORS ├── LICENSE ├── Makefile ├── README.md ├── qtcreator/ │ ├── COPYING.txt │ ├── EkamDashboard.json.in │ ├── EkamDashboard.pluginspec.in │ ├── ekamdashboard.pro │ ├── ekamdashboard.qrc │ ├── ekamdashboard_global.h │ ├── ekamdashboardconstants.h │ ├── ekamdashboardplugin.cpp │ ├── ekamdashboardplugin.h │ ├── ekamtreewidget.cpp │ └── ekamtreewidget.h ├── src/ │ ├── base/ │ │ ├── Debug.cpp │ │ ├── Debug.h │ │ ├── Hash.cpp │ │ ├── Hash.h │ │ ├── OwnedPtr.cpp │ │ ├── OwnedPtr.h │ │ ├── Promise.cpp │ │ ├── Promise.h │ │ ├── Promise_test.cpp │ │ ├── Table.h │ │ ├── Table_test.cpp │ │ ├── sha256.cpp │ │ └── sha256.h │ ├── ekam/ │ │ ├── Action.cpp │ │ ├── Action.h │ │ ├── ActionUtil.cpp │ │ ├── ActionUtil.h │ │ ├── ConsoleDashboard.cpp │ │ ├── ConsoleDashboard.h │ │ ├── CppActionFactory.cpp │ │ ├── CppActionFactory.h │ │ ├── Dashboard.cpp │ │ ├── Dashboard.h │ │ ├── Driver.cpp │ │ ├── Driver.h │ │ ├── ExecPluginActionFactory.cpp │ │ ├── ExecPluginActionFactory.h │ │ ├── MuxDashboard.cpp │ │ ├── MuxDashboard.h │ │ ├── ProtoDashboard.cpp │ │ ├── ProtoDashboard.h │ │ ├── SimpleDashboard.cpp │ │ ├── SimpleDashboard.h │ │ ├── Tag.cpp │ │ ├── Tag.h │ │ ├── dashboard.capnp │ │ ├── ekam-client.cpp │ │ ├── ekam-langserve.c++ │ │ ├── ekam.cpp │ │ ├── ekam.ekam-manifest │ │ ├── initNetworkDashboardStub.cpp │ │ ├── langserve.capnp │ │ └── rules/ │ │ ├── compile.ekam-rule │ │ ├── include.ekam-rule │ │ ├── install.ekam-rule │ │ ├── intercept.c │ │ ├── intercept.ekam-rule │ │ ├── proto.ekam-rule │ │ └── test.ekam-rule │ └── os/ │ ├── ByteStream.cpp │ ├── ByteStream.h │ ├── DiskFile.cpp │ ├── DiskFile.h │ ├── EpollEventManager.cpp │ ├── EpollEventManager.h │ ├── EventGroup.cpp │ ├── EventGroup.h │ ├── EventManager.cpp │ ├── EventManager.h │ ├── File.cpp │ ├── File.h │ ├── KqueueEventManager.h │ ├── OsHandle.cpp │ ├── OsHandle.h │ ├── PollEventManager.h │ ├── Socket.cpp │ ├── Socket.h │ ├── Subprocess.cpp │ └── Subprocess.h └── vscode/ ├── .gitignore ├── LICENSE ├── README.md ├── package.json ├── src/ │ └── extension.ts └── tsconfig.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ tmp eclipse/bin .cproject .project bin lib deps ================================================ FILE: CONTRIBUTING.md ================================================ ## Contributing to Ekam ### Building Use `make continuous` to build Ekam continuously and watch for changes. For LSP support in Visual Studio Code, run `make setup-vscode`. `make continuous` requires you already have an install of Ekam. You can install Ekam by building it once (`make`) and then installing `bin/ekam{,-client,-langserve}` to a directory in PATH. Note that `g++` tends to generate spurious warnings, so you may want to use `make continuous CXX=clang++` instead. ### Overview Ekam has multiple components. - `ekam`: This is the main binary. It actually builds the code, and uses capnp to communicate with the other binaries. - `ekam-client`: This is a small tool to view the build output from `ekam`. - `ekam-langserve`: This is a Language Server Protocol client that communicates with `ekam`. - `ekam-bootstrap`: This is the same as `ekam`, but doesn't support the remote capnp protocol. As a result, some flags (like `-n`) aren't supported, and prebuilt capnp clients won't be able to connect. `ekam-bootstrap` is built directly by `make`. All other components are built by `ekam-bootstrap`. ================================================ FILE: CONTRIBUTORS ================================================ The following people have made large code contributions to this repository. Those contributions are copyright the respective authors and licensed by them under the same Apache 2.0 license terms as the rest of the library. Kenton Varda : Primary Author Google, Inc.: Employed Kenton when he wrote much of this code, and thus owns copyright. Ekam was a 20% project as is not in any way endorsed by Google. ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Copyright 2015 Sandstorm Development Group, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. .SUFFIXES: .PHONY: all install clean deps continuous setup-vscode # You may override the following vars on the command line to suit # your config. CXX=g++ CXXFLAGS=-O2 -Wall PARALLEL=$(shell nproc) # for `make continuous` EKAM=ekam define color @printf '\033[0;34m==== $1 ====\033[0m\n' endef all: bin/ekam bin/ekam: bin/ekam-bootstrap | deps $(call color,building ekam with ekam) @rm -f bin/ekam @CXX="$(CXX)" CXXFLAGS="-std=c++14 $(CXXFLAGS) -pthread" LIBS="-pthread" bin/ekam-bootstrap -j$(PARALLEL) @test -e bin/ekam && printf "=====================================================\nSUCCESS\nOutput is at bin/ekam\n=====================================================\n" # NOTE: Needs a full install of Ekam instead of the bootstrap so that LSP is available. # To avoid recompiling 3 times, this requires the user to have Ekam already installed. continuous: $(call color,building ekam with ekam continuously) @CXX="$(CXX)" CXXFLAGS="-std=c++14 $(CXXFLAGS) -pthread" LIBS="-pthread" $(EKAM) -j$(PARALLEL) -n :41315 -c setup-vscode: vscode/vscode-ekam-0.2.0.vsix code --install-extension $< vscode/vscode-ekam-0.2.0.vsix: cd vscode && npm install cd vscode && npm run package deps: deps/capnproto deps/capnproto: $(call color,downloading capnproto) @mkdir -p deps git clone https://github.com/capnproto/capnproto.git deps/capnproto SOURCES=$(shell cd src; find base os ekam -name '*.cpp' | \ grep -v KqueueEventManager | grep -v PollEventManager | \ grep -v ProtoDashboard | grep -v ekam-client | grep -v _test) HEADERS=$(shell find src/base src/os src/ekam -name '*.h') # Use a subdirectory for bootstrapping so the object files don't overlap with the Ekam build. OBJ_DIR := tmp/bootstrap OBJECTS=$(addprefix $(OBJ_DIR)/, $(SOURCES:.cpp=.o)) $(OBJ_DIR)/%.o: src/%.cpp $(HEADERS) @echo $(HEADERS) @mkdir -p $(@D) $(CXX) -Isrc -fPIC -std=c++14 -pthread -o $@ -c $< bin/ekam-bootstrap: $(OBJECTS) $(call color,compiling bootstrap ekam) @mkdir -p bin $(CXX) -Isrc -std=c++14 -pthread $(OBJECTS) -o $@ clean: rm -rf bin lib tmp $(OBJ_DIR) ================================================ FILE: README.md ================================================ # Ekam Build System Ekam ("make" backwards) is a build system which automatically figures out what to build and how to build it purely based on the source code. No separate "makefile" is needed. Ekam works by exploration. For example, when it encounters a file ending in ".cpp", it tries to compile the file, intercepting system calls to find out its dependencies (e.g. included headers). If some of these are missing, Ekam continues to explore until it finds headers matching them. When Ekam builds an object file and discovers that it contains a "main" symbol, it tries to link it, searching for other object files to satisfy all symbol references therein. When Ekam sees a test, it runs the test, again intercepting system calls to dynamically discover what inputs the test needs (which may not have been built yet). Ekam can be extended to understand new file types by writing simple shell scripts telling it what to do with them. Thus Ekam is, in fact, Make in reverse. Make starts by reading a Makefile, sees what executables it wants to build, then from there figures out what source files need to be compiled to link into them, then compiles them. Ekam starts by looking for source files to compile, then determines what executables it can build from them, and, in the end, might output a Makefile describing what it did. Ekam is a work in progress. ## Building Ekam ### Warning Ekam is an experimental project that is not ready for wide use. That said, I (Kenton) have successfully used Ekam as my primary build system throughout the development of [Cap'n Proto](https://capnproto.org), [Sandstorm](https://sandstorm.io), and the [Cloudflare Workers](https://developers.cloudflare.com/workers/about/) runtime. ### Supported Platforms At this time, Ekam only runs on Linux. It requires GCC 5+ or Clang 4+, as it uses C++14 features. In the past, Ekam worked on FreeBSD and Max OSX, but the code to support that atrophied and was eventually deleted. Ekam uses a lot of OS-specific hacks and so is unlikely to work on any platform which is not explicitly supported. We'd like to see other platforms supported, but Ekam's primary user and maintainer right now ([Sandstorm.io](https://sandstorm.io)) is itself highly Linux-specific, so there is not much pressure. (Let us know if you want to help.) ### Bootstrapping the build Download and compile Ekam like so: git clone https://github.com/capnproto/ekam.git cd ekam make If successful, Ekam should have built itself, with the output binary at "bin/ekam". Yes, we use make in order to bootstrap Ekam, mostly just because it's slightly nicer than a shell script. ### Compiling Ekam with Ekam Compiling Ekam requires the GCC flag `-std=gnu++0x` to enable C++11 features, but currently there is no way for the code itself to specify compiler flags that it requires. You can only specify them via environment variable. So, to build Ekam with Ekam, type this command at the top of the Ekam repository: CXXFLAGS=-std=gnu++0x ekam -j4 The `-j4` tells Ekam to run up to four tasks at once. You may want to adjust this number depending on how many CPU cores you have. Note that Ekam looks for a directory called `src` within the current directory, and scans it for source code. The Ekam source repository is already set up with such a `src` subdirectory containing the Ekam code. You could, however, place the entire Ekam repository _inside_ some other directory called `src`, and then run Ekam from the directory above that, and it will still find the code. The Protocol Buffers instructions below will take advantage of this to create a directory tree containing both Ekam and protobufs. Ekam places its output in siblings of `src` called `tmp` (for intermediate files), `bin` (for output binaries), `lib` (for output libraries, although currently Ekam doesn't support building libraries), etc. These are intended to model Unix directory tree conventions. ## Continuous Building If you invoke Ekam with the `-c` option, it will watch the source tree for changes and rebuild derived files as needed. In this way, you can simply leave Ekam running while you work on your code, and get information about errors almost immediately on saving. Note that continuous building is the only way to do incremental builds with Ekam -- any time you run a new Ekam process, it always starts from scratch. I generally just leave Ekam running in a console window 24/7. ## IDE plugins and other external clients Ekam can, while running, export a network interface which allows other programs to query the state of the build, including receiving the task tree and error logs. ### Ekam Server If you pass the `-n` flag to Ekam, it will listen for connections on a port and stream build status updates to anyone who connects. Invoke like: ekam -n :41315 ### Ekam Client `ekam-client` is a very simple program that prints an Ekam build status stream to the console exactly as Ekam itself does. Currently `ekam-client` doesn't actually know how to create a network connection but instead reads the stream from standard input, so you can invoke it like this: nc localhost 41315 | ekam-client `ekam-client` is mostly just a tech demo, since it displays the same info that is already visible in the console where Ekam itself is running. ### Visual Studio Code Plugin The `vscode` directory in your Ekam repository contains source code for a [Visual Studio Code](https://code.visualstudio.com/) plugin. [See its README for more info.](vscode/README.md) ### Qt Creator Plugin The `qtcreator` directory in the Ekam repository contains source code for a [Qt Creator](http://qt-project.org/wiki/Category:Tools::QtCreator) plugin. Qt Creator is an open source C++ IDE that, despite its name, is quite nice even for non-Qt code. To build the Qt Creator plugin, you'll want to download the Qt Creator source code release and build it from scratch. Then, go into the `qtcreator` directory in the Ekam repo and do the following: export QTC_SOURCE= qmake make This should build the plugin and install it directly into your Qt Creator build. Now you can start Qt Creator and choose the "Ekam Actions" view in the navigation frame. The plugin connects to a local Ekam instance running on port 41315; so, start ekam like this: ekam -c -n :41315 The plugin will also add error markers to you source code, visible in the issue tracker. Double-clicking on a failed rule in the tree view will navigate the issues view to the first message from that action, which you can in turn use to navigate to the source location of the error. ## Using Ekam in your own code ### Preferred project layout An Ekam project directory must contain a directory called `src` which contains all source code. Ekam will ignore everything other than this `src` directory. Ekam will create sibling directories called `tmp`, `bin`, and `lib`; you probably shouldn't have any files in those directories to start, because it's nice to be able to `rm -rf` them to clean. ### Import the rule files In order for Ekam to be able to build a project, it needs to find its rule files, which are under `src/ekam/rules` in Ekam's own source code. You will want to symlink this directory into your project's source tree somewhere. It does not have to be anywhere in particular, but `src/ekam-rules` is probably a good bet. ### Compiler flags You can set the following environment variables to control how Ekam compiles your code: * `CXX`: Sets the C++ compiler, e.g. `CXX=clang++`. * `CXXFLAGS`: Sets C++ compilation flags, e.g. `CXXFLAGS=-std=c++11 -O2 -Wall`. * `LIBS`: Sets linker flags, e.g. `LIBS=-lsodium -lz` ### Building binaries When Ekam finds or compiles a `.o` file that defines the symbol `main`, it will attempt to link the file with other `.o` files defining all needed symbols (transitively) in order to produce a binary. The binary takes the name of the `.o` with the extension removed, e.g. `foo.o` produces a binary `foo`. The binary is initially just dropped into the `tmp` directory, which mirrors the `src` directory. For example, `src/foo/bar.c++` will be compiled to `src/foo/bar.o`, which will link (if it defines `main`) into `src/foo/bar`. If you'd like for the binary to be output to the `bin` directory, you must declare a manifest file with the extension `.ekam-manifest`. This file is a simple text file where each line names a file and its output directory. For example: bar bin This says: "Once you've built `bar` (within this directory), copy it into `bin`." You can also choose to rename the file when installing: bar bin/baz This says: "Once you've built `bar` (within this directory), copy it to `bin/baz`." ### Building libraries Currently, Ekam does not support building libraries. This seems complicated to support since there's no automated way for Ekam to decide what makes a good library. You'd need to declare some set of modules representing the public interface, which is a bit sad. For now, Ekam is suitable for projects where the final output is a binary. A "library" in Ekam is just a directory that contains some code. It does not build to a `.a` or `.so` file; instead, the objects may be directly linked into binaries found in other directories as needed. ### Handling Magic Singleton Registries There is a common pattern in C++ code in which a particular file's symbols are not referenced from other files, but instead the file registers itself in some global registry at program startup which makes its functionality discoverable to the rest of the program. By default, this pattern does not work with Ekam, because Ekam has no way to tell which of these magic self-registering files should be linked into which binaries, since their symbols are not referenced from other `.o` files. To solve this problem, don't. This design pattern sucks and you should not use it. Rewrite your code so that its functionality is not dependent on what objects were specified on the link line. For example, have your `main()` function explicitly call `registerFoo()`, `registerBar()`, etc., for all the magic modules you want registered, ideally passing a registry object to each function so you don't need a singleton registry. See [Singletons Considered Harmful](http://www.object-oriented-security.org/lets-argue/singletons) for extended discussion. (We make a special exception for test frameworks, described below.) ### Debugging If you want `gdb` to be able to find your code when debugging Ekam-built binaries, you should set up a couple symlinks as follows: mkdir ekam-provider ln -s ../src ekam-provider/canonical ln -s ../src ekam-provider/c++header To understand the reason for these symlinks, see the explanation of `intercept.so` later in this document. ### Tests If Ekam builds a binary which ends in `-test` or `_test`, it will run that binary and report pass/fail depending on the exit status. Passing tests are marked in green in the output (whereas regular successful build actions are blue) in order to help you develop a pavlovian attachment to writing tests and seeing them pass. Ekam has additional built-in support for two test frameworks: Google Test and KJ tests. When Ekam compiles a source file which declares test cases using one of these frameworks (e.g. using Google Test's `TEST` or `TEST_F` macros, or KJ's `KJ_TEST` macro) but without a `main` function, it will automatically link against the respective framework's test runner (which supplies `main`) in order to produce a test binary. Note that the detection of test declarations is based on linker symbols, not on scanning the source code, so don't worry if you've declared your own wrapper macros. In order for Google Test or KJ test integration to work, the respective test framework's code must be in your source tree as a dependency (see below). Note that tests are run with `intercept.so` injected, which has implications if your test does any filesystem access. See the explanation of `intercept.so` later in this document. ### Dependencies If your project depends on other projects, and you want to build those other projects as part of your own build (rather than require the user to install the libraries on their system), you should follow the pattern that Ekam itself does. The basic idea is to have a directory called `deps` into which you download the repositories of your external dependencies. Then, under `src`, place symlinks that deep-link into `deps` and pull out the code you want. For example, to depend on Cap'n Proto, you would do something like: mkdir deps git clone https://github.com/sandstorm-io/capnproto.git deps/capnproto ln -s ../deps/capnproto/c++/src/{kj,capnp} src Note that `deps` should not be checked into your repository. Instead, you should provide a script that people run after cloning your repo that downloads dependencies. For Ekam's own build, we handle this in the Makefile, so you can look at that for an example. We may eventually add functionality to Ekam to handle dependencies so that you no longer need a Makefile for this. **Now, here's the fun part:** If you work on a lot of different projects, instead of nesting dependencies as described above, you can symlink `deps` to `..`, such that all your git clones are siblings: rm -rf deps ln -s .. deps You can do this with Ekam, for instance, if you happen to have Cap'n Proto cloned as a sibling to Ekam. ### Non-Ekam-clean Dependencies If your dependency is not organized for Ekam, you might need to link to its top-level directory rather than a source subdirectory. For instance, Google Test separates its source code into `src` and `include` directories, so if you only link in its `src` you'll miss the headers. Instead, link in the whole repo like: ln -s ../../gtest src When Ekam sees a directory named `include`, it adds that directory to the include path for all other code it compiles. Thus the Google Test headers will end up includable by your code. Note that projects imported this way will usually have a bunch of errors reported since they are not Ekam-clean. However, often (such as in the case of Google Test) the important parts will compile well enough to use. ### Per-directory compile options To specify some additional compiler options that should apply within a particular directory (and all children, recursively), you may create a file called `compile.ekam-flags` and populate it with simple variable assignments in shell syntax. For example: # Define FOO to 1 when compiling code in this directory. CXXFLAGS=$CXXFLAGS -DFOO=1 `.ekam-flags` files are actually executed using `/bin/sh` just before invoking the compiler. When multiple flags files are in-scope, the flags file in the outermost directory runs first. ### Cross-compiling Ekam currently supports cross-compiling to multiple target architectures at once, by listing additional (non-host) architectures in the `CROSS_TARGETS` environment variable: CROSS_TARGETS="aarch64-linux-gnu" ekam Notes: * This is designed to work e.g. with the `crossbuild-essential-*` Debian packages. * Ekam always compiles for the host architecture in addition to these targets. This is to support building tools like the Cap'n Proto compiler and then immediately using them in the same build. * Ekam assumes the inter-object dependencies are the same on all targets. This tends to mean that it work well for targetting alternate CPU architectures, but not as well for targeting other operating systems. * You may specify target-specific CXXFLAS and LIBS like `CXXFLAGS_aarch64_linux_gnu` and `LIBS_aarch64_linux_gnu`. If present, these completely replace the default `CXXFLAGS` and `LIBS`. * If any unit tests are built, Ekam will try to use qemu to run them. ## Custom Rules You may teach Ekam how to handle a new type of file -- or introduce a one-off build rule not triggered by any file -- by creating a rule file. A rule file is any executable with the extension `.ekam-rule`. Often, they are shell scripts, but this is not a requirement. Rule files can themselves be the output of other rules, so you could compile a C++ program that acts as a rule. Ekam executes custom rules and then communicates with them on stdin/stdout using a simple line-based text protocol described below. The rule may also write error logs intended for the user to stderr. When Ekam encounters an executable with the `.ekam-rule` extension, it first runs it with no arguments in order to "learn" it. At that time, the program may tell ekam what kinds of files it should trigger on using the `trigger` command (below). When Ekam encounters a trigger file, it will execute the rule again with the trigger file's canonical name as the argument. Alternatively, if you just want to implement a one-off action, then the rule may simply perform that action at "learn" time without registering any triggers. ### Canonical file names A file's "canonical" name is the name without the `src/` or `tmp/` prefix. Thus canonical names do not distinguish between source files and build outputs. No two files can have the same canonical name -- it is an error for a build action to output a file which exists in the `src` directory, or for two actions to produce the same file under `tmp`. You can ask Ekam to map a canonical name to a physical disk name using the `findInput` command. ### Tags Ekam rules are triggered by tags. A tag is a simple string, conventionally of the format `type:value`. Tags are applied to files. For example, a `.o` file declaring the symbol `main` might be given the tag `c++symbol:main`. Rules may assign new tags to any file (source or output). Ekam has some default rules that assign tags meant for broad use: * `canonical:`: Each file receives this tag, where `` is its canonical name. * `filetype:`: Each regular file that has a file type extension receives this tag, e.g. `filetype:.c++`. * `directory:*`: Each directory receives this tag. ### Commands A rule may perform the following commands by writing them to standard output. * `trigger `: Used during the learning phase to tell Ekam that the rule should be executed on any file tagged with ``. * `verb `: Use during the learning phase to tell Ekam the rule's "verb", which is what is displayed to the user when the rule later runs. This should be a simple, descriptive word. For instance, for a C++ compile action, the verb is `compile`. * `silent`: Use during the learning phase to indicate that when this command later runs, it should not be reported to the user unless it fails. Use this to reduce noise caused by very simple commands that perform trivial actions. * `findInput `: Obtains the canonical name of the given file. Ekam will reply by writing one line to the rule's standard input containing the full disk path of the file (e.g. including `src/` or `tmp/`). Ekam will remember that the build action depended on this file, so if the file changes, the action will be re-run. If no match was found, Ekam will return a blank line. * `findProvider `: Find a file tagged with ``. If there are multiple matches, Ekam heuristically chooses the "preferred" one, which generally means the one closest in the directory tree to the file which triggered the rule. The path is returned as with `findInput`. Also as with `findInput`, the file is considered a dependency of the action. Ekam will re-run this action if the file changes *or* if the file Ekam chose to match `` changes. * `findModifiers `: Search for the file `` in the trigger file's directory and every parent up to the source root. For each place that it is found (in order starting from the greatest ancestor), return the full disk path and mark it as an input. After returning all results, return a blank line to indicate the end of the list. This command is intended for finding "modifier" files which specify options that should apply within a particular directory. For instance, `compile.ekam-flags` is implemented this way. * `noteInput `: Tells Ekam that the action depends on ``, which is a path outside of the project's source tree. For instance, `/usr/include/stdlib.h`. Currently Ekam ignores this, but in theory it could watch these files and re-run the action if they change. * `newOutput `: Create a new output file with the given canonical name. Ekam replies by writing the on-disk path where the file should be created to the rule's standard input. * `provide `: Tag `` (a canonical name) with ``. The file must be a known input our output of this rule; i.e. it must have been the subeject of a previous call to `findInput`, `findProvider`, or `newOutput`. * `install `: Take the canonical filename `` and copy it to ``, where `` should start with `bin/`, `lib/`, etc. * `passed`: Indicate that this action ran a test, and the test passed. ### `intercept.so` Sometimes, it's hard to know what a build tool's exact inputs and outputs will be ahead of time. For instance, a C++ compiler run will need to input all of the header files `#include`ed by the source file. There's no reasonable way to know what these might be in advance, much less look up the locations of files to satisfy each. To solve this, Ekam implements a library which can be injected into a tool via `LD_PRELOAD` in order to intercept filesystem calls, automatically issues the proper Ekam commands to register inputs and outputs, and then map the call to the real disk path. The interceptor library is implemented in `src/ekam/rules/intercept.c` and built by `src/ekam/rules/intercept.ekam-rule`. You may request it by requesting the tag `special:ekam-interceptor`, which maps to the compiled `intercept.so`, which you may then inject into an arbitrary command using `LD_PRELOAD`. See `src/ekam/rules/compile.ekam-rule` to see how this is done with the C++ compiler. When filesystem calls are being intercepted, an attempt to open a regular filename will be heuristically mapped to the closest file whose canonical name contains the full requested name as a suffix. For example, when processing `src/foo/bar/baz`, if the tool tries to open `qux/corge`, this could map to `src/foo/bar/qux/corge` or `src/qux/corge` or `src/grault/qux/corge`, but **not** `src/grault/corge` nor `src/qux/corge/grault` nor `src/corge`. If you want to open a file by tag, the special virtual path `/ekam-provider//` can be used. For example, since the `include.ekam-rule` rule tags all C++ header files with `c++header:`, when we invoke the compiler using the inteceptor, we pass the flag `-I/ekam-provider/c++header`. If you want to open a file purely by its whole canonical path (not using the heuristic that finds nearby files), you may do so by opening `/ekam-provider/canonical/`, since as described above every file gets tagged with `canonical:`. ## Get Involved Have a question about Ekam, or want to contribute? Talk to us on the [Ekam discussion group](https://groups.google.com/group/ekam-tool). Ekam is currently developed as part of the [Sandstorm.io](https://sandstorm.io) project and is primarily used by Sandstorm. If you like Ekam, consider [getting involved with Sandstorm](https://github.com/sandstorm-io/sandstorm/wiki/Get-Involved). ================================================ FILE: qtcreator/COPYING.txt ================================================ Code in this directory is licensed under Apache 2.0 like normal. However, the icons under images/ are from the Qt Creator source code, and are thus subject to Qt Creator's copyright and license. This probably means that the compiled plugin cannot legally be redistributed because it combines incompatible licenses (Apache vs. LGPL). I suppose I should either replace the icons or relicense the code. Let me know if anyone actually cares. ================================================ FILE: qtcreator/EkamDashboard.json.in ================================================ { \"Name\" : \"EkamDashboard\", \"Version\" : \"0.0.1\", \"CompatVersion\" : \"0.0.1\", \"Vendor\" : \"KentonVarda\", \"Copyright\" : \"(C) Kenton Varda and Google Inc.\", \"License\" : \"Apache License v2.0\", \"Category\" : \"Qt Creator\", \"Description\" : \"Connects to the Ekam continuous build daemon and displays error markers.\", \"Url\" : \"http://ekam.googlecode.com\", $$dependencyList } ================================================ FILE: qtcreator/EkamDashboard.pluginspec.in ================================================ KentonVarda (C) Google Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Connects to the Ekam continuous build daemon and displays error markers. http://ekam.googlecode.com $$dependencyList ================================================ FILE: qtcreator/ekamdashboard.pro ================================================ # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. QMAKE_CXXFLAGS += -std=c++11 `pkg-config --cflags capnp` QMAKE_EXT_CPP += .c++ TARGET = EkamDashboard TEMPLATE = lib DEFINES += EKAMDASHBOARD_LIBRARY # EkamDashboard files SOURCES += ekamdashboardplugin.cpp \ ekamtreewidget.cpp HEADERS += ekamdashboardplugin.h\ ekamdashboard_global.h\ ekamdashboardconstants.h \ ekamtreewidget.h CAPNPS += dashboard.capnp INCLUDEPATH += . capnp_header.name = capnproto header capnp_header.input = CAPNPS capnp_header.output = ${QMAKE_FILE_BASE}.capnp.h capnp_header.commands = \ capnp compile -oc++ --src-prefix=`dirname ${QMAKE_FILE_NAME}` ${QMAKE_FILE_NAME}; \ mv ${QMAKE_FILE_BASE}.capnp.c++ ${QMAKE_FILE_BASE}.capnp-noautolink.c++ capnp_header.variable_out = GENERATED_FILES QMAKE_EXTRA_COMPILERS += capnp_header capnp_src.name = capnproto src capnp_src.input = CAPNPS capnp_src.output = ${QMAKE_FILE_BASE}.capnp-noautolink.c++ capnp_src.depends = ${QMAKE_FILE_BASE}.capnp.h capnp_src.commands = true capnp_src.variable_out = GENERATED_SOURCES QMAKE_EXTRA_COMPILERS += capnp_src # Qt Creator linking ## set the QTC_SOURCE environment variable to override the setting here QTCREATOR_SOURCES = $$(QTC_SOURCE) isEmpty(QTCREATOR_SOURCES):error("Please set QTC_SOURCE to your Qt Creator source directory.") ## set the QTC_BUILD environment variable to override the setting here IDE_BUILD_TREE = $$(QTC_BUILD) isEmpty(IDE_BUILD_TREE):IDE_BUILD_TREE=$$(QTC_SOURCE) ## uncomment to build plugin into user config directory ## /plugins/ ## where is e.g. ## "%LOCALAPPDATA%\Nokia\qtcreator" on Windows Vista and later ## "$XDG_DATA_HOME/Nokia/qtcreator" or "~/.local/share/data/Nokia/qtcreator" on Linux ## "~/Library/Application Support/Nokia/Qt Creator" on Mac # USE_USER_DESTDIR = yes PROVIDER = KentonVarda ###### If the plugin can be depended upon by other plugins, this code needs to be outsourced to ###### _dependencies.pri, where is the name of the directory containing the ###### plugin's sources. QTC_PLUGIN_NAME = EkamDashboard QTC_LIB_DEPENDS += \ # nothing here at this time QTC_PLUGIN_DEPENDS += \ coreplugin \ projectexplorer \ cpptools \ texteditor QTC_PLUGIN_RECOMMENDS += \ # optional plugin dependencies. nothing here at this time ###### End _dependencies.pri contents ###### QT += network include($$QTCREATOR_SOURCES/src/qtcreatorplugin.pri) LIBS += -L$$IDE_PLUGIN_PATH/Nokia `pkg-config --libs capnp-rpc` RESOURCES += \ ekamdashboard.qrc ================================================ FILE: qtcreator/ekamdashboard.qrc ================================================ images/state-blocked.png images/state-deleted.png images/state-done.png images/state-failed.png images/state-passed.png images/state-pending.png images/state-running.png images/dir.png images/dir-blocked.png images/dir-deleted.png images/dir-done.png images/dir-failed.png images/dir-passed.png images/dir-pending.png images/dir-running.png ================================================ FILE: qtcreator/ekamdashboard_global.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef EKAMDASHBOARD_GLOBAL_H #define EKAMDASHBOARD_GLOBAL_H #include #if defined(EKAMDASHBOARD_LIBRARY) # define EKAMDASHBOARDSHARED_EXPORT Q_DECL_EXPORT #else # define EKAMDASHBOARDSHARED_EXPORT Q_DECL_IMPORT #endif #endif // EKAMDASHBOARD_GLOBAL_H ================================================ FILE: qtcreator/ekamdashboardconstants.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef EKAMDASHBOARDCONSTANTS_H #define EKAMDASHBOARDCONSTANTS_H namespace EkamDashboard { namespace Constants { const char ACTION_ID[] = "EkamDashboard.Action"; const char MENU_ID[] = "EkamDashboard.Menu"; const char TASK_CATEGORY_ID[] = "EkamDashboard.Menu"; } // namespace EkamDashboard } // namespace Constants #endif // EKAMDASHBOARDCONSTANTS_H ================================================ FILE: qtcreator/ekamdashboardplugin.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "ekamdashboardplugin.h" #include "ekamdashboardconstants.h" #include "ekamtreewidget.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace EkamDashboard { namespace Internal { QString toQString(kj::ArrayPtr str) { return QString::fromUtf8(str.begin(), str.size()); } class EkamDashboardPlugin::FakeAsyncInput final: public kj::AsyncInputStream { public: kj::Promise read(void* buffer, size_t minBytes, size_t maxBytes) override { return tryRead(buffer, minBytes, maxBytes).then([=](size_t result) { KJ_REQUIRE(result >= minBytes, "Premature EOF") { // Pretend we read zeros from the input. memset(reinterpret_cast(buffer) + result, 0, minBytes - result); return minBytes; } return result; }); } kj::Promise tryRead(void* buffer, size_t minBytes, size_t maxBytes) override { Request request(buffer, minBytes, maxBytes); if (byteQueue.size() > 0) { KJ_IF_MAYBE(size, request.consumeFrom(byteQueue)) { return *size; } } requests.push_back(kj::mv(request)); return requests.back().finishLater(); } void add(QByteArray bytes) { while (!requests.empty()) { auto& request = requests.front(); KJ_IF_MAYBE(size, request.consumeFrom(bytes)) { request.fulfill(*size); requests.pop_front(); } else { // Not enough bytes to satisfy the request. return; } } byteQueue.append(bytes); } private: class Request { public: Request(void* buffer, size_t minBytes, size_t maxBytes) : pos(reinterpret_cast(buffer)), minLeft(minBytes), maxLeft(maxBytes), alreadyRead(0) {} kj::Maybe consumeFrom(QByteArray& bytes) { size_t n = kj::min(maxLeft, bytes.size()); memcpy(pos, bytes.data(), n); bytes.remove(0, n); if (n >= minLeft) { return alreadyRead + n; } else { pos += n; minLeft -= n; maxLeft -= n; alreadyRead += n; return nullptr; } } kj::Promise finishLater() { auto paf = kj::newPromiseAndFulfiller(); fulfiller = kj::mv(paf.fulfiller); return kj::mv(paf.promise); } void fulfill(size_t amount) { fulfiller->fulfill(kj::mv(amount)); } private: kj::byte* pos; size_t minLeft; size_t maxLeft; size_t alreadyRead; kj::Own> fulfiller; }; std::deque requests; QByteArray byteQueue; }; EkamDashboardPlugin::EkamDashboardPlugin() : hub(0), socket(0), seenHeader(false), waitScope(eventLoop), fakeInput(kj::heap()), readTask(nullptr) {} EkamDashboardPlugin::~EkamDashboardPlugin() noexcept { // Unregister objects from the plugin manager's object pool // Delete members } bool EkamDashboardPlugin::initialize(const QStringList &arguments, QString *errorString) { // Register objects in the plugin manager's object pool // Load settings // Add actions to menus // Connect to other plugins' signals // In the initialize method, a plugin can be sure that the plugins it // depends on have initialized their members. Q_UNUSED(arguments) Q_UNUSED(errorString) Core::ActionManager *am = Core::ActionManager::instance(); QAction *action = new QAction(tr("EkamDashboard action"), this); Core::Command *cmd = am->registerAction(action, Constants::ACTION_ID, Core::Context(Core::Constants::C_GLOBAL)); cmd->setDefaultKeySequence(QKeySequence(tr("Ctrl+Alt+Meta+A"))); connect(action, SIGNAL(triggered()), this, SLOT(triggerAction())); Core::ActionContainer *menu = am->createMenu(Constants::MENU_ID); menu->menu()->setTitle(tr("EkamDashboard")); menu->addAction(cmd); am->actionContainer(Core::Constants::M_TOOLS)->addMenu(menu); addAutoReleasedObject(new EkamTreeWidgetFactory(this)); return true; } void EkamDashboardPlugin::extensionsInitialized() { // Retrieve objects from the plugin manager's object pool // In the extensionsInitialized method, a plugin can be sure that all // plugins that depend on it are completely initialized. ExtensionSystem::PluginManager *pm = ExtensionSystem::PluginManager::instance(); hub = pm->getObject(); hub->addCategory(Core::Id(Constants::TASK_CATEGORY_ID), QLatin1String("Ekam task")); socket = new QTcpSocket(this); connect(socket, SIGNAL(error(QAbstractSocket::SocketError)), this, SLOT(socketError(QAbstractSocket::SocketError))); connect(socket, SIGNAL(readyRead()), this, SLOT(socketReady())); tryConnect(); } ExtensionSystem::IPlugin::ShutdownFlag EkamDashboardPlugin::aboutToShutdown() { // Save settings // Disconnect from signals that are not needed during shutdown // Hide UI (if you add UI that is not in the main window directly) delete socket; socket = 0; clearActions(); return SynchronousShutdown; } void EkamDashboardPlugin::triggerAction() { // qDebug() << "triggerAction() qdebug"; QMessageBox::information(Core::ICore::mainWindow(), tr("Action triggered"), tr("This is an action from EkamDashboard.")); hub->addTask(ProjectExplorer::Task( ProjectExplorer::Task::Error, QLatin1String("test error"), Utils::FileName::fromUserInput(QLatin1String("/home/kenton/code/src/base/OwnedPtr.h")), 10, Core::Id(Constants::TASK_CATEGORY_ID))); } void EkamDashboardPlugin::socketError(QAbstractSocket::SocketError error) { // qDebug() << "Socket error: " << error; reset(); } void EkamDashboardPlugin::reset() { if (!resetting) { resetting = true; clearActions(); QTimer::singleShot(5000, this, SLOT(retryConnection())); } } void EkamDashboardPlugin::retryConnection() { resetting = false; // Cancel any async parsing still happening. readTask = nullptr; // Reset the fake input to clear out its buffers. fakeInput = kj::heap(); tryConnect(); } void EkamDashboardPlugin::socketReady() { fakeInput->add(socket->readAll()); eventLoop.run(); } void EkamDashboardPlugin::tryConnect() { seenHeader = false; // qDebug() << "Trying to connect..."; socket->connectToHost(QLatin1String("localhost"), 41315); readTask = messageLoop().eagerlyEvaluate( [this](kj::Exception&& e){ KJ_LOG(ERROR, e); reset(); }); } kj::Promise EkamDashboardPlugin::messageLoop() { return capnp::readMessage(*fakeInput).then([this](kj::Own&& message) { if (!seenHeader) { seenHeader = true; ekam::proto::Header::Reader header = message->getRoot(); // qDebug() << "Received header: " << kj::str(header).cStr(); projectRoot = toQString(header.getProjectRoot()); } else { ekam::proto::TaskUpdate::Reader update = message->getRoot(); // qDebug() << "Received task update: " << kj::str(update).cStr(); ActionState*& slot = actions[update.getId()]; if (slot == 0) { slot = new ActionState(this, update); } else { slot->applyUpdate(update); } if (slot->isDead()) { delete slot; actions.remove(update.getId()); } } return messageLoop(); }); } QString EkamDashboardPlugin::findFile(const QString& canonicalPath) { QString srcpath = projectRoot + QLatin1String("/src/") + canonicalPath; QString tmppath = projectRoot + QLatin1String("/tmp/") + canonicalPath; if (QFile::exists(srcpath)) { return srcpath; } else if (QFile::exists(tmppath)) { return tmppath; } else { return canonicalPath; } } QList EkamDashboardPlugin::allActions() { QList result; foreach (ActionState* action, actions) { if (!action->isHidden()) { result << action; } } return result; } void EkamDashboardPlugin::clearActions() { foreach (ActionState* action, actions) { delete action; } actions.clear(); } // ======================================================================================= ActionState::ActionState( EkamDashboardPlugin *plugin, ekam::proto::TaskUpdate::Reader initialUpdate) : plugin(plugin), state(initialUpdate.getState()), verb(toQString(initialUpdate.getVerb())), noun(toQString(initialUpdate.getNoun())), path(plugin->findFile(noun)), silent(initialUpdate.getSilent()) { auto log = initialUpdate.getLog(); if (log != nullptr) { consumeLog(log); } if (!isHidden()) { plugin->unhideAction(this); } } ActionState::~ActionState() noexcept { emit removed(); clearTasks(); } void ActionState::applyUpdate(ekam::proto::TaskUpdate::Reader update) { if (update.getState() != ekam::proto::TaskUpdate::State::UNCHANGED && update.getState() != state) { bool wasHidden = isHidden(); // Invalidate log when the task is deleted or it is re-running or scheduled to re-run. if (update.getState() == ekam::proto::TaskUpdate::State::PENDING || update.getState() == ekam::proto::TaskUpdate::State::RUNNING || update.getState() == ekam::proto::TaskUpdate::State::DELETED) { clearTasks(); } state = update.getState(); if (isHidden()) { if (wasHidden) { emit removed(); } } else { if (wasHidden) { plugin->unhideAction(this); } else { emit stateChanged(state); } } } auto log = update.getLog(); if (log != nullptr) { consumeLog(log); } if (state != ekam::proto::TaskUpdate::State::RUNNING && !leftoverLog.empty()) { parseLogLine(toQString(leftoverLog)); leftoverLog.resize(0); } } void ActionState::clearTasks() { emit clearedTasks(); foreach (const ProjectExplorer::Task& task, tasks) { plugin->taskHub()->removeTask(task); } tasks.clear(); leftoverLog.resize(0); } void ActionState::consumeLog(kj::StringPtr log) { while (true) { KJ_IF_MAYBE(pos, log.findFirst('\n')) { leftoverLog.addAll(log.begin(), log.begin() + *pos); log = log.slice(*pos + 1); parseLogLine(toQString(leftoverLog)); leftoverLog.resize(0); } else { leftoverLog.addAll(log); return; } } } void ActionState::parseLogLine(QString line) { if (tasks.size() > 100) return; // avoid performance problems with too many tasks static const QRegExp FILE(QLatin1String("^([^ :]+):(.*)")); static const QRegExp INDEX(QLatin1String("^([0-9]+):(.*)")); static const QRegExp WARNING(QLatin1String("(.*[^a-zA-Z0-9])?warning:(.*)"), Qt::CaseInsensitive); static const QRegExp ERROR(QLatin1String("(.*[^a-zA-Z0-9])?error:(.*)"), Qt::CaseInsensitive); static const QRegExp FAILURE(QLatin1String(" *failure *"), Qt::CaseInsensitive); static const QRegExp FULL_LOG(QLatin1String("full log: tmp/(.*)")); ProjectExplorer::Task::TaskType type = ProjectExplorer::Task::Unknown; QString file; int lineNo = -1; int columnNo = -1; // OMGWTF matching a QRegExp modifies the QRegExp object rather than returning some sort of match // object, so we must make copies. Hopefully using the copy constructor rather than constructing // directly from the pattern strings means they won't be re-compiled every time. QRegExp fullLog = FULL_LOG; QRegExp fileRe = FILE; if (fullLog.exactMatch(line)) { file = fullLog.capturedTexts()[1]; file = plugin->findFile(file); if (!file.startsWith(QLatin1Char('/'))) { file.clear(); } } else if (fileRe.exactMatch(line)) { file = fileRe.capturedTexts()[1]; if (file.startsWith(QLatin1String("/ekam-provider/c++header/"))) { file.remove(0, strlen("/ekam-provider/c++header/")); } else if (file.startsWith(QLatin1String("/ekam-provider/canonical/"))) { file.remove(0, strlen("/ekam-provider/canonical/")); } file = plugin->findFile(file); if (file.startsWith(QLatin1Char('/'))) { line = fileRe.capturedTexts()[2]; } else { // Failed to find the file on disk. Maybe it's not actually a file. Leave it in the error // text. file.clear(); } } QRegExp indexRe = INDEX; if (indexRe.exactMatch(line)) { type = ProjectExplorer::Task::Unknown; lineNo = indexRe.capturedTexts()[1].toInt(); line = indexRe.capturedTexts()[2]; if (indexRe.exactMatch(line)) { columnNo = indexRe.capturedTexts()[1].toInt(); line = indexRe.capturedTexts()[2]; } } QRegExp errorRe = ERROR; QRegExp warningRe = WARNING; QRegExp failureRe = FAILURE; if (errorRe.exactMatch(line)) { type = ProjectExplorer::Task::Error; } else if (warningRe.exactMatch(line)) { type = ProjectExplorer::Task::Warning; } else if (failureRe.exactMatch(line)) { type = ProjectExplorer::Task::Error; } // Qt Creator tasks don't support column numbers, so add it back into the error text. if (columnNo != -1) { line = QLatin1String("(col ") + QString::number(columnNo) + QLatin1String(") ") + line; } tasks.append(ProjectExplorer::Task(type, line, Utils::FileName::fromUserInput(file), lineNo, Core::Id(Constants::TASK_CATEGORY_ID))); plugin->taskHub()->addTask(tasks.back()); emit addedTask(tasks.back()); } } // namespace Internal } // namespace EkamDashboard ================================================ FILE: qtcreator/ekamdashboardplugin.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef EKAMDASHBOARD_H #define EKAMDASHBOARD_H #include "ekamdashboard_global.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "dashboard.capnp.h" namespace ProjectExplorer { class TaskHub; } namespace EkamDashboard { namespace Internal { class EkamDashboardPlugin; class ActionState: public QObject { Q_OBJECT public: ActionState(EkamDashboardPlugin* plugin, ekam::proto::TaskUpdate::Reader initialUpdate); ~ActionState() noexcept; // Returns true if the action went from silent to non-silent, and thus a newAction event should // be fired. void applyUpdate(ekam::proto::TaskUpdate::Reader update); bool isDead() { return state == ekam::proto::TaskUpdate::State::DELETED; } ekam::proto::TaskUpdate::State getState() { return state; } const QString& getVerb() { return verb; } const QString& getNoun() { return noun; } const QString& getPath() { return path; } bool isHidden() { return silent && state != ekam::proto::TaskUpdate::State::FAILED; } ProjectExplorer::Task* firstTask() { return tasks.empty() ? 0 : &tasks.first(); } signals: void removed(); void stateChanged(ekam::proto::TaskUpdate::State state); void clearedTasks(); void addedTask(const ProjectExplorer::Task& task); private: EkamDashboardPlugin* plugin; ekam::proto::TaskUpdate::State state; QString verb; QString noun; QString path; bool silent; kj::Vector leftoverLog; QList tasks; void clearTasks(); void consumeLog(kj::StringPtr log); void parseLogLine(QString line); }; class EkamDashboardPlugin : public ExtensionSystem::IPlugin { Q_OBJECT Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QtCreatorPlugin" FILE "EkamDashboard.json") public: EkamDashboardPlugin(); ~EkamDashboardPlugin() noexcept; bool initialize(const QStringList &arguments, QString *errorString); void extensionsInitialized(); ShutdownFlag aboutToShutdown(); ProjectExplorer::TaskHub* taskHub() { return hub; } QString findFile(const QString& canonicalPath); QList allActions(); void unhideAction(ActionState* action) { emit newAction(action); } signals: void newAction(ActionState* action); private slots: void triggerAction(); void socketError(QAbstractSocket::SocketError); void retryConnection(); void socketReady(); private: class FakeAsyncInput; ProjectExplorer::TaskHub* hub; QTcpSocket* socket; bool seenHeader; QString projectRoot; QHash actions; kj::EventLoop eventLoop; kj::WaitScope waitScope; kj::Own fakeInput; kj::Promise readTask; bool resetting = false; void reset(); void tryConnect(); kj::Promise messageLoop(); void clearActions(); }; } // namespace Internal } // namespace EkamDashboard #endif // EKAMDASHBOARD_H ================================================ FILE: qtcreator/ekamtreewidget.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "ekamtreewidget.h" #include "ekamdashboardplugin.h" #include #include #include #include namespace EkamDashboard { namespace Internal { static constexpr ekam::proto::TaskUpdate::State DEFAULT_STATE = ekam::proto::TaskUpdate::State::UNCHANGED; // This defines the priority ordering of states. The state (and icon) for a directory will be // chosen based on the highest-priority state of its children. static const ekam::proto::TaskUpdate::State ORDERED_STATES[] = { DEFAULT_STATE, ekam::proto::TaskUpdate::State::DELETED, ekam::proto::TaskUpdate::State::DONE, ekam::proto::TaskUpdate::State::PASSED, ekam::proto::TaskUpdate::State::FAILED, ekam::proto::TaskUpdate::State::PENDING, ekam::proto::TaskUpdate::State::BLOCKED, ekam::proto::TaskUpdate::State::RUNNING, }; static int STATE_PRIORITIES[16]; struct StatePrioritiesInitializer { StatePrioritiesInitializer() { for (size_t i = 0; i < (sizeof(STATE_PRIORITIES) / sizeof(STATE_PRIORITIES[0])); i++) { STATE_PRIORITIES[i] = -1; } for (size_t i = 0; i < (sizeof(ORDERED_STATES) / sizeof(ORDERED_STATES[0])); i++) { STATE_PRIORITIES[static_cast(ORDERED_STATES[i])] = i; } } }; static StatePrioritiesInitializer statePrioritiesInitializer; // ======================================================================================= EkamTreeNode::EkamTreeNode(EkamTreeModel* tree) : QObject(tree), tree(tree), isDirectory(true), parentNode(0), action(0), state(DEFAULT_STATE) {} EkamTreeNode::EkamTreeNode(EkamTreeNode* parent, const QString& name, bool isDirectory) : QObject(parent), tree(parent->tree), isDirectory(isDirectory), name(name), parentNode(parent), action(0), state(DEFAULT_STATE) {} EkamTreeNode::~EkamTreeNode() {} int EkamTreeNode::row() { if (this == tree->root) { return -1; } int result = parentNode->childNodes.indexOf(this); if (result == -1) { qWarning() << "parentNode->childNodes doesn't contain this?"; } return result; } QModelIndex EkamTreeNode::index() { if (this == tree->root) { return QModelIndex(); } int rowNum = row(); if (rowNum == -1) { return QModelIndex(); } return tree->createIndex(rowNum, 0, this); } void EkamTreeNode::actionStateChanged(ekam::proto::TaskUpdate::State newState) { stateChanged(newState); } void EkamTreeNode::actionRemoved() { setAction(0); if (parentNode != 0) { parentNode->removeChild(this); } // QModelIndex myIndex = index(); // emit tree->dataChanged(myIndex, myIndex); } void EkamTreeNode::createNode(const QString& noun, const QString& verb, ActionState* action) { int slash = noun.indexOf(QLatin1Char('/')); QString childName = (slash == -1) ? QString(QLatin1String("%1 (%2)")).arg(noun, verb) : noun.mid(0, slash); QList::iterator iter = childNodes.begin(); while (iter < childNodes.end() && (*iter)->name < childName) { ++iter; } EkamTreeNode* selectedNode; bool isNew = iter == childNodes.end() || (*iter)->name != childName; if (isNew) { QModelIndex i = index(); int r = iter - childNodes.begin(); tree->beginInsertRows(i, r, r); selectedNode = new EkamTreeNode(this, childName, slash != -1); childNodes.insert(iter, selectedNode); } else { selectedNode = *iter; } if (slash == -1) { selectedNode->setAction(action); } else { selectedNode->createNode(noun.right(noun.size() - slash - 1), verb, action); } if (isNew) { tree->endInsertRows(); } else { QModelIndex nodeIndex = selectedNode->index(); emit tree->dataChanged(nodeIndex, nodeIndex); } } QVariant EkamTreeNode::data(int role) { switch (role) { case Qt::DisplayRole: case Qt::EditRole: // string title return name; case Qt::ToolTipRole: // string tooltip if (action == 0) { return QVariant(); } else { action->getPath(); } case Qt::DecorationRole: // icon if (isDirectory) { switch (state) { case ekam::proto::TaskUpdate::State::DELETED: return QIcon(QLatin1String(":/ekamdashboard/images/dir-deleted.png")); case ekam::proto::TaskUpdate::State::PENDING: return QIcon(QLatin1String(":/ekamdashboard/images/dir-pending.png")); case ekam::proto::TaskUpdate::State::RUNNING: return QIcon(QLatin1String(":/ekamdashboard/images/dir-running.png")); case ekam::proto::TaskUpdate::State::DONE: return QIcon(QLatin1String(":/ekamdashboard/images/dir-done.png")); case ekam::proto::TaskUpdate::State::PASSED: return QIcon(QLatin1String(":/ekamdashboard/images/dir-passed.png")); case ekam::proto::TaskUpdate::State::FAILED: return QIcon(QLatin1String(":/ekamdashboard/images/dir-failed.png")); case ekam::proto::TaskUpdate::State::BLOCKED: // Use pending icon for blocked. return QIcon(QLatin1String(":/ekamdashboard/images/dir-pending.png")); case DEFAULT_STATE: return QIcon(QLatin1String(":/ekamdashboard/images/dir.png")); } } else { switch (state) { case DEFAULT_STATE: case ekam::proto::TaskUpdate::State::DELETED: return QIcon(QLatin1String(":/ekamdashboard/images/state-deleted.png")); case ekam::proto::TaskUpdate::State::PENDING: return QIcon(QLatin1String(":/ekamdashboard/images/state-pending.png")); case ekam::proto::TaskUpdate::State::RUNNING: return QIcon(QLatin1String(":/ekamdashboard/images/state-running.png")); case ekam::proto::TaskUpdate::State::DONE: return QIcon(QLatin1String(":/ekamdashboard/images/state-done.png")); case ekam::proto::TaskUpdate::State::PASSED: return QIcon(QLatin1String(":/ekamdashboard/images/state-passed.png")); case ekam::proto::TaskUpdate::State::FAILED: return QIcon(QLatin1String(":/ekamdashboard/images/state-failed.png")); case ekam::proto::TaskUpdate::State::BLOCKED: // Use pending icon for blocked. return QIcon(QLatin1String(":/ekamdashboard/images/state-pending.png")); } } qWarning() << "Can't get here."; break; case Qt::FontRole: // We could bold this or something. QFont result; if (isDirectory) { result.setBold(true); } return result; } return QVariant(); } void EkamTreeNode::setAction(ActionState* newAction) { if (action != 0) { disconnect(action, SIGNAL(stateChanged(ekam::proto::TaskUpdate::State)), this, SLOT(actionStateChanged(ekam::proto::TaskUpdate::State))); disconnect(action, SIGNAL(removed()), this, SLOT(actionRemoved())); } action = newAction; stateChanged(action == 0 ? DEFAULT_STATE : action->getState()); if (action != 0) { connect(action, SIGNAL(stateChanged(ekam::proto::TaskUpdate::State)), this, SLOT(actionStateChanged(ekam::proto::TaskUpdate::State))); connect(action, SIGNAL(removed()), this, SLOT(actionRemoved())); } } void EkamTreeNode::stateChanged(ekam::proto::TaskUpdate::State newState) { if (state != newState) { state = newState; QModelIndex myIndex = index(); emit tree->dataChanged(myIndex, myIndex); if (parentNode != 0) { parentNode->childStateChanged(); } } } void EkamTreeNode::childStateChanged() { ekam::proto::TaskUpdate::State maxState = DEFAULT_STATE; int maxStatePriority = STATE_PRIORITIES[static_cast(maxState)]; foreach (EkamTreeNode* child, childNodes) { int childPriority = STATE_PRIORITIES[static_cast(child->state)]; if (childPriority > maxStatePriority) { maxState = child->state; maxStatePriority = childPriority; } } stateChanged(maxState); } void EkamTreeNode::removeChild(EkamTreeNode* child) { int r = child->row(); tree->beginRemoveRows(index(), r, r); childNodes.erase(childNodes.begin() + r); tree->endRemoveRows(); if (childNodes.empty() && parentNode != 0) { parentNode->removeChild(this); } } // ======================================================================================= EkamTreeModel::EkamTreeModel(EkamDashboardPlugin* plugin, QObject* parent) : QAbstractItemModel(parent), plugin(plugin), root(new EkamTreeNode(this)) { // qDebug() << "EkamTreeModel::EkamTreeModel(...)"; connect(plugin, SIGNAL(newAction(ActionState*)), this, SLOT(newAction(ActionState*))); foreach (ActionState* action, plugin->allActions()) { newAction(action); } } EkamTreeModel::~EkamTreeModel() {} QModelIndex EkamTreeModel::index(int row, int column, const QModelIndex & parent) const { // qDebug() << "EkamTreeModel::index(" << row << ", " << column << ", " << parent << ")"; if (column != 0) { // Invalid. return QModelIndex(); } EkamTreeNode* parentNode = indexToNode(parent); if (row < 0 || row >= parentNode->childCount()) { // Out of bounds. return QModelIndex(); } return createIndex(row, 0, parentNode->getChild(row)); } QModelIndex EkamTreeModel::parent(const QModelIndex &index) const { // qDebug() << "EkamTreeModel::parent(" << index << ")"; EkamTreeNode* node = indexToNode(index); if (node == root) { // This should never happen? qWarning() << "Called parent() on invisible root object?"; return QModelIndex(); } else { return reinterpret_cast(node->parent())->index(); } } QVariant EkamTreeModel::data(const QModelIndex &index, int role) const { // qDebug() << "EkamTreeModel::data(" << index << ", " << role << ") = " // << indexToNode(index)->data(role); return indexToNode(index)->data(role); } int EkamTreeModel::rowCount(const QModelIndex & parent) const { // qDebug() << "EkamTreeModel::rowCount(" << parent << ") = " // << indexToNode(parent)->childCount(); return indexToNode(parent)->childCount(); } int EkamTreeModel::columnCount(const QModelIndex & parent) const { // qDebug() << "EkamTreeModel::columnCount(" << parent << ") = 1"; Q_UNUSED(parent); return 1; } bool EkamTreeModel::hasChildren(const QModelIndex & parent) const { // qDebug() << "EkamTreeModel::hasChildren(" << parent << ") = " // << (indexToNode(parent)->childCount() > 0); return indexToNode(parent)->childCount() > 0; } void EkamTreeModel::newAction(ActionState* action) { // qDebug() << "EkamTreeModel::newAction(" << action->getVerb() << ":" << action->getNoun() << ")"; root->createNode(action->getNoun(), action->getVerb(), action); } // ======================================================================================= EkamTreeWidget::EkamTreeWidget(EkamDashboardPlugin* plugin) : QWidget(), plugin(plugin) { model = new EkamTreeModel(plugin, this); view = new Utils::NavigationTreeView(this); view->setModel(model); setFocusProxy(view); QVBoxLayout *layout = new QVBoxLayout(); layout->addWidget(view); layout->setContentsMargins(0, 0, 0, 0); setLayout(layout); connect(view, SIGNAL(doubleClicked(QModelIndex)), this, SLOT(jumpTo(QModelIndex))); } EkamTreeWidget::~EkamTreeWidget() {} void EkamTreeWidget::jumpTo(const QModelIndex& index) { ActionState* action = model->indexToNode(index)->getAction(); if (action != 0) { const ProjectExplorer::Task* task = action->firstTask(); Core::EditorManager* editorManager = Core::EditorManager::instance(); if (task == 0) { // Open in editor. editorManager->openEditor(action->getPath()); } else { // Open first task in editor and issues list. plugin->taskHub()->taskMarkClicked(task->taskId); QString name = task->file.toString(); if (name.isEmpty()) { name = action->getPath(); } editorManager->openEditorAt(name, task->line); } } } // ======================================================================================= EkamTreeWidgetFactory::EkamTreeWidgetFactory(EkamDashboardPlugin* plugin) : INavigationWidgetFactory(), plugin(plugin) { setId("EkamActions"); setDisplayName(QLatin1String("Ekam Actions")); setPriority(100); } EkamTreeWidgetFactory::~EkamTreeWidgetFactory() {} Core::NavigationView EkamTreeWidgetFactory::createWidget() { Core::NavigationView result; EkamTreeWidget* tree = new EkamTreeWidget(plugin); result.widget = tree; return result; } } // namespace Internal } // namespace EkamDashboard ================================================ FILE: qtcreator/ekamtreewidget.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef EKAMTREEWIDGET_H #define EKAMTREEWIDGET_H #include #include #include #include #include #include "dashboard.capnp.h" namespace EkamDashboard { namespace Internal { class EkamDashboardPlugin; class EkamTreeModel; class ActionState; class EkamTreeNode : public QObject { Q_OBJECT public: explicit EkamTreeNode(EkamTreeModel* tree); EkamTreeNode(EkamTreeNode* parent, const QString& name, bool isDirectory); virtual ~EkamTreeNode(); int row(); QModelIndex index(); void createNode(const QString& noun, const QString& verb, ActionState* action); QVariant data(int role); int childCount() { return childNodes.size(); } EkamTreeNode* getChild(int index) { return childNodes.at(index); } ActionState* getAction() { return action; } private slots: void actionStateChanged(ekam::proto::TaskUpdate::State newState); void actionRemoved(); private: EkamTreeModel* tree; bool isDirectory; QString name; EkamTreeNode* parentNode; QList childNodes; ActionState* action; ekam::proto::TaskUpdate::State state; void setAction(ActionState* newAction); void stateChanged(ekam::proto::TaskUpdate::State newState); void childStateChanged(); void removeChild(EkamTreeNode* child); }; class EkamTreeModel : public QAbstractItemModel { Q_OBJECT public: EkamTreeModel(EkamDashboardPlugin* plugin, QObject* parent = 0); virtual ~EkamTreeModel(); virtual QModelIndex index(int row, int column, const QModelIndex & parent = QModelIndex()) const; virtual QModelIndex parent(const QModelIndex &index) const; virtual QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const; virtual int rowCount(const QModelIndex & parent = QModelIndex()) const; virtual int columnCount(const QModelIndex & parent = QModelIndex()) const; virtual bool hasChildren(const QModelIndex & parent = QModelIndex()) const; EkamTreeNode* indexToNode(const QModelIndex& index) const { return index.isValid() ? reinterpret_cast(index.internalPointer()) : root; } private slots: void newAction(ActionState* action); private: friend class EkamTreeNode; EkamDashboardPlugin* plugin; EkamTreeNode* root; }; class EkamTreeWidget : public QWidget { Q_OBJECT public: explicit EkamTreeWidget(EkamDashboardPlugin *plugin = 0); virtual ~EkamTreeWidget(); private slots: void jumpTo(const QModelIndex& index); private: EkamDashboardPlugin* plugin; EkamTreeModel* model; QTreeView* view; }; class EkamTreeWidgetFactory : public Core::INavigationWidgetFactory { Q_OBJECT public: explicit EkamTreeWidgetFactory(EkamDashboardPlugin* plugin); virtual ~EkamTreeWidgetFactory(); virtual Core::NavigationView createWidget(); private: EkamDashboardPlugin* plugin; }; } // namespace Internal } // namespace EkamDashboard #endif // EKAMTREEWIDGET_H ================================================ FILE: src/base/Debug.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Debug.h" #include namespace ekam { DebugMessage::Severity DebugMessage::logLevel = WARNING; int DebugMessage::counter = 0; static const char* SEVERITY_NAMES[] = { "INFO", "WARNING", "ERROR" }; DebugMessage::DebugMessage(Severity severity, const char* filename, int line) { *this << "ekam debug: " << SEVERITY_NAMES[severity] << ": " << filename << ":" << line << ": "; } DebugMessage::~DebugMessage() { // TODO: We really need to buffer the message and write it all at once to avoid interleaved // text when multiprocessing. fputs("\n", stderr); fflush(stderr); ++counter; } DebugMessage& DebugMessage::operator<<(const char* value) { fputs(value, stderr); return *this; } DebugMessage& DebugMessage::operator<<(const std::string& value) { fwrite(value.data(), sizeof(char), value.size(), stderr); return *this; } #define HANDLE_TYPE(TYPE, FORMAT) \ DebugMessage& DebugMessage::operator<<(TYPE value) { \ fprintf(stderr, FORMAT, value); \ return *this; \ } HANDLE_TYPE(char, "%c"); HANDLE_TYPE(signed char, "%hhd"); HANDLE_TYPE(unsigned char, "%hhu"); HANDLE_TYPE(short, "%hd"); HANDLE_TYPE(unsigned short, "%hu"); HANDLE_TYPE(int, "%d"); HANDLE_TYPE(unsigned int, "%u"); HANDLE_TYPE(long, "%ld"); HANDLE_TYPE(unsigned long, "%lu"); HANDLE_TYPE(long long, "%lld"); HANDLE_TYPE(unsigned long long, "%llu"); HANDLE_TYPE(float, "%g"); HANDLE_TYPE(double, "%g"); HANDLE_TYPE(const void*, "%p"); } // namespace ekam ================================================ FILE: src/base/Debug.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_BASE_DEBUGLOG_H_ #define KENTONSCODE_BASE_DEBUGLOG_H_ #include namespace ekam { class DebugMessage { public: enum Severity { INFO, WARNING, ERROR }; DebugMessage(Severity severity, const char* filename, int line); ~DebugMessage(); DebugMessage& operator<<(const char* value); DebugMessage& operator<<(const std::string& value); DebugMessage& operator<<(char value); DebugMessage& operator<<(signed char value); DebugMessage& operator<<(unsigned char value); DebugMessage& operator<<(short value); DebugMessage& operator<<(unsigned short value); DebugMessage& operator<<(int value); DebugMessage& operator<<(unsigned int value); DebugMessage& operator<<(long value); DebugMessage& operator<<(unsigned long value); DebugMessage& operator<<(long long value); DebugMessage& operator<<(unsigned long long value); DebugMessage& operator<<(float value); DebugMessage& operator<<(double value); DebugMessage& operator<<(const void* ptr); inline static bool shouldLog(Severity severity, const char*, int) { return severity >= logLevel; } inline static void setLogLevel(Severity severity) { logLevel = severity; } // Useful for detecting if any log messages have been printed, e.g. to avoid clobbering them // with terminal manipulations. inline static int getMessageCount() { return counter; } private: static Severity logLevel; static int counter; }; #define DEBUG_LOG(SEVERITY) \ if (!::ekam::DebugMessage::shouldLog(::ekam::DebugMessage::SEVERITY, __FILE__, __LINE__)) {} \ else ::ekam::DebugMessage(::ekam::DebugMessage::SEVERITY, __FILE__, __LINE__) #define DEBUG_INFO DEBUG_LOG(INFO) #define DEBUG_WARNING DEBUG_LOG(WARNING) #define DEBUG_ERROR DEBUG_LOG(ERROR) } // namespace ekam #endif // KENTONSCODE_BASE_DEBUGLOG_H_ ================================================ FILE: src/base/Hash.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Hash.h" #include #include "sha256.h" namespace ekam { namespace { char HexDigit(unsigned int value) { value &= 0x0F; if (value < 10) { return '0' + value; } else { return 'a' + value - 10; } } } // anonymous namespace Hash Hash::of(const std::string& data) { return Builder().add(data).build(); } Hash Hash::of(void* data, size_t size) { return Builder().add(data, size).build(); } // Note: Since this is in static space it will be automatically initialized to zero. const Hash Hash::NULL_HASH; std::string Hash::toString() const { std::string result; result.reserve(sizeof(hash) * 2); for (unsigned int i = 0; i < sizeof(hash); i++) { result.push_back(HexDigit(hash[i] >> 4)); result.push_back(HexDigit(hash[i])); } return result; } Hash::Builder::Builder() { SHA256_Init(&context); } Hash::Builder& Hash::Builder::add(const std::string& data) { SHA256_Update(&context, data.data(), data.size()); return *this; } Hash::Builder& Hash::Builder::add(void* data, size_t size) { SHA256_Update(&context, data, size); return *this; } Hash Hash::Builder::build() { Hash result; SHA256_Final(result.hash, &context); return result; } } // namespace ekam ================================================ FILE: src/base/Hash.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_BASE_HASH_H_ #define KENTONSCODE_BASE_HASH_H_ #include #include #include #include "sha256.h" namespace ekam { class Hash { public: inline Hash() {} class Builder { public: Builder(); Builder& add(const std::string& data); Builder& add(void* data, size_t size); Hash build(); private: SHA256Context context; }; static Hash of(const std::string& data); static Hash of(void* data, size_t size); static const Hash NULL_HASH; std::string toString() const; inline bool operator==(const Hash& other) const { return memcmp(hash, other.hash, sizeof(hash)) == 0; } inline bool operator!=(const Hash& other) const { return memcmp(hash, other.hash, sizeof(hash)) != 0; } inline bool operator<(const Hash& other) const { return memcmp(hash, other.hash, sizeof(hash)) < 0; } inline bool operator>(const Hash& other) const { return memcmp(hash, other.hash, sizeof(hash)) > 0; } inline bool operator<=(const Hash& other) const { return memcmp(hash, other.hash, sizeof(hash)) <= 0; } inline bool operator>=(const Hash& other) const { return memcmp(hash, other.hash, sizeof(hash)) >= 0; } class StlHashFunc { public: inline size_t operator()(const Hash& h) const { return h.shortHash; } }; private: union { unsigned char hash[32]; size_t shortHash; }; }; } // namespace ekam #endif // KENTONSCODE_BASE_HASH_H_ ================================================ FILE: src/base/OwnedPtr.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "OwnedPtr.h" namespace ekam { } // namespace ekam ================================================ FILE: src/base/OwnedPtr.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_BASE_OWNEDPTR_H_ #define KENTONSCODE_BASE_OWNEDPTR_H_ #include #include #include #include #include #include #include #ifdef __CDT_PARSER__ #define noexcept #define constexpr namespace std { struct nullptr_t; } #endif namespace ekam { template inline void deleteEnsuringCompleteType(T* ptr) { enum { type_must_be_complete = sizeof(T) }; delete ptr; } template class OwnedPtr { public: OwnedPtr() : ptr(NULL) {} OwnedPtr(const OwnedPtr&) = delete; OwnedPtr(OwnedPtr&& other) : ptr(other.releaseRaw()) {} template OwnedPtr(OwnedPtr&& other) : ptr(other.releaseRaw()) {} OwnedPtr(std::nullptr_t) : ptr(NULL) {} ~OwnedPtr() { deleteEnsuringCompleteType(ptr); } OwnedPtr& operator=(const OwnedPtr&) = delete; OwnedPtr& operator=(OwnedPtr&& other) { reset(other.releaseRaw()); return *this; } template OwnedPtr& operator=(OwnedPtr&& other) { reset(other.releaseRaw()); return *this; } T* get() const { return ptr; } T* operator->() const { assert(ptr != NULL); return ptr; } T& operator*() const { assert(ptr != NULL); return *ptr; } OwnedPtr release() { return OwnedPtr(releaseRaw()); } void clear() { reset(NULL); } bool operator==(const T* other) { return ptr == other; } bool operator!=(const T* other) { return ptr != other; } private: T* ptr; explicit OwnedPtr(T* ptr) : ptr(ptr) {} void reset(T* newValue) { T* oldValue = ptr; ptr = newValue; deleteEnsuringCompleteType(oldValue); } T* releaseRaw() { T* result = ptr; ptr = NULL; return result; } template friend class OwnedPtr; template friend OwnedPtr newOwned(Params&&... params); template friend class SmartPtr; template friend class OwnedPtrVector; template friend class OwnedPtrDeque; template friend class OwnedPtrQueue; template friend class OwnedPtrMap; }; template OwnedPtr newOwned(Params&&... params) { return OwnedPtr(new T(std::forward(params)...)); } template class Indirect { public: template Indirect(Params&&... params): ptr(newOwned(std::forward(params)...)) {} Indirect(Indirect&& other): ptr(other.ptr.release()) {} Indirect(const Indirect& other): ptr(newOwned(*other.ptr)) {} Indirect& operator=(Indirect&& other) { ptr = other.ptr.release(); return *this; } Indirect& operator=(const Indirect& other) { ptr = newOwned(*other.ptr); return *this; } bool operator==(const Indirect& other) const { return *ptr == *other.ptr; } bool operator!=(const Indirect& other) const { return *ptr != *other.ptr; } const T& operator*() const { return *ptr; } T& operator*() { return *ptr; } const T* operator->() const { return ptr.get(); } T* operator->() { return ptr.get(); } private: OwnedPtr ptr; }; // TODO: Hide this somewhere private? class Refcount { public: Refcount(): strong(1), weak(0) {} Refcount(const Refcount& other) = delete; Refcount& operator=(const Refcount& other) = delete; static void inc(Refcount* r) { if (r != NULL) ++r->strong; } static bool dec(Refcount* r) { if (r == NULL) { return false; } else if (--r->strong == 0) { if (r->weak == 0) { delete r; } return true; } else { return false; } } static void incWeak(Refcount* r) { if (r != NULL) ++r->weak; } static void decWeak(Refcount* r) { if (r != NULL && --r->weak == 0 && r->strong == 0) { delete r; } } static bool release(Refcount* r) { if (r != NULL && r->strong == 1) { dec(r); return true; } else { return false; } } static bool isLive(Refcount* r) { return r != NULL && r->strong > 0; } private: int strong; int weak; }; template class SmartPtr { public: SmartPtr() : ptr(NULL), refcount(NULL) {} SmartPtr(std::nullptr_t) : ptr(NULL), refcount(NULL) {} ~SmartPtr() { if (Refcount::dec(refcount)) { deleteEnsuringCompleteType(ptr); } } SmartPtr(const SmartPtr& other) : ptr(other.ptr), refcount(other.refcount) { Refcount::inc(refcount); } template SmartPtr(const SmartPtr& other) : ptr(other.ptr), refcount(other.refcount) { Refcount::inc(refcount); } SmartPtr& operator=(const SmartPtr& other) { reset(other.ptr, other.refcount); return *this; } template SmartPtr& operator=(const SmartPtr& other) { reset(other.ptr, other.refcount); return *this; } SmartPtr(SmartPtr&& other) : ptr(other.ptr), refcount(other.refcount) { other.ptr = NULL; other.refcount = NULL; } template SmartPtr(SmartPtr&& other) : ptr(other.ptr), refcount(other.refcount) { other.ptr = NULL; other.refcount = NULL; } SmartPtr& operator=(SmartPtr&& other) { // Move pointers to locals before reset() in case &other == this. T* tempPtr = other.ptr; Refcount* tempRefcount = other.refcount; other.ptr = NULL; other.refcount = NULL; reset(NULL, NULL); ptr = tempPtr; refcount = tempRefcount; return *this; } template SmartPtr& operator=(SmartPtr&& other) { // Move pointers to locals before reset() in case &other == this. T* tempPtr = other.ptr; Refcount* tempRefcount = other.refcount; other.ptr = NULL; other.refcount = NULL; reset(NULL, NULL); ptr = tempPtr; refcount = tempRefcount; return *this; } template SmartPtr(OwnedPtr&& other) : ptr(other.releaseRaw()), refcount(ptr == NULL ? NULL : new Refcount()) {} template SmartPtr& operator=(OwnedPtr&& other) { reset(other.releaseRaw()); return *this; } T* get() const { return ptr; } T* operator->() const { assert(ptr != NULL); return ptr; } T& operator*() const { assert(ptr != NULL); return *ptr; } template bool release(OwnedPtr* other) { if (Refcount::release(refcount)) { other->reset(ptr); ptr = NULL; return true; } else { return false; } } void clear() { reset(NULL); } bool operator==(const T* other) { return ptr == other; } bool operator!=(const T* other) { return ptr != other; } bool operator==(const SmartPtr& other) { return ptr == other.ptr; } bool operator!=(const SmartPtr& other) { return ptr != other.ptr; } void allocate() { reset(new T()); } template void allocate(const P1& p1) { reset(new T(p1)); } template void allocate(const P1& p1, const P2& p2) { reset(new T(p1, p2)); } template void allocate(const P1& p1, const P2& p2, const P3& p3) { reset(new T(p1, p2, p3)); } template void allocate(const P1& p1, const P2& p2, const P3& p3, const P4& p4) { reset(new T(p1, p2, p3, p4)); } template void allocateSubclass() { reset(new Sub()); } template void allocateSubclass(const P1& p1) { reset(new Sub(p1)); } template void allocateSubclass(const P1& p1, const P2& p2) { reset(new Sub(p1, p2)); } template void allocateSubclass(const P1& p1, const P2& p2, const P3& p3) { reset(new Sub(p1, p2, p3)); } template void allocateSubclass(const P1& p1, const P2& p2, const P3& p3, const P4& p4) { reset(new Sub(p1, p2, p3, p4)); } private: T* ptr; Refcount* refcount; inline void reset(T* newValue) { reset(newValue, newValue == NULL ? NULL : new Refcount()); Refcount::dec(refcount); } void reset(T* newValue, Refcount* newRefcount) { T* oldValue = ptr; Refcount* oldRefcount = refcount; ptr = newValue; refcount = newRefcount; Refcount::inc(refcount); if (Refcount::dec(oldRefcount)) { deleteEnsuringCompleteType(oldValue); } } template friend class SmartPtr; template friend class WeakPtr; template friend class OwnedPtr; template friend class OwnedPtrVector; template friend class OwnedPtrQueue; template friend class OwnedPtrMap; }; template class WeakPtr { public: WeakPtr(): ptr(NULL), refcount(NULL) {} WeakPtr(const WeakPtr& other): ptr(other.ptr), refcount(other.refcount) { Refcount::incWeak(refcount); } WeakPtr(const SmartPtr& other): ptr(other.ptr), refcount(other.refcount) { Refcount::incWeak(refcount); } WeakPtr(std::nullptr_t): ptr(nullptr), refcount(nullptr) {} ~WeakPtr() { Refcount::decWeak(refcount); } WeakPtr& operator=(const WeakPtr& other) { Refcount* oldRefcount = refcount; ptr = other.ptr; refcount = other.refcount; Refcount::incWeak(refcount); Refcount::decWeak(oldRefcount); return *this; } template WeakPtr& operator=(const WeakPtr& other) { Refcount* oldRefcount = refcount; ptr = other.ptr; refcount = other.refcount; Refcount::incWeak(refcount); Refcount::decWeak(oldRefcount); return *this; } template WeakPtr& operator=(const SmartPtr& other) { Refcount* oldRefcount = refcount; ptr = other.ptr; refcount = other.refcount; Refcount::incWeak(refcount); Refcount::decWeak(oldRefcount); return *this; } WeakPtr& operator=(std::nullptr_t) { Refcount::decWeak(refcount); ptr = nullptr; refcount = nullptr; return *this; } template operator SmartPtr() const { SmartPtr result; if (Refcount::isLive(refcount)) { result.reset(ptr, refcount); } return result; } private: T* ptr; Refcount* refcount; }; template class OwnedPtrVector { public: OwnedPtrVector() {} OwnedPtrVector(const OwnedPtrVector&) = delete; OwnedPtrVector(OwnedPtrVector&& other) { vec.swap(other.vec); } ~OwnedPtrVector() { for (typename std::vector::const_iterator iter = vec.begin(); iter != vec.end(); ++iter) { deleteEnsuringCompleteType(*iter); } } OwnedPtrVector& operator=(const OwnedPtrVector&) = delete; int size() const { return vec.size(); } T* get(int index) const { return vec[index]; } bool empty() const { return vec.empty(); } void add(OwnedPtr ptr) { vec.push_back(ptr.releaseRaw()); } void set(int index, OwnedPtr ptr) { deleteEnsuringCompleteType(vec[index]); vec[index] = ptr->releaseRaw(); } OwnedPtr release(int index) { T* result = vec[index]; vec[index] = NULL; return OwnedPtr(result); } OwnedPtr releaseBack() { T* result = vec.back(); vec.pop_back(); return OwnedPtr(result); } OwnedPtr releaseAndShift(int index) { T* result = vec[index]; vec.erase(vec.begin() + index); return OwnedPtr(result); } void clear() { for (typename std::vector::const_iterator iter = vec.begin(); iter != vec.end(); ++iter) { deleteEnsuringCompleteType(*iter); } vec.clear(); } void swap(OwnedPtrVector* other) { vec.swap(other->vec); } class Appender { public: explicit Appender(OwnedPtrVector* vec) : vec(vec) {} void add(OwnedPtr ptr) { vec->add(ptr.release()); } private: OwnedPtrVector* vec; }; Appender appender() { return Appender(this); } private: std::vector vec; }; template class OwnedPtrDeque { public: OwnedPtrDeque() {} ~OwnedPtrDeque() { for (typename std::deque::const_iterator iter = q.begin(); iter != q.end(); ++iter) { deleteEnsuringCompleteType(*iter); } } int size() const { return q.size(); } T* get(int index) const { return q[index]; } bool empty() const { return q.empty(); } void pushFront(OwnedPtr ptr) { q.push_front(ptr.releaseRaw()); } OwnedPtr popFront() { T* ptr = q.front(); q.pop_front(); return OwnedPtr(ptr); } void pushBack(OwnedPtr ptr) { q.push_back(ptr.releaseRaw()); } OwnedPtr popBack() { T* ptr = q.back(); q.pop_back(); return OwnedPtr(ptr); } OwnedPtr releaseAndShift(int index) { T* ptr = q[index]; q.erase(q.begin() + index); return OwnedPtr(ptr); } void clear() { for (typename std::deque::const_iterator iter = q.begin(); iter != q.end(); ++iter) { deleteEnsuringCompleteType(*iter); } q.clear(); } void swap(OwnedPtrDeque* other) { q.swap(other->q); } private: std::deque q; }; template class OwnedPtrQueue { public: OwnedPtrQueue() {} ~OwnedPtrQueue() { clear(); } int size() const { return q.size(); } bool empty() const { return q.empty(); } void push(OwnedPtr ptr) { q.push(ptr.releaseRaw()); } OwnedPtr pop() { T* ptr = q.front(); q.pop(); return OwnedPtr(ptr); } void clear() { while (!q.empty()) { deleteEnsuringCompleteType(q.front()); q.pop(); } } class Appender { public: Appender(OwnedPtrQueue* q) : q(q) {} void add(OwnedPtr ptr) { q->push(ptr); } private: OwnedPtrQueue* q; }; Appender appender() { return Appender(this); } private: std::queue q; }; template , typename EqualsFunc = std::equal_to > class OwnedPtrMap { typedef std::unordered_map InnerMap; public: OwnedPtrMap() {} ~OwnedPtrMap() { for (typename InnerMap::const_iterator iter = map.begin(); iter != map.end(); ++iter) { deleteEnsuringCompleteType(iter->second); } } bool empty() const { return map.empty(); } int size() const { return map.size(); } bool contains(const Key& key) const { return map.count(key) > 0; } T* get(const Key& key) const { typename InnerMap::const_iterator iter = map.find(key); if (iter == map.end()) { return NULL; } else { return iter->second; } } void add(const Key& key, OwnedPtr ptr) { T* value = ptr.releaseRaw(); std::pair insertResult = map.insert(std::make_pair(key, value)); if (!insertResult.second) { deleteEnsuringCompleteType(insertResult.first->second); insertResult.first->second = value; } } bool addIfNew(const Key& key, OwnedPtr ptr) { T* value = ptr.releaseRaw(); std::pair insertResult = map.insert(std::make_pair(key, value)); if (insertResult.second) { return true; } else { deleteEnsuringCompleteType(value); return false; } } bool release(const Key& key, OwnedPtr* output) { typename InnerMap::iterator iter = map.find(key); if (iter == map.end()) { output->reset(NULL); return false; } else { output->reset(iter->second); map.erase(iter); return true; } } void releaseAll(typename OwnedPtrVector::Appender output) { for (typename InnerMap::const_iterator iter = map.begin(); iter != map.end(); ++iter) { output.add(OwnedPtr(iter->second)); } map.clear(); } bool erase(const Key& key) { typename InnerMap::iterator iter = map.find(key); if (iter == map.end()) { return false; } else { deleteEnsuringCompleteType(iter->second); map.erase(iter); return true; } } void clear() { for (typename InnerMap::const_iterator iter = map.begin(); iter != map.end(); ++iter) { deleteEnsuringCompleteType(iter->second); } map.clear(); } void swap(OwnedPtrMap* other) { map.swap(other->map); } class Iterator { public: Iterator(const OwnedPtrMap& map) : nextIter(map.map.begin()), end(map.map.end()) {} bool next() { if (nextIter == end) { return false; } else { iter = nextIter; ++nextIter; return true; } } const Key& key() { return iter->first; } T* value() { return iter->second; } private: typename InnerMap::const_iterator iter; typename InnerMap::const_iterator nextIter; typename InnerMap::const_iterator end; }; private: InnerMap map; }; } // namespace ekam #endif // KENTONSCODE_BASE_OWNEDPTR_H_ ================================================ FILE: src/base/Promise.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Promise.h" namespace ekam { Runnable::~Runnable() {} PendingRunnable::~PendingRunnable() {} Executor::~Executor() noexcept(false) {} } // namespace ekam ================================================ FILE: src/base/Promise.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_BASE_PROMISE_H_ #define KENTONSCODE_BASE_PROMISE_H_ #include #include #include #include #include #include #include #include "OwnedPtr.h" #include "Debug.h" namespace ekam { // ======================================================================================= // TODO: Put elsewhere enum class Void { VOID }; template class ValuePack; template class ValuePack: private ValuePack { public: ValuePack(Head&& head, Tail&&... tail) : ValuePack(std::forward(tail)...), head(std::forward(head)) {} ValuePack(const ValuePack& other) = delete; ValuePack(ValuePack&& other) = default; ValuePack& operator=(const ValuePack& other) = delete; ValuePack& operator=(ValuePack&& other) { head = std::move(other.head); ValuePack::operator=(std::move(other)); } template auto apply(const Func& func, Params&&... params) const -> decltype(func(std::declval()..., std::declval(), std::declval()...)) { return ValuePack::apply(func, std::forward(params)..., head); } template auto applyMoving(const Func& func, Params&&... params) -> decltype(func(std::declval()..., std::declval(), std::declval()...)) { return ValuePack::applyMoving(func, std::forward(params)..., std::move(head)); } private: Head head; }; template <> class ValuePack<> { public: ValuePack() {} template auto apply(const Func& func, Params&&... params) const -> decltype(func(std::declval()...)) { func(std::forward(params)...); } template auto applyMoving(const Func& func, Params&&... params) -> decltype(func(std::declval()...)) { func(std::forward(params)...); } }; // ======================================================================================= // TODO: Put elsewhere class WeakLink { public: WeakLink(): other(nullptr) {} WeakLink(WeakLink* other): other(other) { if (other != nullptr) { other->disentangle(); other->other = this; } } ~WeakLink() { disentangle(); } void entangle(WeakLink* other) { disentangle(); this->other = other; if (other != nullptr) { other->disentangle(); other->other = this; } } bool isEntangled() { return other != nullptr; } private: WeakLink* other; void disentangle() { if (other != nullptr) { other->other = nullptr; other = nullptr; } } }; // ======================================================================================= template class Promise; class Runnable { public: virtual ~Runnable(); virtual void run() = 0; }; class PendingRunnable { public: virtual ~PendingRunnable(); }; class Executor { public: virtual ~Executor() noexcept(false); virtual OwnedPtr runLater(OwnedPtr runnable) = 0; template class When; template When::type...> when(Types&&... params); }; template class LambdaRunnable: public Runnable { public: LambdaRunnable(Func&& func): func(std::move(func)) {} ~LambdaRunnable() {} void run() { func(); } private: Func func; }; template OwnedPtr newLambdaRunnable(Func&& func) { return newOwned>(std::move(func)); } // ======================================================================================= template class MaybeException { public: MaybeException(): broken(false) { new(&this->value) T; } MaybeException(T value): broken(false) { new(&this->value) T(std::move(value)); } MaybeException(std::exception_ptr error): broken(true) { new(&this->error) std::exception_ptr(std::move(error)); } MaybeException(const MaybeException& other) { if (broken) { new(&error) std::exception_ptr(other.error); } else { new(&value) T(other.value); } } MaybeException(MaybeException&& other): broken(other.broken) { if (broken) { new(&error) std::exception_ptr(std::move(other.error)); } else { new(&value) T(std::move(other.value)); } } ~MaybeException() { if (broken) { error.~exception_ptr(); } else { value.~T(); } } MaybeException& operator=(const MaybeException& other) { this->~MaybeException(); broken = other.broken; if (broken) { new(&error) std::exception_ptr(other.error); } else { new(&value) T(other.value); } return *this; } MaybeException& operator=(MaybeException&& other) { this->~MaybeException(); broken = other.broken; if (broken) { new(&error) std::exception_ptr(std::move(other.error)); } else { new(&value) T(std::move(other.value)); } return *this; } bool isException() { return broken; } const T& get() { if (broken) { std::rethrow_exception(error); } return value; } T release() { if (broken) { std::rethrow_exception(error); } return std::move(value); } private: bool broken; union { T value; std::exception_ptr error; }; }; template <> class MaybeException { public: MaybeException(): broken(false) {} MaybeException(std::exception_ptr error): broken(true) { new(&this->error) std::exception_ptr(std::move(error)); } MaybeException(const MaybeException& other) { if (broken) { new(&error) std::exception_ptr(other.error); } } MaybeException(MaybeException&& other): broken(other.broken) { if (broken) { new(&error) std::exception_ptr(std::move(other.error)); } } ~MaybeException() { if (broken) { error.~exception_ptr(); } } MaybeException& operator=(const MaybeException& other) { this->~MaybeException(); broken = other.broken; if (broken) { new(&error) std::exception_ptr(other.error); } return *this; } MaybeException& operator=(MaybeException&& other) { this->~MaybeException(); broken = other.broken; if (broken) { new(&error) std::exception_ptr(std::move(other.error)); } return *this; } bool isException() { return broken; } Void get() { if (broken) { std::rethrow_exception(error); } return Void::VOID; } Void release() { if (broken) { std::rethrow_exception(error); } return Void::VOID; } private: bool broken; union { std::exception_ptr error; }; }; // ======================================================================================= template class PromiseFulfiller { public: typedef T PromiseType; class Callback; }; namespace promiseInternal { struct PromiseConstructors; template class DependentPromiseFulfiller; class PromiseListener { public: virtual void dependencyDone(bool failed) = 0; }; template class PromiseState { public: PromiseState(): owner(nullptr), listener(nullptr), fulfilled(false), failed(false) {} virtual ~PromiseState() {} Promise* owner; OwnedPtr chainedPromise; // (effectively) private: PromiseListener* listener; bool fulfilled; bool failed; MaybeException value; void setListener(PromiseListener* listener) { if (this->listener != nullptr) { throw std::invalid_argument("Already waiting on this Promise."); } this->listener = listener; if (fulfilled) { listener->dependencyDone(failed); } } void preFulfill() { if (fulfilled) { throw std::logic_error("Already fulfilled this promise."); } fulfilled = true; } void postFulfill(bool failed) { this->failed = failed; if (listener != nullptr) { listener->dependencyDone(failed); } } void fulfill(const T& value) { preFulfill(); this->value = value; postFulfill(false); } void fulfill(T&& value) { preFulfill(); this->value = std::move(value); postFulfill(false); } void fulfill(Promise chainedPromise); void propagateCurrentException() { preFulfill(); this->value = std::current_exception(); postFulfill(true); } T release() { return value.release(); } MaybeException releaseMaybeException() { return std::move(value); } template friend class DependentPromiseFulfiller; friend struct PromiseConstructors; }; template <> class PromiseState { public: PromiseState(): owner(nullptr), listener(nullptr), fulfilled(false), failed(false) {} virtual ~PromiseState() {} Promise* owner; OwnedPtr chainedPromise; private: PromiseListener* listener; bool fulfilled; bool failed; MaybeException value; void setListener(PromiseListener* listener) { if (this->listener != nullptr) { throw std::invalid_argument("Already waiting on this Promise."); } this->listener = listener; if (fulfilled) { listener->dependencyDone(failed); } } void preFulfill() { if (fulfilled) { throw std::logic_error("Already fulfilled this promise."); } fulfilled = true; } void postFulfill(bool failed) { this->failed = failed; if (listener != nullptr) { listener->dependencyDone(failed); } } void fulfill() { preFulfill(); postFulfill(false); } void fulfill(Promise chainedPromise); void propagateCurrentException() { preFulfill(); this->value = std::current_exception(); postFulfill(true); } Void release() { return value.release(); } MaybeException releaseMaybeException() { return std::move(value); } friend class PromiseFulfiller::Callback; template friend class DependentPromiseFulfiller; friend struct PromiseConstructors; }; template struct Unpack { typedef T Type; }; template struct Unpack> { typedef T Type; }; template <> struct Unpack> { typedef Void Type; }; template struct Chain { typedef T Type; }; template struct Chain> { typedef T Type; }; // ======================================================================================= template class FulfillerPromiseState; } // namespace promiseInternal template class PromiseFulfiller::Callback { public: template void fulfill(U&& value) { state->fulfill(std::forward(value)); } void propagateCurrentException() { state->propagateCurrentException(); } private: promiseInternal::PromiseState* state; Callback(promiseInternal::PromiseState* state): state(state) {} ~Callback() {} template friend class promiseInternal::FulfillerPromiseState; }; template <> class PromiseFulfiller::Callback { public: void fulfill() { state->fulfill(); } void fulfill(Promise chain); void propagateCurrentException() { state->propagateCurrentException(); } private: promiseInternal::PromiseState* state; Callback(promiseInternal::PromiseState* state): state(state) {} ~Callback() {} template friend class promiseInternal::FulfillerPromiseState; }; namespace promiseInternal { template class FulfillerPromiseState : public PromiseState { public: template FulfillerPromiseState(Params&&... params) : PromiseState(), callback(this), fulfillerImpl(&callback, std::forward(params)...) {} virtual ~FulfillerPromiseState() {} private: typename PromiseFulfiller::Callback callback; PromiseFulfillerImpl fulfillerImpl; }; // ======================================================================================= template struct CallAndFulfillFunctor { template void operator()(WeakLink* linkToFulfiller, Callback* callback, Func2& func, Params&&... params) const { try { auto result = func(std::forward(params)...); if (linkToFulfiller->isEntangled()) { callback->fulfill(std::move(result)); } } catch (...) { callback->propagateCurrentException(); } } }; template <> struct CallAndFulfillFunctor { template void operator()(WeakLink* linkToFulfiller, Callback* callback, Func2& func, Params&&... params) const { try { func(std::forward(params)...); if (linkToFulfiller->isEntangled()) { callback->fulfill(); } } catch (...) { callback->propagateCurrentException(); } } }; template class DependentPromiseFulfiller : public PromiseFulfiller, public PromiseListener { public: typedef typename PromiseFulfiller::Callback Callback; DependentPromiseFulfiller(Callback* callback, Executor* executor, Func&& func, ExceptionHandler&& exceptionHandler, ParamPack&& params) : callback(callback), executor(executor), func(std::move(func)), exceptionHandler(std::move(exceptionHandler)), params(std::move(params)), dependencyFailed(false), dependenciesLeft(1) { addAllDependencies(); } ~DependentPromiseFulfiller() { if (pendingReadyLater != nullptr) { DEBUG_INFO << "Promise canceled: " << this; } } void dependencyDone(bool failed) { if (failed) { dependencyFailed = true; } if (--dependenciesLeft == 0) { readyLater(); } } Callback* callback; Executor* executor; Func func; ExceptionHandler exceptionHandler; ParamPack params; private: bool dependencyFailed; int dependenciesLeft; OwnedPtr pendingReadyLater; WeakLink weakLink; struct AddDependenciesFunctor { template void operator()(DependentPromiseFulfiller* fulfiller, const Promise& first, Rest&&... rest) const { ++fulfiller->dependenciesLeft; first.state->setListener(fulfiller); operator()(fulfiller, std::forward(rest)...); } template void operator()(DependentPromiseFulfiller* fulfiller, First&& first, Rest&&... rest) const { operator()(fulfiller, std::forward(rest)...); } void operator()(DependentPromiseFulfiller* fulfiller) const { if (--fulfiller->dependenciesLeft == 0) { fulfiller->readyLater(); } } }; void addAllDependencies() { params.apply(AddDependenciesFunctor(), this); } template static U&& unpack(U&& value) { return std::forward(value); } template static typename Unpack>::Type unpack(Promise&& promise) { return promise.state->release(); } struct DoReadyFunctor { template void operator()(WeakLink* linkToFulfiller, Callback* callback, Func& func, Params&&... params) const { CallAndFulfillFunctor(params))...))>()( linkToFulfiller, callback, func, unpack(std::forward(params))...); } }; void ready() { // Move stuff to stack in case promise is deleted during callback. ParamPack localParams(std::move(params)); Func localFunc(std::move(func)); WeakLink linkToFulfiller(&weakLink); localParams.applyMoving(DoReadyFunctor(), &linkToFulfiller, callback, localFunc); } template static U&& unpackMaybeException(U&& value) { return std::forward(value); } template static MaybeException unpackMaybeException(Promise&& promise) { return promise.state->releaseMaybeException(); } struct DoErrorFunctor { template void operator()(WeakLink* linkToFulfiller, Callback* callback, const ExceptionHandler& exceptionHandler, Params&&... params) const { CallAndFulfillFunctor(params))...))>()( linkToFulfiller, callback, exceptionHandler, unpackMaybeException(std::forward(params))...); } }; void error() { // Move stuff to stack in case promise is deleted during callback. ParamPack localParams(std::move(this->params)); ExceptionHandler localExceptionHandler(std::move(this->exceptionHandler)); WeakLink linkToFulfiller(&weakLink); localParams.applyMoving(DoErrorFunctor(), &linkToFulfiller, callback, localExceptionHandler); } void readyLater() { pendingReadyLater = executor->runLater(newLambdaRunnable( [this]() { auto objectToDeleteOnReturn = this->pendingReadyLater.release(); if (this->dependencyFailed) { this->error(); } else { this->ready(); } })); } }; // ======================================================================================= template struct ForceErrorFunctor { template ReturnType operator()(Head&& head, Tail&&... tail) const { return operator()(tail...); } template ReturnType operator()(MaybeException&& head, Tail&&... tail) const { head.get(); return operator()(tail...); } ReturnType operator()() const { throw std::logic_error( "Promise implementation failure: Exception handler called with no exceptions."); } }; } // namespace promiseInternal // ======================================================================================= template class Promise { public: Promise() {} Promise(const Promise& other) = delete; Promise(Promise&& other): state(other.state.release()) { if (state != nullptr) { state->owner = this; } } Promise& operator=(const Promise& other) = delete; Promise& operator=(Promise&& other) { state = other.state.release(); state->owner = this; return *this; } bool operator==(std::nullptr_t) { return state == nullptr; } bool operator!=(std::nullptr_t) { return state != nullptr; } T* operator->() const { return state->getValue()->operator->(); } Promise release() { return std::move(*this); } private: typedef promiseInternal::PromiseState State; explicit Promise(OwnedPtr state): state(state.release()) { while (this->state->chainedPromise != nullptr) { this->state = this->state->chainedPromise.release(); } this->state->owner = this; } OwnedPtr state; friend class Executor; friend struct promiseInternal::PromiseConstructors; template friend class promiseInternal::DependentPromiseFulfiller; friend class promiseInternal::PromiseState; }; inline void PromiseFulfiller::Callback::fulfill(Promise chain) { state->fulfill(chain.release()); } namespace promiseInternal { template void PromiseState::fulfill(Promise chainedPromise) { auto listener = this->listener; auto owner = this->owner; if (owner == nullptr) { throw std::logic_error( "Not supported: Calling fulfill() with a chained promise from the PromiseFulfiller's " "constructor. This is probably not what you intended anyway."); } else { *owner = chainedPromise.release(); // will delete this! if (listener != nullptr) { owner->state->setListener(listener); } } } inline void PromiseState::fulfill(Promise chainedPromise) { auto listener = this->listener; auto owner = this->owner; if (owner == nullptr) { throw std::logic_error( "Not supported: Calling fulfill() with a chained promise from the PromiseFulfiller's " "constructor. This is probably not what you intended anyway."); } else { *owner = chainedPromise.release(); // will delete this! if (listener != nullptr) { owner->state->setListener(listener); } } } struct PromiseConstructors { template static Promise newPromise(Params&&... params) { typedef typename PromiseFulfillerImpl::PromiseType T; return Promise(newOwned>( std::forward(params)...)); } template static Promise::type> newFulfilledPromise(T&& value) { Promise result(newOwned>()); result.state->fulfill(std::forward(value)); return result; } static Promise newFulfilledPromise() { Promise result(newOwned>()); result.state->fulfill(); return result; } template static Promise newPromiseFromCurrentException() { Promise result(newOwned>()); result.state->propagateCurrentException(); return result; } }; } // namespace promiseInternal template Promise newPromise(Params&&... params) { return promiseInternal::PromiseConstructors::newPromise( std::forward(params)...); } template Promise::type> newFulfilledPromise(T&& value) { return promiseInternal::PromiseConstructors::newFulfilledPromise(std::forward(value)); } inline Promise newFulfilledPromise() { return promiseInternal::PromiseConstructors::newFulfilledPromise(); } template Promise newPromiseFromCurrentException() { return promiseInternal::PromiseConstructors::newPromiseFromCurrentException(); } // ======================================================================================= template class Executor::When { typedef ValuePack ParamPack; public: When(const When& other) = delete; When(When&& other) = default; When& operator=(const When& other) = delete; When& operator=(When&& other) = delete; #define RESULT_TYPE \ typename promiseInternal::Chain< \ decltype(func(std::declval::Type>()...))>::Type template auto operator()(Func&& func) -> Promise { typedef RESULT_TYPE ResultType; return operator()(std::move(func), promiseInternal::ForceErrorFunctor()); } template auto operator()(Func&& func, ExceptionHandler&& exceptionHandler) -> Promise { typedef RESULT_TYPE ResultType; typedef promiseInternal::DependentPromiseFulfiller< ResultType, Func, ExceptionHandler, ParamPack> Fulfiller; return newPromise( executor, std::move(func), std::move(exceptionHandler), std::move(params)); } #undef RESULT_TYPE private: Executor* executor; ParamPack params; When(Executor* executor, Types&&... params) : executor(executor), params(std::move(params)...) {} friend class Executor; }; template Executor::When::type...> Executor::when(Types&&... params) { return When::type...>(this, std::move(params)...); } } // namespace ekam #endif // KENTONSCODE_BASE_PROMISE_H_ ================================================ FILE: src/base/Promise_test.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Promise.h" #include #include #include namespace ekam { namespace { class MockExecutor: public Executor { private: class PendingRunnableImpl : public PendingRunnable { public: PendingRunnableImpl(MockExecutor* executor, OwnedPtr runnable) : executor(executor), runnable(runnable.release()) {} ~PendingRunnableImpl() { if (runnable != nullptr) { for (auto iter = executor->queue.begin(); iter != executor->queue.end(); ++iter) { if (*iter == this) { executor->queue.erase(iter); break; } } } } void run() { runnable.release()->run(); } private: MockExecutor* executor; OwnedPtr runnable; }; public: MockExecutor() {} ~MockExecutor() { EXPECT_TRUE(queue.empty()); } void runNext() { ASSERT_FALSE(queue.empty()); auto front = queue.front(); queue.pop_front(); front->run(); } bool empty() { return queue.empty(); } // implements Executor ----------------------------------------------------------------- OwnedPtr runLater(OwnedPtr runnable) { auto result = newOwned(this, runnable.release()); queue.push_back(result.get()); return result.release(); } private: std::deque queue; }; template class MockPromiseFulfiller: public PromiseFulfiller { public: typedef typename PromiseFulfiller::Callback Callback; MockPromiseFulfiller(Callback* callback, Callback** callbackPtr) : callbackPtr(callbackPtr) { *callbackPtr = callback; } ~MockPromiseFulfiller() { *callbackPtr = nullptr; } private: Callback** callbackPtr; }; TEST(PromiseTest, Basic) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller; auto promise = newPromise>(&fulfiller); bool triggered = false; Promise promise2 = mockExecutor.when(promise)( [&triggered](int i) -> int { EXPECT_EQ(5, i); triggered = true; return 123; }); EXPECT_FALSE(triggered); fulfiller->fulfill(5); EXPECT_FALSE(triggered); mockExecutor.runNext(); EXPECT_TRUE(triggered); // Fulfiller deleted because promise has been consumed. EXPECT_TRUE(fulfiller == nullptr); } TEST(PromiseTest, PreFulfilled) { MockExecutor mockExecutor; auto promise = newFulfilledPromise(5); bool triggered = false; Promise promise2 = mockExecutor.when(promise)( [&triggered](int i) -> int { EXPECT_EQ(5, i); triggered = true; return 123; }); EXPECT_FALSE(triggered); mockExecutor.runNext(); EXPECT_TRUE(triggered); } TEST(PromiseTest, Dependent) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller1; auto promise1 = newPromise>(&fulfiller1); PromiseFulfiller::Callback* fulfiller2; auto promise2 = newPromise>(&fulfiller2); Promise promise3 = mockExecutor.when(promise1, promise2)( [](int a, int b) -> int { return a + b; }); int result = 0; Promise promise4 = mockExecutor.when(promise3)( [&result](int a) { result = a; }); EXPECT_TRUE(mockExecutor.empty()); fulfiller1->fulfill(12); EXPECT_TRUE(mockExecutor.empty()); fulfiller2->fulfill(34); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); EXPECT_EQ(result, 46); } TEST(PromiseTest, Chained) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller1; auto promise1 = newPromise>(&fulfiller1); PromiseFulfiller::Callback* fulfiller2; auto promise2 = newPromise>(&fulfiller2); int result = 0; Promise promise3 = mockExecutor.when(promise2)( [&result](int a) { result = a; }); EXPECT_TRUE(mockExecutor.empty()); fulfiller2->fulfill(promise1.release()); EXPECT_TRUE(mockExecutor.empty()); EXPECT_EQ(0, result); fulfiller1->fulfill(123); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); EXPECT_EQ(result, 123); } TEST(PromiseTest, ChainedVoid) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller1; auto promise1 = newPromise>(&fulfiller1); PromiseFulfiller::Callback* fulfiller2; auto promise2 = newPromise>(&fulfiller2); bool triggered = false; Promise promise3 = mockExecutor.when(promise2)( [&triggered](Void) { triggered = true; }); EXPECT_TRUE(mockExecutor.empty()); fulfiller2->fulfill(promise1.release()); EXPECT_TRUE(mockExecutor.empty()); EXPECT_FALSE(triggered); fulfiller1->fulfill(); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); EXPECT_TRUE(triggered); } TEST(PromiseTest, ChainedVoidWhen) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller1; auto promise1 = newPromise>(&fulfiller1); PromiseFulfiller::Callback* fulfiller2 = nullptr; Promise promise3 = mockExecutor.when(promise1)( [&fulfiller2](Void) -> Promise { DEBUG_ERROR << "promise3"; return newPromise>(&fulfiller2); }); bool triggered = false; Promise promise4 = mockExecutor.when(promise3)( [&triggered](Void) { DEBUG_ERROR << "promise4"; triggered = true; }); EXPECT_TRUE(mockExecutor.empty()); fulfiller1->fulfill(); EXPECT_FALSE(mockExecutor.empty()); EXPECT_TRUE(fulfiller2 == nullptr); mockExecutor.runNext(); EXPECT_FALSE(fulfiller2 == nullptr); EXPECT_TRUE(mockExecutor.empty()); EXPECT_FALSE(triggered); ASSERT_TRUE(fulfiller2 != nullptr); fulfiller2->fulfill(); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); EXPECT_TRUE(triggered); } TEST(PromiseTest, ChainedPreFulfilled) { MockExecutor mockExecutor; auto promise1 = newFulfilledPromise(123); PromiseFulfiller::Callback* fulfiller2; auto promise2 = newPromise>(&fulfiller2); int result = 0; Promise promise3 = mockExecutor.when(promise2)( [&result](int a) { result = a; }); EXPECT_TRUE(mockExecutor.empty()); fulfiller2->fulfill(promise1.release()); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); ASSERT_EQ(result, 123); } TEST(PromiseTest, MoveSemantics) { MockExecutor mockExecutor; PromiseFulfiller>::Callback* fulfiller; auto promise = newPromise>>(&fulfiller); OwnedPtr ptr = newOwned(12); int result = 0; Promise promise2 = mockExecutor.when(promise, ptr)( [&result](OwnedPtr i, OwnedPtr j) { result = *i + *j; }); fulfiller->fulfill(newOwned(34)); mockExecutor.runNext(); ASSERT_EQ(result, 46); } TEST(PromiseTest, Cancel) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller; auto promise = newPromise>(&fulfiller); Promise promise2 = mockExecutor.when(promise)( [](int i) { ADD_FAILURE() << "Can't get here."; }); EXPECT_TRUE(mockExecutor.empty()); fulfiller->fulfill(5); EXPECT_FALSE(mockExecutor.empty()); promise2.release(); EXPECT_TRUE(mockExecutor.empty()); } TEST(PromiseTest, VoidPromise) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller; auto promise = newPromise>(&fulfiller); bool triggered = false; Promise promise2 = mockExecutor.when(promise)( [&triggered](Void) { triggered = true; }); EXPECT_FALSE(triggered); fulfiller->fulfill(); EXPECT_FALSE(triggered); mockExecutor.runNext(); EXPECT_TRUE(triggered); } TEST(PromiseTest, Exception) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller1; auto promise1 = newPromise>(&fulfiller1); PromiseFulfiller::Callback* fulfiller2; auto promise2 = newPromise>(&fulfiller2); bool triggered = false; Promise promise3 = mockExecutor.when(promise1, promise2, 123)( [](int i, int j, int k) { ADD_FAILURE() << "Can't get here."; }, [&triggered](MaybeException i, MaybeException j, int k) { triggered = true; ASSERT_TRUE(i.isException()); ASSERT_FALSE(j.isException()); ASSERT_EQ(123, k); ASSERT_EQ(456, j.get()); try { i.get(); ADD_FAILURE() << "Expected exception."; } catch (const std::logic_error& e) { ASSERT_STREQ("test", e.what()); } }); try { throw std::logic_error("test"); } catch (...) { fulfiller1->propagateCurrentException(); } fulfiller2->fulfill(456); EXPECT_FALSE(triggered); mockExecutor.runNext(); EXPECT_TRUE(triggered); } TEST(PromiseTest, ExceptionInCallback) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller1; auto promise1 = newPromise>(&fulfiller1); Promise promise2 = mockExecutor.when(promise1)( [](int a) -> int { throw std::logic_error("test"); }); bool triggered = false; Promise promise3 = mockExecutor.when(promise2)( [](int i) { ADD_FAILURE() << "Can't get here."; }, [&triggered](MaybeException i) { triggered = true; ASSERT_TRUE(i.isException()); try { i.get(); ADD_FAILURE() << "Expected exception."; } catch (const std::logic_error& e) { ASSERT_STREQ("test", e.what()); } }); EXPECT_TRUE(mockExecutor.empty()); fulfiller1->fulfill(12); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); ASSERT_TRUE(triggered); } TEST(PromiseTest, ExceptionPropagation) { MockExecutor mockExecutor; PromiseFulfiller::Callback* fulfiller1; auto promise1 = newPromise>(&fulfiller1); Promise promise2 = mockExecutor.when(promise1)( [](int a) { ADD_FAILURE() << "Can't get here."; }); bool triggered = false; Promise promise3 = mockExecutor.when(promise2)( [](Void) { ADD_FAILURE() << "Can't get here."; }, [&triggered](MaybeException i) { triggered = true; ASSERT_TRUE(i.isException()); try { i.get(); ADD_FAILURE() << "Expected exception."; } catch (const std::logic_error& e) { ASSERT_STREQ("test", e.what()); } }); EXPECT_TRUE(mockExecutor.empty()); try { throw std::logic_error("test"); } catch (...) { fulfiller1->propagateCurrentException(); } EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); EXPECT_FALSE(mockExecutor.empty()); mockExecutor.runNext(); ASSERT_TRUE(triggered); } } // namespace } // namespace ekam ================================================ FILE: src/base/Table.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_BASE_TABLE_H_ #define KENTONSCODE_BASE_TABLE_H_ #include #include #include namespace ekam { // ======================================================================================= // Internal helpers. Please ignore. template class DummyMap { public: typedef std::pair value_type; typedef value_type* iterator; typedef const value_type* const_iterator; inline iterator begin() { return NULL; } inline iterator end() { return NULL; } inline const_iterator begin() const { return NULL; } inline const_iterator end() const { return NULL; } inline iterator insert(const value_type& value) { return NULL; } inline int erase(const iterator& iter) { return 0; } inline void swap(DummyMap& other) {} }; template struct ChooseType; template struct ChooseType : public Choices::Choice0 {}; template struct ChooseType : public Choices::Choice1 {}; template struct ChooseType : public Choices::Choice2 {}; // ======================================================================================= template , typename Eq = std::equal_to > struct IndexedColumn { typedef T Value; typedef std::unordered_multimap Index; }; template , typename Eq = std::equal_to > struct UniqueColumn { typedef T Value; typedef std::unordered_map Index; }; template , typename Eq = std::equal_to > struct Column { typedef T Value; typedef DummyMap Index; }; struct EmptyColumn { struct Value {}; typedef DummyMap Index; }; template class Table { private: struct Columns { #define CHOICE(INDEX) \ struct Choice##INDEX : public Column##INDEX { \ static inline typename Column##INDEX::Index* index(Table* table) { \ return &table->index##INDEX; \ } \ static inline const typename Column##INDEX::Index& index(const Table& table) { \ return table.index##INDEX; \ } \ template \ static inline const typename Column##INDEX::Value& cell(const Row& row) { \ return row.cell##INDEX; \ } \ }; CHOICE(0) CHOICE(1) CHOICE(2) #undef CHOICE }; template class Column : public ChooseType {}; public: Table() : deletedCount(0) {} ~Table() {} class Row { public: inline Row() {} template inline const typename Column::Value& cell() const { return Column::cell(*this); } private: typename Column<0>::Value cell0; typename Column<1>::Value cell1; typename Column<2>::Value cell2; bool deleted; inline Row(const typename Column<0>::Value& cell0, const typename Column<1>::Value& cell1, const typename Column<2>::Value& cell2) : cell0(cell0), cell1(cell1), cell2(cell2), deleted(false) {} friend class Table; }; class RowIterator { public: inline RowIterator() {} RowIterator(const Table& table) : current(NULL), nextIter(table.rows.begin()), end(table.rows.end()) {} bool next() { while (true) { if (nextIter == end) { return false; } else if (!nextIter->deleted) { current = &*nextIter; ++nextIter; return true; } ++nextIter; } } template inline const typename Column::Value& cell() const { return Column::cell(*current); } private: const Row* current; typename std::vector::const_iterator nextIter; const typename std::vector::const_iterator end; }; template class SearchIterator { public: inline SearchIterator() {} SearchIterator(const Table& table, const typename Column::Value& value) : table(table), current(NULL), range(Column::index(table).equal_range(value)) {} inline bool next() { while (true) { if (range.first == range.second) { return false; } const Row* row = &table.rows[range.first->second]; if (!row->deleted) { current = row; ++range.first; return true; } ++range.first; } } template inline const typename Column::Value& cell() const { return Column::cell(*current); } private: typedef typename Column::Index::const_iterator InnerIter; const Table& table; const Row* current; std::pair range; }; template const Row* find(const typename Column::Value& value) const { typename Column::Index::const_iterator iter = Column::index(*this).find(value); if (iter == Column::index(*this).end()) { return NULL; } else { const Row* row = &rows[iter->second]; return row->deleted ? NULL : row; } } template const size_t erase(const typename Column::Value& value) { typedef typename Column::Index::iterator ColumnIterator; std::pair range = Column::index(this)->equal_range(value); size_t count = 0; for (ColumnIterator iter = range.first; iter != range.second; ++iter) { if (!rows[iter->second].deleted) { rows[iter->second].deleted = true; ++count; } } deletedCount += count; if (deletedCount >= 16 && deletedCount > rows.size() / 2) { refresh(); } else { // This isn't strictly necessary, but since it's convenient, let's delete the range // from the index. Column::index(this)->erase(range.first, range.second); } return count; } void add(const typename Column<0>::Value& value0, const typename Column<1>::Value& value1 = EmptyColumn::Value(), const typename Column<2>::Value& value2 = EmptyColumn::Value()) { handleInsertResult(index0.insert(typename Column<0>::Index::value_type(value0, rows.size()))); handleInsertResult(index1.insert(typename Column<1>::Index::value_type(value1, rows.size()))); handleInsertResult(index2.insert(typename Column<2>::Index::value_type(value2, rows.size()))); rows.push_back(Row(value0, value1, value2)); } template const bool has(const typename Column::Value& value) { typedef typename Column::Index::iterator ColumnIterator; std::pair range = Column::index(this)->equal_range(value); for (ColumnIterator iter = range.first; iter != range.second; ++iter) { if (!rows[iter->second].deleted) { return true; } } return false; } int size() { return rows.size() - deletedCount; } int capacity() { return rows.capacity(); } template int indexSize() { return Column::index(this)->size(); } private: std::vector rows; size_t deletedCount; typename Column<0>::Index index0; typename Column<1>::Index index1; typename Column<2>::Index index2; void refresh() { std::vector newLocations; newLocations.reserve(rows.size()); { std::vector newRows; newRows.reserve(rows.size() - deletedCount); for (size_t i = 0; i < rows.size(); i++) { if (rows[i].deleted) { newLocations.push_back(-1); } else { newLocations.push_back(newRows.size()); newRows.push_back(rows[i]); } } rows.swap(newRows); deletedCount = 0; } refreshColumn<0>(newLocations); refreshColumn<1>(newLocations); refreshColumn<2>(newLocations); } template void refreshColumn(const std::vector& newLocations) { typedef Column C; typedef typename C::Index::iterator ColumnIterator; typename C::Index newIndex; for (ColumnIterator iter = C::index(this)->begin(); iter != C::index(this)->end(); ++iter) { int newLocation = newLocations[iter->second]; if (newLocation != -1) { newIndex.insert(std::make_pair(iter->first, newLocation)); } } C::index(this)->swap(newIndex); } template void handleInsertResult(const std::pair& result) { if (!result.second) { rows[result.first->second].deleted = true; result.first->second = rows.size(); } } template void handleInsertResult(const Iterator& iter) {} }; } // namespace ekam #endif // KENTONSCODE_BASE_TABLE_H_ ================================================ FILE: src/base/Table_test.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Table.h" #include #include #include #include #include namespace ekam { namespace { #define ASSERT(EXPRESSION) \ if (!(EXPRESSION)) { \ fprintf(stderr, "%s:%d: FAILED: %s\n", __FILE__, __LINE__, #EXPRESSION); \ exit(1); \ } void testTable() { { typedef Table > MyTable; MyTable table; table.add(1234); table.add(5678); const MyTable::Row* row = table.find<0>(1234); ASSERT(table.find<0>(4321) == NULL); ASSERT(row != NULL); ASSERT(row->cell<0>() == 1234); row = table.find<0>(5678); ASSERT(row != NULL); ASSERT(row->cell<0>() == 5678); { std::multiset values; MyTable::RowIterator iter(table); ASSERT(iter.next()); values.insert(iter.cell<0>()); ASSERT(iter.next()); values.insert(iter.cell<0>()); ASSERT(!iter.next()); ASSERT(values.count(1234) == 1); ASSERT(values.count(5678) == 1); } table.erase<0>(1234); ASSERT(table.find<0>(4321) == NULL); ASSERT(table.find<0>(1234) == NULL); row = table.find<0>(5678); ASSERT(row != NULL); ASSERT(row->cell<0>() == 5678); } { typedef Table, IndexedColumn > MyTable; MyTable table; table.add(12, 34); table.add(56, 34); { std::multiset values; MyTable::SearchIterator<1> iter(table, 34); ASSERT(iter.next()); values.insert(iter.cell<0>()); ASSERT(iter.next()); values.insert(iter.cell<0>()); ASSERT(!iter.next()); ASSERT(values.count(12) == 1); ASSERT(values.count(56) == 1); } table.add(12, 78); const MyTable::Row* row = table.find<0>(12); ASSERT(row != NULL); ASSERT(row->cell<0>() == 12); ASSERT(row->cell<1>() == 78); { MyTable::SearchIterator<1> iter(table, 34); ASSERT(iter.next()); ASSERT(iter.cell<0>() == 56); ASSERT(!iter.next()); } } { typedef Table, IndexedColumn, Column > MyTable; MyTable table; table.add("foo", 1, 'f'); table.add("foo", 2, 'o'); table.add("foo", 3, 'o'); table.add("bar", 1, 'b'); table.add("bar", 2, 'a'); table.add("bar", 3, 'r'); { std::map values; MyTable::SearchIterator<0> iter(table, "bar"); ASSERT(iter.next()); ASSERT(iter.cell<0>() == "bar"); values[iter.cell<1>()] = iter.cell<2>(); ASSERT(iter.next()); ASSERT(iter.cell<0>() == "bar"); values[iter.cell<1>()] = iter.cell<2>(); ASSERT(iter.next()); ASSERT(iter.cell<0>() == "bar"); values[iter.cell<1>()] = iter.cell<2>(); ASSERT(!iter.next()); ASSERT(values[1] == 'b'); ASSERT(values[2] == 'a'); ASSERT(values[3] == 'r'); } { std::map values; MyTable::SearchIterator<1> iter(table, 2); ASSERT(iter.next()); ASSERT(iter.cell<1>() == 2); values[iter.cell<0>()] = iter.cell<2>(); ASSERT(iter.next()); ASSERT(iter.cell<1>() == 2); values[iter.cell<0>()] = iter.cell<2>(); ASSERT(!iter.next()); ASSERT(values["foo"] == 'o'); ASSERT(values["bar"] == 'a'); } } { typedef Table, IndexedColumn > MyTable; MyTable table; for (int i = 0; i < 50; i++) { table.add(123, i); table.add(456, i); table.add(789, i); } ASSERT(table.size() == 150); ASSERT(table.capacity() >= 150); ASSERT(table.indexSize<0>() == 150); ASSERT(table.indexSize<1>() == 150); table.erase<0>(123); ASSERT(table.size() == 100); ASSERT(table.capacity() >= 150); ASSERT(table.indexSize<0>() == 100); ASSERT(table.indexSize<1>() == 150); table.erase<0>(456); ASSERT(table.size() == 50); ASSERT(table.capacity() == 50); ASSERT(table.indexSize<0>() == 50); ASSERT(table.indexSize<1>() == 50); const MyTable::Row* row = table.find<1>(5); ASSERT(row != NULL); ASSERT(row->cell<0>() == 789); { std::multiset values; MyTable::SearchIterator<0> iter(table, 789); for (int i = 0; i < 50; i++) { ASSERT(iter.next()); values.insert(iter.cell<1>()); } ASSERT(!iter.next()); for (int i = 0; i < 50; i++) { ASSERT(values.count(i) == 1); } } ASSERT(!MyTable::SearchIterator<0>(table, 123).next()); ASSERT(!MyTable::SearchIterator<0>(table, 456).next()); { std::multiset values0; std::multiset values1; MyTable::RowIterator iter(table); for (int i = 0; i < 50; i++) { ASSERT(iter.next()); values0.insert(iter.cell<0>()); values1.insert(iter.cell<1>()); } ASSERT(!iter.next()); ASSERT(values0.count(789) == 50); for (int i = 0; i < 50; i++) { ASSERT(values1.count(i) == 1); } } } } } // namespace } // namespace ekam int main(int argc, char* argv[]) { ekam::testTable(); return 0; } ================================================ FILE: src/base/sha256.cpp ================================================ /******************************************************************** * Pieces of endian.h from FreeBSD * ********************************************************************/ /*- * Copyright (c) 2002 Thomas Moestl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD: src/sys/sys/endian.h,v 1.6.32.1 2009/08/03 08:13:06 kensmith Exp $ */ #include inline void be32enc(void *pp, uint32_t u) { unsigned char *p = (unsigned char *)pp; p[0] = (u >> 24) & 0xff; p[1] = (u >> 16) & 0xff; p[2] = (u >> 8) & 0xff; p[3] = u & 0xff; } inline uint32_t be32dec(const void *pp) { unsigned char const *p = (unsigned char const *)pp; return ((p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]); } /******************************************************************** * sha256c.c from FreeBSD's libmd * * Modified to compile as C++ and placed in ekam namespace. * ********************************************************************/ /*- * Copyright 2005 Colin Percival * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* #include __FBSDID("$FreeBSD: src/lib/libmd/sha256c.c,v 1.2.10.1 2009/08/03 08:13:06 kensmith Exp $"); */ #include #include #include "sha256.h" namespace ekam { #if BYTE_ORDER == BIG_ENDIAN /* Copy a vector of big-endian uint32_t into a vector of bytes */ #define be32enc_vect(dst, src, len) \ memcpy((void *)dst, (const void *)src, (size_t)len) /* Copy a vector of bytes into a vector of big-endian uint32_t */ #define be32dec_vect(dst, src, len) \ memcpy((void *)dst, (const void *)src, (size_t)len) #else /* BYTE_ORDER != BIG_ENDIAN */ /* * Encode a length len/4 vector of (uint32_t) into a length len vector of * (unsigned char) in big-endian form. Assumes len is a multiple of 4. */ static void be32enc_vect(unsigned char *dst, const uint32_t *src, size_t len) { size_t i; for (i = 0; i < len / 4; i++) be32enc(dst + i * 4, src[i]); } /* * Decode a big-endian length len vector of (unsigned char) into a length * len/4 vector of (uint32_t). Assumes len is a multiple of 4. */ static void be32dec_vect(uint32_t *dst, const unsigned char *src, size_t len) { size_t i; for (i = 0; i < len / 4; i++) dst[i] = be32dec(src + i * 4); } #endif /* BYTE_ORDER != BIG_ENDIAN */ /* Elementary functions used by SHA256 */ #define Ch(x, y, z) ((x & (y ^ z)) ^ z) #define Maj(x, y, z) ((x & (y | z)) | (y & z)) #define SHR(x, n) (x >> n) #define ROTR(x, n) ((x >> n) | (x << (32 - n))) #define S0(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) #define S1(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) #define s0(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHR(x, 3)) #define s1(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHR(x, 10)) /* SHA256 round function */ #define RND(a, b, c, d, e, f, g, h, k) \ t0 = h + S1(e) + Ch(e, f, g) + k; \ t1 = S0(a) + Maj(a, b, c); \ d += t0; \ h = t0 + t1; /* Adjusted round function for rotating state */ #define RNDr(S, W, i, k) \ RND(S[(64 - i) % 8], S[(65 - i) % 8], \ S[(66 - i) % 8], S[(67 - i) % 8], \ S[(68 - i) % 8], S[(69 - i) % 8], \ S[(70 - i) % 8], S[(71 - i) % 8], \ W[i] + k) /* * SHA256 block compression function. The 256-bit state is transformed via * the 512-bit input block to produce a new state. */ static void SHA256_Transform(uint32_t * state, const unsigned char block[64]) { uint32_t W[64]; uint32_t S[8]; uint32_t t0, t1; int i; /* 1. Prepare message schedule W. */ be32dec_vect(W, block, 64); for (i = 16; i < 64; i++) W[i] = s1(W[i - 2]) + W[i - 7] + s0(W[i - 15]) + W[i - 16]; /* 2. Initialize working variables. */ memcpy(S, state, 32); /* 3. Mix. */ RNDr(S, W, 0, 0x428a2f98); RNDr(S, W, 1, 0x71374491); RNDr(S, W, 2, 0xb5c0fbcf); RNDr(S, W, 3, 0xe9b5dba5); RNDr(S, W, 4, 0x3956c25b); RNDr(S, W, 5, 0x59f111f1); RNDr(S, W, 6, 0x923f82a4); RNDr(S, W, 7, 0xab1c5ed5); RNDr(S, W, 8, 0xd807aa98); RNDr(S, W, 9, 0x12835b01); RNDr(S, W, 10, 0x243185be); RNDr(S, W, 11, 0x550c7dc3); RNDr(S, W, 12, 0x72be5d74); RNDr(S, W, 13, 0x80deb1fe); RNDr(S, W, 14, 0x9bdc06a7); RNDr(S, W, 15, 0xc19bf174); RNDr(S, W, 16, 0xe49b69c1); RNDr(S, W, 17, 0xefbe4786); RNDr(S, W, 18, 0x0fc19dc6); RNDr(S, W, 19, 0x240ca1cc); RNDr(S, W, 20, 0x2de92c6f); RNDr(S, W, 21, 0x4a7484aa); RNDr(S, W, 22, 0x5cb0a9dc); RNDr(S, W, 23, 0x76f988da); RNDr(S, W, 24, 0x983e5152); RNDr(S, W, 25, 0xa831c66d); RNDr(S, W, 26, 0xb00327c8); RNDr(S, W, 27, 0xbf597fc7); RNDr(S, W, 28, 0xc6e00bf3); RNDr(S, W, 29, 0xd5a79147); RNDr(S, W, 30, 0x06ca6351); RNDr(S, W, 31, 0x14292967); RNDr(S, W, 32, 0x27b70a85); RNDr(S, W, 33, 0x2e1b2138); RNDr(S, W, 34, 0x4d2c6dfc); RNDr(S, W, 35, 0x53380d13); RNDr(S, W, 36, 0x650a7354); RNDr(S, W, 37, 0x766a0abb); RNDr(S, W, 38, 0x81c2c92e); RNDr(S, W, 39, 0x92722c85); RNDr(S, W, 40, 0xa2bfe8a1); RNDr(S, W, 41, 0xa81a664b); RNDr(S, W, 42, 0xc24b8b70); RNDr(S, W, 43, 0xc76c51a3); RNDr(S, W, 44, 0xd192e819); RNDr(S, W, 45, 0xd6990624); RNDr(S, W, 46, 0xf40e3585); RNDr(S, W, 47, 0x106aa070); RNDr(S, W, 48, 0x19a4c116); RNDr(S, W, 49, 0x1e376c08); RNDr(S, W, 50, 0x2748774c); RNDr(S, W, 51, 0x34b0bcb5); RNDr(S, W, 52, 0x391c0cb3); RNDr(S, W, 53, 0x4ed8aa4a); RNDr(S, W, 54, 0x5b9cca4f); RNDr(S, W, 55, 0x682e6ff3); RNDr(S, W, 56, 0x748f82ee); RNDr(S, W, 57, 0x78a5636f); RNDr(S, W, 58, 0x84c87814); RNDr(S, W, 59, 0x8cc70208); RNDr(S, W, 60, 0x90befffa); RNDr(S, W, 61, 0xa4506ceb); RNDr(S, W, 62, 0xbef9a3f7); RNDr(S, W, 63, 0xc67178f2); /* 4. Mix local working variables into global state */ for (i = 0; i < 8; i++) state[i] += S[i]; } static unsigned char PAD[64] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; /* Add padding and terminating bit-count. */ static void SHA256_Pad(SHA256_CTX * ctx) { unsigned char len[8]; uint32_t r, plen; /* * Convert length to a vector of bytes -- we do this now rather * than later because the length will change after we pad. */ be32enc_vect(len, ctx->count, 8); /* Add 1--64 bytes so that the resulting length is 56 mod 64 */ r = (ctx->count[1] >> 3) & 0x3f; plen = (r < 56) ? (56 - r) : (120 - r); SHA256_Update(ctx, PAD, (size_t)plen); /* Add the terminating bit-count */ SHA256_Update(ctx, len, 8); } /* SHA-256 initialization. Begins a SHA-256 operation. */ void SHA256_Init(SHA256_CTX * ctx) { /* Zero bits processed so far */ ctx->count[0] = ctx->count[1] = 0; /* Magic initialization constants */ ctx->state[0] = 0x6A09E667; ctx->state[1] = 0xBB67AE85; ctx->state[2] = 0x3C6EF372; ctx->state[3] = 0xA54FF53A; ctx->state[4] = 0x510E527F; ctx->state[5] = 0x9B05688C; ctx->state[6] = 0x1F83D9AB; ctx->state[7] = 0x5BE0CD19; } /* Add bytes into the hash */ void SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len) { uint32_t bitlen[2]; uint32_t r; const unsigned char *src = reinterpret_cast(in); /* Number of bytes left in the buffer from previous updates */ r = (ctx->count[1] >> 3) & 0x3f; /* Convert the length into a number of bits */ bitlen[1] = ((uint32_t)len) << 3; bitlen[0] = (uint32_t)(len >> 29); /* Update number of bits */ if ((ctx->count[1] += bitlen[1]) < bitlen[1]) ctx->count[0]++; ctx->count[0] += bitlen[0]; /* Handle the case where we don't need to perform any transforms */ if (len < 64 - r) { memcpy(&ctx->buf[r], src, len); return; } /* Finish the current block */ memcpy(&ctx->buf[r], src, 64 - r); SHA256_Transform(ctx->state, ctx->buf); src += 64 - r; len -= 64 - r; /* Perform complete blocks */ while (len >= 64) { SHA256_Transform(ctx->state, src); src += 64; len -= 64; } /* Copy left over data into buffer */ memcpy(ctx->buf, src, len); } /* * SHA-256 finalization. Pads the input data, exports the hash value, * and clears the context state. */ void SHA256_Final(unsigned char digest[32], SHA256_CTX * ctx) { /* Add padding */ SHA256_Pad(ctx); /* Write the hash */ be32enc_vect(digest, ctx->state, 32); /* Clear the context state */ memset((void *)ctx, 0, sizeof(*ctx)); } } // namespace ekam ================================================ FILE: src/base/sha256.h ================================================ /******************************************************************** * sha256.h from FreeBSD's libmd * * Modified to compile as C++ and placed in ekam namespace. * ********************************************************************/ /*- * Copyright 2005 Colin Percival * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD: src/lib/libmd/sha256.h,v 1.2.10.1 2009/08/03 08:13:06 kensmith Exp $ */ #ifndef _SHA256_H_ #define _SHA256_H_ #include namespace ekam { typedef struct SHA256Context { uint32_t state[8]; uint32_t count[2]; unsigned char buf[64]; } SHA256_CTX; void SHA256_Init(SHA256_CTX *); void SHA256_Update(SHA256_CTX *, const void *, size_t); void SHA256_Final(unsigned char [32], SHA256_CTX *); char *SHA256_End(SHA256_CTX *, char *); char *SHA256_File(const char *, char *); char *SHA256_FileChunk(const char *, char *, off_t, off_t); char *SHA256_Data(const void *, unsigned int, char *); } // namespace ekam #endif /* !_SHA256_H_ */ ================================================ FILE: src/ekam/Action.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Action.h" namespace ekam { BuildContext::~BuildContext() noexcept(false) {} Action::~Action() {} ActionFactory::~ActionFactory() {} const int BuildContext::INSTALL_LOCATION_COUNT; const char* const BuildContext::INSTALL_LOCATION_NAMES[INSTALL_LOCATION_COUNT] = { "bin", "lib", "node_modules" }; } // namespace ekam ================================================ FILE: src/ekam/Action.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_ACTION_H_ #define KENTONSCODE_EKAM_ACTION_H_ #include #include #include #include "base/OwnedPtr.h" #include "os/File.h" #include "Tag.h" #include "os/EventManager.h" namespace ekam { class ActionFactory; class ProcessExitCallback { public: virtual ~ProcessExitCallback(); // Negative = signal number. virtual void done(int exit_status) = 0; }; class BuildContext { public: virtual ~BuildContext() noexcept(false); virtual File* findProvider(Tag id) = 0; virtual File* findInput(const std::string& path) = 0; enum InstallLocation { BIN, LIB, NODE_MODULES }; static const int INSTALL_LOCATION_COUNT = 3; static const char* const INSTALL_LOCATION_NAMES[INSTALL_LOCATION_COUNT]; virtual void provide(File* file, const std::vector& tags) = 0; virtual void install(File* file, InstallLocation location, const std::string& name) = 0; virtual void log(const std::string& text) = 0; virtual OwnedPtr newOutput(const std::string& path) = 0; virtual void addActionType(OwnedPtr factory) = 0; virtual void passed() = 0; virtual void failed() = 0; }; class Action { public: virtual ~Action(); virtual bool isSilent() { return false; } virtual std::string getVerb() = 0; virtual Promise start(EventManager* eventManager, BuildContext* context) = 0; }; class ActionFactory { public: virtual ~ActionFactory(); virtual void enumerateTriggerTags(std::back_insert_iterator > iter) = 0; virtual OwnedPtr tryMakeAction(const Tag& id, File* file) = 0; }; } // namespace ekam #endif // KENTONSCODE_EKAM_ACTION_H_ ================================================ FILE: src/ekam/ActionUtil.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "ActionUtil.h" #include namespace ekam { Logger::Logger(BuildContext* context, OwnedPtr stream) : context(context), stream(stream.release()) {} Logger::~Logger() {} Promise Logger::run(EventManager* eventManager) { return eventManager->when(stream->readAsync(eventManager, buffer, sizeof(buffer)))( [=](size_t size) -> Promise { if (size == 0) { return newFulfilledPromise(); } context->log(std::string(buffer, size)); return run(eventManager); }, [=](MaybeException error) { try { error.get(); } catch (const std::exception& e) { context->log(e.what()); context->failed(); throw; } catch (...) { context->log("unknown exception"); context->failed(); throw; } }); } // ======================================================================================= LineReader::LineReader(ByteStream* stream) : stream(stream) {} LineReader::~LineReader() {} Promise> LineReader::readLine(EventManager* eventManager) { std::string::size_type endpos = leftover.find_first_of('\n'); if (endpos != std::string::npos) { auto result = newOwned(leftover, 0, endpos); leftover.erase(0, endpos + 1); return newFulfilledPromise(result.release()); } return eventManager->when(stream->readAsync(eventManager, buffer, sizeof(buffer)))( [=](size_t size) -> Promise> { if (size == 0) { if (leftover.empty()) { // No more data. return newFulfilledPromise(OwnedPtr(nullptr)); } else { // Still have a line of text that had no trailing newline. auto line = newOwned(std::move(leftover)); leftover.clear(); return newFulfilledPromise(line.release()); } } leftover.append(buffer, size); return readLine(eventManager); }); } } // namespace ekam ================================================ FILE: src/ekam/ActionUtil.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_ACTIONUTIL_H_ #define KENTONSCODE_EKAM_ACTIONUTIL_H_ #include #include "os/ByteStream.h" #include "Action.h" namespace ekam { class Logger { public: Logger(BuildContext* context, OwnedPtr stream); ~Logger(); Promise run(EventManager* eventManager); private: class ReadAllFulfiller; BuildContext* context; OwnedPtr stream; char buffer[4096]; }; class LineReader { public: LineReader(ByteStream* stream); ~LineReader(); Promise> readLine(EventManager* eventManager); private: ByteStream* stream; std::string leftover; char buffer[4096]; }; } // namespace ekam #endif // KENTONSCODE_EKAM_ACTIONUTIL_H_ ================================================ FILE: src/ekam/ConsoleDashboard.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "ConsoleDashboard.h" #include #include #include #include #include "base/Debug.h" namespace ekam { class ConsoleDashboard::LogFormatter { public: LogFormatter(const std::string& text) : text(text.data()), end(text.data() + text.size()), wrapped(false) { eatWhitespace(); } inline bool atEnd() { return text == end; } std::string getLine(int startColumn, int windowWidth) { std::string result; int column = startColumn; result.reserve((windowWidth - column) * 2); if (wrapped) { result.append(" "); column += 2; wrapped = false; } while (text < end && *text != '\n' && column < windowWidth) { if (isalnum(*text)) { const char* wordEnd = text; do { ++wordEnd; } while (wordEnd < end && isalnum(*wordEnd)); int length = wordEnd - text; if (column + length <= windowWidth) { // Word fits on this line. bool isColored = false; if (strncasecmp(text, "error", length) == 0 || strncasecmp(text, "fail", length) == 0 || strncasecmp(text, "failed", length) == 0 || strncasecmp(text, "fatal", length) == 0) { result.append(ANSI_COLOR_CODES[RED]); isColored = true; } else if (strncasecmp(text, "warning", length) == 0) { result.append(ANSI_COLOR_CODES[YELLOW]); isColored = true; } result.append(text, wordEnd); text = wordEnd; column += length; if (isColored) { result.append(ANSI_CLEAR_COLOR); } } else if (length >= 20 || column == startColumn) { // Really long word. Break it. int spaceAvailable = windowWidth - column; result.append(text, spaceAvailable); text += spaceAvailable; column = windowWidth; } else { // Word doesn't fit in remaining space. End the line. break; } } else { switch (*text) { case '\t': column = (column & ~0x7) + 8; result.push_back(*text); break; case '\033': // escape // ignore -- could be harmful // TODO: Parse and remove the subsequent terminal instruction to make the output not // suck. Or better yet, parse the sequence and interpret it. If it is an SGR code // (e.g. color), pass it through, and just make sure to reset later. break; default: if (*text >= '\0' && *text < ' ') { // Ignore control character or weird whitespace. } else { result.push_back(*text); ++column; } break; } ++text; } } wrapped = !eatWhitespace(); return result; } private: const char* text; const char* end; bool wrapped; // Eat whitespace up to and including a newline. Returns true if a newline was eaten. bool eatWhitespace() { while (text < end && isspace(*text) && *text != '\n') { ++text; } if (text < end && *text == '\n') { ++text; return true; } else { return false; } } bool tryHighlight(const char* word, Color color, int windowWidth, int* column, std::string* output) { int length = strlen(word); if (windowWidth - *column < length) { // Don't highlight words broken at line wrapping. return false; } if (end - text < length) { // Not enough characters left to match word. return false; } if (strncasecmp(text, word, length) != 0) { // Does not match. return false; } if (end - text > length && isalpha(text[length])) { // Character after word is a letter. Don't highlight. return false; } if (!output->empty() && isalpha((*output)[output->size() - 1])) { // Character before word was a letter. Don't highlight. return false; } output->append(ANSI_COLOR_CODES[color]); output->append(text, text + length); output->append(ANSI_CLEAR_COLOR); text += length; *column += length; return true; } }; class ConsoleDashboard::TaskImpl : public Dashboard::Task { public: TaskImpl(ConsoleDashboard* dashboard, const std::string& verb, const std::string& noun, Silence silence); ~TaskImpl(); // implements Task --------------------------------------------------------------------- void setState(TaskState state); void addOutput(const std::string& text); private: ConsoleDashboard* dashboard; TaskState state; Silence silence; std::string verb; std::string noun; std::string outputText; void removeFromRunning(); void writeFinalLog(Color verbColor, const char* icon); friend class ConsoleDashboard; }; ConsoleDashboard::TaskImpl::TaskImpl(ConsoleDashboard* dashboard, const std::string& verb, const std::string& noun, Silence silence) : dashboard(dashboard), state(PENDING), silence(silence), verb(verb), noun(noun) {} ConsoleDashboard::TaskImpl::~TaskImpl() { if (state == RUNNING) { dashboard->clearRunning(); removeFromRunning(); dashboard->drawRunning(); } } void ConsoleDashboard::TaskImpl::setState(TaskState state) { // If state was previously BLOCKED, and we managed to un-block, then we don't care about the // reason why we were blocked, so clear the text. if (this->state == BLOCKED && (state == PENDING || state == RUNNING)) { outputText.clear(); } if (this->state == RUNNING) { if (silence != SILENT) removeFromRunning(); } this->state = state; dashboard->clearRunning(); switch (state) { case PENDING: // Don't display. break; case RUNNING: if (silence != SILENT) dashboard->runningTasks.push_back(this); break; case DONE: writeFinalLog(DONE_COLOR, " "); break; case PASSED: writeFinalLog(PASSED_COLOR, "✔"); break; case FAILED: writeFinalLog(FAILED_COLOR, "✘"); break; case BLOCKED: // Don't display. break; } dashboard->drawRunning(); } void ConsoleDashboard::TaskImpl::addOutput(const std::string& text) { outputText.append(text); } void ConsoleDashboard::TaskImpl::removeFromRunning() { for (std::vector::iterator iter = dashboard->runningTasks.begin(); iter != dashboard->runningTasks.end(); ++iter) { if (*iter == this) { dashboard->runningTasks.erase(iter); break; } } } void ConsoleDashboard::TaskImpl::writeFinalLog(Color verbColor, const char* icon) { // Silent tasks should not be written to the log, unless they had error messages. if (silence != SILENT || !outputText.empty()) { fprintf(dashboard->out, "%s%s %s:%s %s\n", ANSI_COLOR_CODES[verbColor], icon, verb.c_str(), ANSI_CLEAR_COLOR, noun.c_str()); // Write any output we have buffered. if (!outputText.empty()) { LogFormatter formatter(outputText); struct winsize windowSize; ioctl(dashboard->fd, TIOCGWINSZ, &windowSize); for (int i = 0; i < dashboard->maxDisplayedLogLines && !formatter.atEnd(); i++) { std::string line = formatter.getLine(4, windowSize.ws_col); fprintf(dashboard->out, " %s\n", line.c_str()); } if (!formatter.atEnd()) { fprintf(dashboard->out, " ...(log truncated; use -l to increase log limit)...\n"); } outputText.clear(); } } } // ======================================================================================= const char* const ConsoleDashboard::ANSI_COLOR_CODES[] = { "\033[30m", "\033[31m", "\033[32m", "\033[33m", "\033[34m", "\033[35m", "\033[36m", "\033[37m", "\033[1;30m", "\033[1;31m", "\033[1;32m", "\033[1;33m", "\033[1;34m", "\033[1;35m", "\033[1;36m", "\033[1;37m" }; const char* const ConsoleDashboard::ANSI_CLEAR_COLOR = "\033[0m"; // Note: \033[%dF (move cursor up %d lines and to beginning of line) doesn't work on some // terminals. \033[%dA (move cursor up %d lines) does appear to work, so tack \r on to that // to go to the beginning of the line. const char* const ConsoleDashboard::ANSI_MOVE_CURSOR_UP = "\033[%dA\r"; const char* const ConsoleDashboard::ANSI_CLEAR_BELOW_CURSOR = "\033[0J"; const ConsoleDashboard::Color ConsoleDashboard::DONE_COLOR = BRIGHT_BLUE; const ConsoleDashboard::Color ConsoleDashboard::PASSED_COLOR = BRIGHT_GREEN; const ConsoleDashboard::Color ConsoleDashboard::FAILED_COLOR = BRIGHT_RED; const ConsoleDashboard::Color ConsoleDashboard::RUNNING_COLOR = BRIGHT_FUCHSIA; ConsoleDashboard::ConsoleDashboard(FILE* output, int maxDisplayedLogLines) : fd(fileno(output)), out(output), maxDisplayedLogLines(maxDisplayedLogLines), runningTasksLineCount(0), lastDebugMessageCount(DebugMessage::getMessageCount()) {} ConsoleDashboard::~ConsoleDashboard() {} OwnedPtr ConsoleDashboard::beginTask( const std::string& verb, const std::string& noun, Silence silence) { return newOwned(this, verb, noun, silence); } void ConsoleDashboard::clearRunning() { if (lastDebugMessageCount != DebugMessage::getMessageCount()) { // Some debug messages were printed. We don't want to clobber them. So we can't clear. return; } if (runningTasksLineCount > 0) { fprintf(out, ANSI_MOVE_CURSOR_UP, runningTasksLineCount); fputs(ANSI_CLEAR_BELOW_CURSOR, out); } } void ConsoleDashboard::drawRunning() { struct winsize windowSize; ioctl(fd, TIOCGWINSZ, &windowSize); // Leave a few lines for completed tasks. int spaceForTasks = windowSize.ws_row - 4; bool allTasksShown = static_cast(runningTasks.size()) < spaceForTasks; int displayCount = allTasksShown ? runningTasks.size() : spaceForTasks - 1; runningTasksLineCount = allTasksShown ? runningTasks.size() : spaceForTasks; for (int i = 0; i < displayCount; i++) { TaskImpl* task = runningTasks[i]; int spaceForNoun = windowSize.ws_col - task->verb.size() - 2; fprintf(out, "%s↺ %s:%s ", ANSI_COLOR_CODES[RUNNING_COLOR], task->verb.c_str(), ANSI_CLEAR_COLOR); if (static_cast(task->noun.size()) > spaceForNoun) { std::string shortenedNoun = task->noun.substr(task->noun.size() - spaceForNoun + 3); fprintf(out, "...%s", shortenedNoun.c_str()); } else { fputs(task->noun.c_str(), out); } putc('\n', out); } if (!allTasksShown) { fputs("...(more)...\n", out); } fflush(out); lastDebugMessageCount = DebugMessage::getMessageCount(); } } // namespace ekam ================================================ FILE: src/ekam/ConsoleDashboard.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_CONSOLEDASHBOARD_H_ #define KENTONSCODE_EKAM_CONSOLEDASHBOARD_H_ #include #include #include "Dashboard.h" namespace ekam { class ConsoleDashboard : public Dashboard { public: ConsoleDashboard(FILE* output, int maxDisplayedLogLines); ~ConsoleDashboard(); // implements Dashboard ---------------------------------------------------------------- OwnedPtr beginTask(const std::string& verb, const std::string& noun, Silence silence); private: class TaskImpl; class LogFormatter; int fd; FILE* out; int maxDisplayedLogLines; std::vector runningTasks; int runningTasksLineCount; int lastDebugMessageCount; enum Color { BLACK, RED, GREEN, YELLOW, BLUE, FUCHSIA, CYAN, WHITE, GRAY, BRIGHT_RED, BRIGHT_GREEN, BRIGHT_YELLOW, BRIGHT_BLUE, BRIGHT_FUCHSIA, BRIGHT_CYAN, BRIGHT_WHITE, }; static const char* const ANSI_COLOR_CODES[]; static const char* const ANSI_CLEAR_COLOR; static const char* const ANSI_MOVE_CURSOR_UP; static const char* const ANSI_CLEAR_BELOW_CURSOR; static const Color DONE_COLOR; static const Color PASSED_COLOR; static const Color FAILED_COLOR; static const Color RUNNING_COLOR; void clearRunning(); void drawRunning(); }; } // namespace ekam #endif // KENTONSCODE_EKAM_CONSOLEDASHBOARD_H_ ================================================ FILE: src/ekam/CppActionFactory.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "CppActionFactory.h" #include #include #include "base/Debug.h" #include "os/ByteStream.h" #include "os/Subprocess.h" #include "ActionUtil.h" namespace ekam { // compile action: produces object file, tags for all symbols declared therein. // (Compile action is now implemented in compile.ekam-rule.) // link action: triggers on "main" tag. namespace { OwnedPtr getDepsFile(File* objectFile) { return objectFile->parent()->relative(objectFile->basename() + ".deps"); } bool isTestName(const std::string& name) { std::string::size_type pos = name.find_last_of("_-"); if (pos == std::string::npos) { return false; } else { std::string suffix = name.substr(pos + 1); return suffix == "test" || suffix == "unittest" || suffix == "regtest"; } } } // namespace class LinkAction : public Action { public: enum Mode { NORMAL, GTEST, KJTEST, NODEJS, LIBFUZZER }; LinkAction(File* file, Mode mode); ~LinkAction(); // implements Action ------------------------------------------------------------------- std::string getVerb(); Promise start(EventManager* eventManager, BuildContext* context); private: class DepsSet { public: DepsSet() {} ~DepsSet() {} void addObject(BuildContext* context, File* objectFile); void enumerate(OwnedPtrVector::Appender output) { deps.releaseAll(output); } private: OwnedPtrMap deps; }; static const Tag GTEST_MAIN; static const Tag KJTEST_MAIN; static const Tag TEST_EXECUTABLE; OwnedPtr file; Mode mode; Promise startTarget(EventManager* eventManager, BuildContext* context, const std::string& base, OwnedPtrVector& flatDeps, const std::string& target); }; const Tag LinkAction::GTEST_MAIN = Tag::fromName("gtest:main"); const Tag LinkAction::KJTEST_MAIN = Tag::fromName("kjtest:main"); const Tag LinkAction::TEST_EXECUTABLE = Tag::fromName("test:executable"); LinkAction::LinkAction(File* file, Mode mode) : file(file->clone()), mode(mode) {} LinkAction::~LinkAction() {} std::string LinkAction::getVerb() { return "link"; } void LinkAction::DepsSet::addObject(BuildContext* context, File* objectFile) { if (deps.contains(objectFile)) { return; } OwnedPtr ptr = objectFile->clone(); File* rawptr = ptr.get(); // cannot inline due to undefined evaluation order deps.add(rawptr, ptr.release()); OwnedPtr depsFile = getDepsFile(objectFile); if (depsFile->exists()) { std::string data = depsFile->readAll(); std::string::size_type prevPos = 0; std::string::size_type pos = data.find_first_of('\n'); while (pos != std::string::npos) { std::string symbolName(data, prevPos, pos - prevPos); File* file = context->findProvider(Tag::fromName("c++symbol:" + symbolName)); if (file != NULL) { addObject(context, file); } prevPos = pos + 1; pos = data.find_first_of('\n', prevPos); } } } // --------------------------------------------------------------------------------------- Promise LinkAction::start(EventManager* eventManager, BuildContext* context) { DepsSet deps; if (mode == GTEST) { File* gtestMain = context->findProvider(GTEST_MAIN); if (gtestMain == NULL) { context->log("Cannot find gtest_main.o."); context->failed(); return newFulfilledPromise(); } deps.addObject(context, gtestMain); } else if (mode == KJTEST) { File* kjtestMain = context->findProvider(KJTEST_MAIN); if (kjtestMain == NULL) { context->log("Cannot find kj/test.o."); context->failed(); return newFulfilledPromise(); } deps.addObject(context, kjtestMain); } deps.addObject(context, file.get()); OwnedPtrVector flatDeps; deps.enumerate(flatDeps.appender()); std::string base, ext; splitExtension(file->canonicalName(), &base, &ext); if (mode == NODEJS) { base += ".node"; } auto promise = startTarget(eventManager, context, base, flatDeps, ""); const char* targets = getenv("CROSS_TARGETS"); if (targets != NULL) { auto addTarget = [&](std::string target) { OwnedPtrVector targetDeps; for (int i = 0; i < flatDeps.size(); i++) { std::string name, ext; splitExtension(flatDeps.get(i)->basename(), &name, &ext); targetDeps.add(flatDeps.get(i)->parent()->relative(name + '.' + target + ext)); } promise = eventManager->when(std::move(promise))( [this, target = std::move(target), targetDeps = std::move(targetDeps), eventManager, context, base](Void) mutable { return startTarget(eventManager, context, base, targetDeps, target); }); }; while (const char* spacepos = strchr(targets, ' ')) { addTarget(std::string(targets, spacepos)); targets = spacepos + 1; } addTarget(targets); } return promise; } Promise LinkAction::startTarget( EventManager* eventManager, BuildContext* context, const std::string& base, OwnedPtrVector& flatDeps, const std::string& target) { const char* cxx = getenv("CXX"); auto subprocess = newOwned(); std::string compiler = cxx == NULL ? "c++" : cxx; auto slashpos = compiler.find_last_of('/'); std::string compilerName = slashpos < 0 ? compiler : compiler.substr(slashpos + 1); if (target.empty()) { subprocess->addArgument(std::move(compiler)); } else if (strstr(compilerName.c_str(), "clang")) { subprocess->addArgument(std::move(compiler)); subprocess->addArgument("-target"); subprocess->addArgument(target); } else { subprocess->addArgument(target + '-' + compiler); } if (mode == NODEJS) { subprocess->addArgument("-shared"); } else if (context->findProvider(Tag::fromName("canonical:" + base + ".link-static")) != nullptr) { subprocess->addArgument("-static"); } if (mode == LIBFUZZER) { const auto arg = getenv("EKAM_LIBFUZZER_LINKER_ARG"); subprocess->addArgument(arg == nullptr ? "-fsanitize=fuzzer" : arg); } subprocess->addArgument("-o"); auto executableFile = context->newOutput(target.empty() ? base : (base + "." + target)); subprocess->addArgument(executableFile.get(), File::WRITE); if (isTestName(base)) { std::vector tags; tags.push_back(TEST_EXECUTABLE); context->provide(executableFile.get(), tags); } for (int i = 0; i < flatDeps.size(); i++) { subprocess->addArgument(flatDeps.get(i), File::READ); } const char* libs = nullptr; if (!target.empty()) { // Look for arch-specific libs. std::string targetLibsName = "LIBS_" + target; for (char& c: targetLibsName) { if (c == '-') c = '_'; } libs = getenv(targetLibsName.c_str()); } if (libs == nullptr) { libs = getenv("LIBS"); } if (libs != NULL) { while (const char* spacepos = strchr(libs, ' ')) { subprocess->addArgument(std::string(libs, spacepos)); libs = spacepos + 1; } subprocess->addArgument(libs); } auto logStream = subprocess->captureStdoutAndStderr(); auto subprocessWaitOp = eventManager->when(subprocess->start(eventManager))( [context](ProcessExitCode exitCode) { if (exitCode.wasSignaled() || exitCode.getExitCode() != 0) { context->failed(); } }); auto logger = newOwned(context, logStream.release()); auto logOp = logger->run(eventManager); return eventManager->when(subprocessWaitOp, logOp, logger, subprocess, executableFile)( [](Void, Void, OwnedPtr, OwnedPtr, OwnedPtr){}); } // ======================================================================================= const Tag CppActionFactory::MAIN_SYMBOLS[] = { Tag::fromName("c++symbol:main"), Tag::fromName("c++symbol:_main") }; const Tag CppActionFactory::GTEST_TEST = Tag::fromName("gtest:test"); const Tag CppActionFactory::KJTEST_TEST = Tag::fromName("kjtest:test"); const Tag CppActionFactory::NODEJS_MODULE = Tag::fromName("nodejs:module"); const Tag CppActionFactory::LIBFUZZER_SYMBOL = Tag::fromName("c++symbol:LLVMFuzzerTestOneInput"); CppActionFactory::CppActionFactory() {} CppActionFactory::~CppActionFactory() {} void CppActionFactory::enumerateTriggerTags( std::back_insert_iterator > iter) { for (unsigned int i = 0; i < (sizeof(MAIN_SYMBOLS) / sizeof(MAIN_SYMBOLS[0])); i++) { *iter++ = MAIN_SYMBOLS[i]; } *iter++ = GTEST_TEST; *iter++ = KJTEST_TEST; *iter++ = NODEJS_MODULE; if (getenv("EKAM_LIBFUZZER_ENABLE") != nullptr) { *iter++ = LIBFUZZER_SYMBOL; } } OwnedPtr CppActionFactory::tryMakeAction(const Tag& id, File* file) { OwnedPtr result; for (unsigned int i = 0; i < (sizeof(MAIN_SYMBOLS) / sizeof(MAIN_SYMBOLS[0])); i++) { if (id == MAIN_SYMBOLS[i]) { return newOwned(file, LinkAction::NORMAL); } } if (id == LIBFUZZER_SYMBOL) { return newOwned(file, LinkAction::LIBFUZZER); } if (id == GTEST_TEST) { return newOwned(file, LinkAction::GTEST); } if (id == KJTEST_TEST) { return newOwned(file, LinkAction::KJTEST); } if (id == NODEJS_MODULE) { return newOwned(file, LinkAction::NODEJS); } return nullptr; } } // namespace ekam ================================================ FILE: src/ekam/CppActionFactory.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_CPPACTIONFACTORY_H_ #define KENTONSCODE_EKAM_CPPACTIONFACTORY_H_ #include #include #include "Action.h" namespace ekam { class CppActionFactory: public ActionFactory { public: CppActionFactory(); ~CppActionFactory(); // implements ActionFactory ------------------------------------------------------------ void enumerateTriggerTags(std::back_insert_iterator > iter); OwnedPtr tryMakeAction(const Tag& id, File* file); private: static const Tag MAIN_SYMBOLS[]; static const Tag LIBFUZZER_SYMBOL; static const Tag GTEST_TEST; static const Tag KJTEST_TEST; static const Tag NODEJS_MODULE; }; } // namespace ekam #endif // KENTONSCODE_EKAM_CPPACTIONFACTORY_H_ ================================================ FILE: src/ekam/Dashboard.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Dashboard.h" namespace ekam { Dashboard::~Dashboard() {} Dashboard::Task::~Task() {} } ================================================ FILE: src/ekam/Dashboard.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_DASHBOARD_H_ #define KENTONSCODE_EKAM_DASHBOARD_H_ #include #include "base/OwnedPtr.h" namespace ekam { class Dashboard { public: virtual ~Dashboard(); enum TaskState { PENDING, // Default state. RUNNING, DONE, PASSED, // Like DONE, but should be displayed prominently (hint: test result). FAILED, BLOCKED }; class Task { public: virtual ~Task(); virtual void setState(TaskState state) = 0; virtual void addOutput(const std::string& text) = 0; }; enum Silence { SILENT, NORMAL }; virtual OwnedPtr beginTask(const std::string& verb, const std::string& noun, Silence silence) = 0; }; class EventManager; OwnedPtr initNetworkDashboard(EventManager* eventManager, const std::string& address, OwnedPtr dashboardToWrap); } // namespace ekam #endif // KENTONSCODE_EKAM_DASHBOARD_H_ ================================================ FILE: src/ekam/Driver.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Driver.h" #include #include #include #include #include #include #include "base/Debug.h" #include "os/EventGroup.h" namespace ekam { namespace { int fileDepth(const std::string& name) { int result = 0; for (unsigned int i = 0; i < name.size(); i++) { if (name[i] == '/') { ++result; } } return result; } int commonPrefixLength(const std::string& srcName, const std::string& bestMatchName) { std::string::size_type n = std::min(srcName.size(), bestMatchName.size()); for (unsigned int i = 0; i < n; i++) { if (srcName[i] != bestMatchName[i]) { return i; } } return n; } } // namespace class Driver::ActionDriver : public BuildContext, public EventGroup::ExceptionHandler { public: ActionDriver(Driver* driver, OwnedPtr action, File* srcfile, Hash srcHash, OwnedPtr task); ~ActionDriver(); void start(); // implements BuildContext ------------------------------------------------------------- File* findProvider(Tag id); File* findInput(const std::string& path); void provide(File* file, const std::vector& tags); void install(File* file, InstallLocation location, const std::string& name); void log(const std::string& text); OwnedPtr newOutput(const std::string& path); void addActionType(OwnedPtr factory); void passed(); void failed(); // implements ExceptionHandler --------------------------------------------------------- void threwException(const std::exception& e); void threwUnknownException(); void noMoreEvents(); private: Driver* driver; OwnedPtr action; OwnedPtr srcfile; Hash srcHash; OwnedPtr dashboardTask; // TODO: Get rid of "state". Maybe replace with "status" or something, but don't try to // track both whether we're running and what the status was at the same time. (I already // had to split isRunning into a separate boolean due to issues with this.) enum { PENDING, RUNNING, DONE, PASSED, FAILED } state; EventGroup eventGroup; Promise asyncCallbackOp; bool isRunning; Promise runningAction; OwnedPtrVector outputs; struct Installation { File* file; InstallLocation location; std::string name; }; std::vector installations; OwnedPtrVector provisions; OwnedPtrVector > providedTags; OwnedPtrVector providedFactories; // True if returned() is currently on the stack. Causes destructor to abort. Used for // debugging. bool currentlyExecutingReturned = false; void ensureRunning(); void queueDoneCallback(); void returned(); void reset(); Provision* choosePreferredProvider(const Tag& tag); File* provideInternal(File* file, const std::vector& tags); friend class Driver; }; Driver::ActionDriver::ActionDriver(Driver* driver, OwnedPtr action, File* srcfile, Hash srcHash, OwnedPtr task) : driver(driver), action(action.release()), srcfile(srcfile->clone()), srcHash(srcHash), dashboardTask(task.release()), state(PENDING), eventGroup(driver->eventManager, this), isRunning(false) {} Driver::ActionDriver::~ActionDriver() { assert(!currentlyExecutingReturned); } void Driver::ActionDriver::start() { assert(state == PENDING); assert(!driver->dependencyTable.has(this)); assert(outputs.empty()); assert(provisions.empty()); assert(installations.empty()); assert(providedFactories.empty()); assert(!isRunning); state = RUNNING; isRunning = true; dashboardTask->setState(Dashboard::RUNNING); asyncCallbackOp = eventGroup.when()( [this]() { asyncCallbackOp.release(); runningAction = action->start(&eventGroup, this); }); } File* Driver::ActionDriver::findProvider(Tag tag) { ensureRunning(); Provision* provision = choosePreferredProvider(tag); driver->dependencyTable.add(tag, this, provision); if (provision == NULL) { return NULL; } else { return provision->file.get(); } } File* Driver::ActionDriver::findInput(const std::string& path) { ensureRunning(); return findProvider(Tag::fromFile(path)); } void Driver::ActionDriver::provide(File* file, const std::vector& tags) { provideInternal(file, tags); } File* Driver::ActionDriver::provideInternal(File* file, const std::vector& tags) { ensureRunning(); // Find existing provision for this file, if any. // TODO: Convert provisions into a map? Provision* provision = NULL; for (int i = 0; i < provisions.size(); i++) { if (provisions.get(i)->file->equals(file)) { provision = provisions.get(i); providedTags.get(i)->insert(providedTags.get(i)->end(), tags.begin(), tags.end()); break; } } if (provision == NULL) { auto ownedProvision = newOwned(); provision = ownedProvision.get(); provision->creator = this; provisions.add(ownedProvision.release()); providedTags.add(newOwned>(tags)); } provision->file = file->clone(); return provision->file.get(); } void Driver::ActionDriver::install(File* file, InstallLocation location, const std::string& name) { ensureRunning(); std::string tagName(INSTALL_LOCATION_NAMES[location]); tagName.push_back(':'); tagName.append(name); std::vector tags; tags.push_back(Tag::fromName(tagName)); File* ownedFile = provideInternal(file, tags); Installation installation = { ownedFile, location, name }; installations.push_back(installation); } void Driver::ActionDriver::log(const std::string& text) { ensureRunning(); dashboardTask->addOutput(text); } OwnedPtr Driver::ActionDriver::newOutput(const std::string& path) { ensureRunning(); OwnedPtr file = driver->tmp->relative(path); recursivelyCreateDirectory(file->parent().get()); OwnedPtr result = file->clone(); std::vector tags; tags.push_back(Tag::DEFAULT_TAG); provide(file.get(), tags); outputs.add(file.release()); return result; } void Driver::ActionDriver::addActionType(OwnedPtr factory) { ensureRunning(); providedFactories.add(factory.release()); } void Driver::ActionDriver::noMoreEvents() { if (isRunning) { if (state == RUNNING) { state = DONE; queueDoneCallback(); } } } void Driver::ActionDriver::passed() { ensureRunning(); if (state == FAILED) { // Ignore passed() after failed(). return; } state = PASSED; queueDoneCallback(); } void Driver::ActionDriver::failed() { ensureRunning(); if (state == FAILED) { // Ignore redundant call to failed(). return; } else if (state == DONE) { // (done callback should already be queued) throw std::runtime_error("Called failed() after success()."); } else if (state == PASSED) { // (done callback should already be queued) throw std::runtime_error("Called failed() after passed()."); } else { state = FAILED; queueDoneCallback(); } } void Driver::ActionDriver::ensureRunning() { if (!isRunning) { throw std::runtime_error("Action is not running."); } } void Driver::ActionDriver::queueDoneCallback() { asyncCallbackOp = driver->eventManager->when()( [this]() { asyncCallbackOp.release(); Driver* driver = this->driver; returned(); // may delete this driver->startSomeActions(); }); } void Driver::ActionDriver::threwException(const std::exception& e) { ensureRunning(); dashboardTask->addOutput(std::string("uncaught exception: ") + e.what() + "\n"); asyncCallbackOp.release(); state = FAILED; returned(); } void Driver::ActionDriver::threwUnknownException() { ensureRunning(); dashboardTask->addOutput("uncaught exception of unknown type\n"); asyncCallbackOp.release(); state = FAILED; returned(); } // Poor man's kj::defer. template class Deferred { public: inline Deferred(Func&& func): func(func), canceled(false) {} inline ~Deferred() noexcept(false) { if (!canceled) func(); } // This move constructor is usually optimized away by the compiler. inline Deferred(Deferred&& other): func(other.func), canceled(false) { other.canceled = true; } private: Func func; bool canceled; }; template Deferred defer(Func&& func) { // Returns an object which will invoke the given functor in its destructor. The object is not // copyable but is movable with the semantics you'd expect. Since the return type is private, // you need to assign to an `auto` variable. return Deferred(std::forward(func)); } void Driver::ActionDriver::returned() { ensureRunning(); currentlyExecutingReturned = true; auto _ = defer([this]() { currentlyExecutingReturned = false; }); // Cancel anything still running. runningAction.release(); isRunning = false; // Pull self out of driver->activeActions. OwnedPtr self; for (int i = 0; i < driver->activeActions.size(); i++) { if (driver->activeActions.get(i) == this) { self = driver->activeActions.releaseAndShift(i); break; } } driver->completedActionPtrs.add(this, self.release()); if (state == FAILED) { // Failed, possibly due to missing dependencies. provisions.clear(); installations.clear(); providedTags.clear(); providedFactories.clear(); outputs.clear(); dashboardTask->setState(Dashboard::BLOCKED); } else { dashboardTask->setState(state == PASSED ? Dashboard::PASSED : Dashboard::DONE); // Remove outputs which were deleted before the action completed. Some actions create // files and then delete them immediately. OwnedPtrVector provisionsToFilter; provisions.swap(&provisionsToFilter); for (int i = 0; i < provisionsToFilter.size(); i++) { if (provisionsToFilter.get(i)->file->exists()) { provisions.add(provisionsToFilter.release(i)); } } // Register providers. But, don't allow our own dependencies to depend on them. std::unordered_set deps; driver->getTransitiveDependencies(this, &deps); for (int i = 0; i < provisions.size(); i++) { driver->registerProvider(provisions.get(i), *providedTags.get(i), deps); } providedTags.clear(); // Not needed anymore. // Register factories. for (int i = 0; i < providedFactories.size(); i++) { driver->addActionFactory(providedFactories.get(i)); driver->rescanForNewFactory(providedFactories.get(i)); } // Install files. for (size_t i = 0; i < installations.size(); i++) { File* installDir = driver->installDirs[installations[i].location]; OwnedPtr target = installDir->relative(installations[i].name); if (target->exists()) { target->unlink(); } else { if (!installDir->isDirectory()) { // Can't rely on recursivelyCreateDirectory() for the root directory because it will // call parent() on it which will throw. installDir->createDirectory(); } recursivelyCreateDirectory(target->parent().get()); } target->link(installations[i].file); } } } void Driver::ActionDriver::reset() { assert(!currentlyExecutingReturned); if (state == PENDING) { // Nothing to do. return; } OwnedPtr self; if (isRunning) { dashboardTask->setState(Dashboard::BLOCKED); runningAction.release(); asyncCallbackOp.release(); for (int i = 0; i < driver->activeActions.size(); i++) { if (driver->activeActions.get(i) == this) { self = driver->activeActions.releaseAndShift(i); break; } } isRunning = false; } else { if (!driver->completedActionPtrs.release(this, &self)) { throw std::logic_error("Action not running or pending, but not in completedActionPtrs?"); } } state = PENDING; // Put on back of queue (as opposed to front) so that actions which are frequently reset // don't get redundantly rebuilt too much. We add the action to the queue before resetting // dependents so that this action gets re-run before its dependents. // TODO: The second point probably doesn't help much when multiprocessing. Maybe the // action queue should really be a graph that remembers what depended on what the last // time we ran them, and avoids re-running any action before re-running actions on which it // depended last time. driver->pendingActions.pushBack(self.release()); // Reset dependents. for (int i = 0; i < provisions.size(); i++) { driver->resetDependentActions(provisions.get(i)); } // Actions created by any provided ActionFactories must be deleted. for (int i = 0; i < providedFactories.size(); i++) { ActionFactory* factory = providedFactories.get(i); std::vector actionsToDelete; for (ActionTriggersTable::SearchIterator iter(driver->actionTriggersTable, factory); iter.next();) { // Can't call reset() directly here because it may invalidate our iterator. actionsToDelete.push_back(iter.cell()); } for (size_t j = 0; j < actionsToDelete.size(); j++) { actionsToDelete[j]->reset(); // TODO: Use better data structure for pendingActions. For now we have to iterate // through the whole thing to find the action we're deleting. We iterate from the back // since it's likely the action was just added there. for (int k = driver->pendingActions.size() - 1; k >= 0; k--) { if (driver->pendingActions.get(k) == actionsToDelete[j]) { driver->pendingActions.releaseAndShift(k); break; } } } driver->actionTriggersTable.erase(factory); driver->triggers.erase(factory); } // Remove all entries in dependencyTable pointing at this action. driver->dependencyTable.erase(this); provisions.clear(); installations.clear(); providedTags.clear(); providedFactories.clear(); outputs.clear(); } Driver::Provision* Driver::ActionDriver::choosePreferredProvider(const Tag& tag) { TagTable::SearchIterator iter(driver->tagTable, tag); if (!iter.next()) { return NULL; } else { std::string srcName = srcfile->canonicalName(); Provision* bestMatch = iter.cell(); if (iter.next()) { // There are multiple files with this tag. We must choose which one we like best. std::string bestMatchName = bestMatch->file->canonicalName(); int bestMatchDepth = fileDepth(bestMatchName); int bestMatchCommonPrefix = commonPrefixLength(srcName, bestMatchName); do { Provision* candidate = iter.cell(); std::string candidateName = candidate->file->canonicalName(); int candidateDepth = fileDepth(candidateName); int candidateCommonPrefix = commonPrefixLength(srcName, candidateName); if (candidateCommonPrefix < bestMatchCommonPrefix) { // Prefer provider that is closer in the directory tree. continue; } else if (candidateCommonPrefix == bestMatchCommonPrefix) { if (candidateDepth > bestMatchDepth) { // Prefer provider that is less deeply nested. continue; } else if (candidateDepth == bestMatchDepth) { // Arbitrarily -- but consistently -- choose one. int diff = bestMatchName.compare(candidateName); if (diff < 0) { // Prefer file that comes first alphabetically. continue; } else if (diff == 0) { // TODO: Is this really an error? I think it is for the moment, but someday it // may not be, if multiple actions are allowed to produce outputs with the same // canonical names. DEBUG_ERROR << "Two providers have same file name: " << bestMatchName; continue; } } } // If we get here, the candidate is better than the existing best match. bestMatch = candidate; bestMatchName.swap(candidateName); bestMatchDepth = candidateDepth; bestMatchCommonPrefix = candidateCommonPrefix; } while(iter.next()); } return bestMatch; } } // ======================================================================================= Driver::Driver(EventManager* eventManager, Dashboard* dashboard, File* tmp, File* installDirs[BuildContext::INSTALL_LOCATION_COUNT], int maxConcurrentActions, ActivityObserver* activityObserver) : eventManager(eventManager), dashboard(dashboard), tmp(tmp), maxConcurrentActions(maxConcurrentActions), activityObserver(activityObserver) { if (!tmp->isDirectory()) { tmp->createDirectory(); } for (int i = 0; i < BuildContext::INSTALL_LOCATION_COUNT; i++) { this->installDirs[i] = installDirs[i]; } } Driver::~Driver() {} void Driver::addActionFactory(ActionFactory* factory) { std::vector triggerTags; factory->enumerateTriggerTags(std::back_inserter(triggerTags)); for (unsigned int i = 0; i < triggerTags.size(); i++) { triggers.add(triggerTags[i], factory); } } void Driver::addSourceFile(File* file) { OwnedPtr provision; if (rootProvisions.release(file, &provision)) { // Source file was modified. Reset all actions dependent on the old version. resetDependentActions(provision.get()); } // Apply default tag. std::vector tags; tags.push_back(Tag::DEFAULT_TAG); provision = newOwned(); provision->creator = nullptr; provision->file = file->clone(); registerProvider(provision.get(), tags, std::unordered_set()); File* key = provision->file.get(); // cannot inline due to undefined evaluation order rootProvisions.add(key, provision.release()); startSomeActions(); } void Driver::removeSourceFile(File* file) { OwnedPtr provision; if (rootProvisions.release(file, &provision)) { resetDependentActions(provision.get()); // In case some active actions were canceled. startSomeActions(); } else { DEBUG_ERROR << "Tried to remove source file that wasn't ever added: " << file->canonicalName(); } } void Driver::startSomeActions() { while (activeActions.size() < maxConcurrentActions && !pendingActions.empty()) { if (activityObserver != nullptr) activityObserver->startingAction(); OwnedPtr actionDriver = pendingActions.popFront(); ActionDriver* ptr = actionDriver.get(); activeActions.add(actionDriver.release()); try { ptr->start(); } catch (const std::exception& e) { ptr->threwException(e); } catch (...) { ptr->threwUnknownException(); } } if (activeActions.size() == 0) { bool hasFailures = dumpErrors(); if (activityObserver != nullptr) activityObserver->idle(hasFailures); } } void Driver::rescanForNewFactory(ActionFactory* factory) { // Apply triggers. std::vector triggerTags; factory->enumerateTriggerTags(std::back_inserter(triggerTags)); for (unsigned int i = 0; i < triggerTags.size(); i++) { for (TagTable::SearchIterator iter(tagTable, triggerTags[i]); iter.next();) { Provision* provision = iter.cell(); OwnedPtr action = factory->tryMakeAction(triggerTags[i], provision->file.get()); if (action != NULL) { queueNewAction(factory, action.release(), provision); } } } } void Driver::queueNewAction(ActionFactory* factory, OwnedPtr action, Provision* provision) { OwnedPtr task = dashboard->beginTask( action->getVerb(), provision->file->canonicalName(), action->isSilent() ? Dashboard::SILENT : Dashboard::NORMAL); OwnedPtr actionDriver = newOwned(this, action.release(), provision->file.get(), provision->contentHash, task.release()); actionTriggersTable.add(factory, provision, actionDriver.get()); // Put new action on front of queue because it was probably triggered by another action that // just completed, and it's good to run related actions together to improve cache locality. pendingActions.pushFront(actionDriver.release()); } void Driver::getTransitiveDependencies( ActionDriver* action, std::unordered_set* deps) { if (action != nullptr && deps->insert(action).second) { for (ActionTriggersTable::SearchIterator iter(actionTriggersTable, action); iter.next();) { getTransitiveDependencies(iter.cell()->creator, deps); } for (DependencyTable::SearchIterator iter(dependencyTable, action); iter.next();) { Provision* provision = iter.cell(); if (provision != nullptr) { getTransitiveDependencies(provision->creator, deps); } } } } void Driver::registerProvider(Provision* provision, const std::vector& tags, const std::unordered_set& dependencies) { provision->contentHash = provision->file->contentHash(); for (std::vector::const_iterator iter = tags.begin(); iter != tags.end(); ++iter) { const Tag& tag = *iter; tagTable.add(tag, provision); resetDependentActions(tag, dependencies); fireTriggers(tag, provision); } } void Driver::resetDependentActions(const Tag& tag, const std::unordered_set& dependencies) { std::unordered_set provisionsToReset; std::vector actionsToReset; for (DependencyTable::SearchIterator iter(dependencyTable, tag); iter.next();) { ActionDriver* action = iter.cell(); // Don't reset an action that contributed to the creation of this tag in the first place, since // that would lead to an infinite loop of rebuilding the same action. if (dependencies.count(action) == 0) { Provision* previousProvider = iter.cell(); if (action->choosePreferredProvider(tag) != previousProvider) { // We can't just call reset() here because it could invalidate our iterator. actionsToReset.push_back(action); } } else { DEBUG_INFO << "Action's inputs are affected by its outputs."; } } for (size_t i = 0; i < actionsToReset.size(); i++) { // Only reset the action if it is still in the dependency table. If not, it was already // reset (and possibly deleted!) elsewhere. if (dependencyTable.find(actionsToReset[i]) != nullptr) { actionsToReset[i]->reset(); } } } void Driver::resetDependentActions(Provision* provision) { // Reset dependents of this provision. { std::vector actionsToReset; for (DependencyTable::SearchIterator iter(dependencyTable, provision); iter.next();) { // Can't call reset() directly here because it may invalidate our iterator. actionsToReset.push_back(iter.cell()); } for (size_t j = 0; j < actionsToReset.size(); j++) { // Only reset the action if it is still in the dependency table. If not, it was already // reset (and possibly deleted!) elsewhere. if (dependencyTable.find(actionsToReset[j]) != nullptr) { actionsToReset[j]->reset(); } } if (dependencyTable.erase(provision) > 0) { DEBUG_ERROR << "Resetting dependents should have removed this provision from " "dependencyTable."; } } // Everything triggered by this provision must be deleted. { std::vector actionsToDelete; for (ActionTriggersTable::SearchIterator iter(actionTriggersTable, provision); iter.next();) { // Can't call reset() directly here because it may invalidate our iterator. actionsToDelete.push_back(iter.cell()); } for (size_t j = 0; j < actionsToDelete.size(); j++) { actionsToDelete[j]->reset(); // TODO: Use better data structure for pendingActions. For now we have to iterate // through the whole thing to find the action we're deleting. We iterate from the back // since it's likely the action was just added there. for (int k = pendingActions.size() - 1; k >= 0; k--) { if (pendingActions.get(k) == actionsToDelete[j]) { pendingActions.releaseAndShift(k); break; } } } actionTriggersTable.erase(provision); } tagTable.erase(provision); } void Driver::fireTriggers(const Tag& tag, Provision* provision) { for (TriggerTable::SearchIterator iter(triggers, tag); iter.next();) { ActionFactory* factory = iter.cell(); OwnedPtr triggeredAction = factory->tryMakeAction(tag, provision->file.get()); if (triggeredAction != NULL) { queueNewAction(factory, triggeredAction.release(), provision); } } } bool Driver::dumpErrors() { bool hasFailures = false; for (OwnedPtrMap::Iterator iter(completedActionPtrs); iter.next();) { if (iter.key()->state == ActionDriver::FAILED) { iter.value()->dashboardTask->setState(Dashboard::FAILED); hasFailures = true; } } return hasFailures; } } // namespace ekam ================================================ FILE: src/ekam/Driver.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_DRIVER_H_ #define KENTONSCODE_EKAM_DRIVER_H_ #include #include #include #include #include "base/OwnedPtr.h" #include "os/File.h" #include "Action.h" #include "Tag.h" #include "Dashboard.h" #include "base/Table.h" namespace ekam { class Driver { public: class ActivityObserver { public: virtual void startingAction() = 0; virtual void idle(bool hasFailures) = 0; }; Driver(EventManager* eventManager, Dashboard* dashboard, File* tmp, File* installDirs[BuildContext::INSTALL_LOCATION_COUNT], int maxConcurrentActions, ActivityObserver* activityObserver = nullptr); ~Driver(); void addActionFactory(ActionFactory* factory); void addSourceFile(File* file); void removeSourceFile(File* file); private: class ActionDriver; EventManager* eventManager; Dashboard* dashboard; File* tmp; File* installDirs[BuildContext::INSTALL_LOCATION_COUNT]; int maxConcurrentActions; ActivityObserver* activityObserver; class TriggerTable : public Table, IndexedColumn > { public: static const int TAG = 0; static const int FACTORY = 1; }; TriggerTable triggers; struct Provision { ActionDriver* creator; // possibly null OwnedPtr file; Hash contentHash; }; class TagTable : public Table, IndexedColumn > { public: static const int TAG = 0; static const int PROVISION = 1; }; TagTable tagTable; OwnedPtrVector activeActions; OwnedPtrDeque pendingActions; OwnedPtrMap completedActionPtrs; class DependencyTable : public Table, IndexedColumn, IndexedColumn > { public: static const int TAG = 0; static const int ACTION = 1; static const int PROVISION = 2; }; DependencyTable dependencyTable; class ActionTriggersTable : public Table, IndexedColumn, IndexedColumn > { public: static const int FACTORY = 0; static const int PROVISION = 1; static const int ACTION = 2; }; ActionTriggersTable actionTriggersTable; OwnedPtrMap rootProvisions; void startSomeActions(); void rescanForNewFactory(ActionFactory* factory); void queueNewAction(ActionFactory* factory, OwnedPtr action, Provision* provision); void getTransitiveDependencies(ActionDriver* action, std::unordered_set* deps); void registerProvider(Provision* provision, const std::vector& tags, const std::unordered_set& dependencies); void resetDependentActions(const Tag& tag, const std::unordered_set& dependencies); void resetDependentActions(Provision* provision); void fireTriggers(const Tag& tag, Provision* provision); bool dumpErrors(); }; } // namespace ekam #endif // KENTONSCODE_EKAM_DRIVER_H_ ================================================ FILE: src/ekam/ExecPluginActionFactory.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "ExecPluginActionFactory.h" #include #include #include #include "os/Subprocess.h" #include "ActionUtil.h" #include "base/Debug.h" namespace ekam { namespace { std::string splitToken(std::string* line) { std::string::size_type pos = line->find_first_of(' '); std::string result; if (pos == std::string::npos) { result = *line; line->clear(); } else { result.assign(*line, 0, pos); line->erase(0, pos + 1); } return result; } } // namespace // ======================================================================================= class PluginDerivedActionFactory : public ActionFactory { public: PluginDerivedActionFactory(OwnedPtr executable, std::string&& verb, bool silent, std::vector&& triggers); ~PluginDerivedActionFactory(); // implements ActionFactory ----------------------------------------------------------- void enumerateTriggerTags(std::back_insert_iterator > iter); OwnedPtr tryMakeAction(const Tag& id, File* file); private: OwnedPtr executable; std::string verb; bool silent; std::vector triggers; }; // ======================================================================================= class PluginDerivedAction : public Action { public: PluginDerivedAction(File* executable, const std::string& verb, bool silent, File* file) : executable(executable->clone()), verb(verb), silent(silent) { if (file != NULL) { this->file = file->clone(); } } ~PluginDerivedAction() {} // implements Action ------------------------------------------------------------------- std::string getVerb() { return verb; } bool isSilent() { return silent; } Promise start(EventManager* eventManager, BuildContext* context); private: class CommandReader; OwnedPtr executable; std::string verb; bool silent; OwnedPtr file; // nullable }; class PluginDerivedAction::CommandReader { public: CommandReader(BuildContext* context, OwnedPtr requestStream, OwnedPtr responseStream, File* executable, File* input) : context(context), executable(executable->clone()), requestStream(requestStream.release()), responseStream(responseStream.release()), lineReader(this->requestStream.get()), silent(false) { if (input != NULL) { this->input = input->clone(); knownFiles.add(input->canonicalName(), input->clone()); } std::string junk; splitExtension(executable->basename(), &verb, &junk); } ~CommandReader() {} Promise readAll(EventManager* eventManager) { return eventManager->when(lineReader.readLine(eventManager))( [=](OwnedPtr line) -> Promise { if (line == nullptr) { eof(); return newFulfilledPromise(); } consume(*line); return readAll(eventManager); }, [=](MaybeException> error) { try { error.get(); } catch (const std::exception& e) { context->log(e.what()); context->failed(); throw; } catch (...) { context->log("unknown exception"); context->failed(); throw; } }); } private: void consume(const std::string& line) { if (findInCache(line)) return; std::string args = line; std::string command = splitToken(&args); if (command == "verb") { verb = args; } else if (command == "silent") { silent = true; } else if (command == "trigger") { triggers.push_back(Tag::fromName(args)); } else if (command == "findProvider" || command == "findInput") { File* provider; if (command == "findProvider") { provider = context->findProvider(Tag::fromName(args)); } else if (input != NULL && args == input->canonicalName()) { provider = input.get(); } else if (findInCache("newOutput " + args)) { // File was originally created by this action. findInCache() already wrote the path, // so just return. return; } else { provider = context->findInput(args); } if (provider != NULL) { OwnedPtr diskRef = provider->getOnDisk(File::READ); std::string path = diskRef->path(); cache.insert(std::make_pair(line, diskRef.get())); diskRefs.add(diskRef.release()); responseStream->writeAll(path.data(), path.size()); knownFiles.add(path, provider->clone()); } responseStream->writeAll("\n", 1); } else if (command == "findModifiers") { auto dir = input->parent(); std::vector results; for (;;) { File* provider = context->findProvider(Tag::fromName( "canonical:" + dir->relative(args)->canonicalName())); if (provider != NULL) { results.push_back(provider); } if (!dir->hasParent()) { break; } dir = dir->parent(); } for (auto iter = results.rbegin(); iter != results.rend(); ++iter) { File* provider = *iter; OwnedPtr diskRef = provider->getOnDisk(File::READ); std::string path = diskRef->path(); diskRefs.add(diskRef.release()); responseStream->writeAll(path.data(), path.size()); knownFiles.add(path, provider->clone()); responseStream->writeAll("\n", 1); } responseStream->writeAll("\n", 1); } else if (command == "newProvider") { // TODO: Create a new output file and register it as a provider. context->log("newProvider not implemented"); context->failed(); } else if (command == "noteInput") { // The action is reading some file outside the working directory. For now we ignore this. // TODO: Pay attention? We could trigger rebuilds when installed tools are updated, etc. } else if (command == "newOutput") { OwnedPtr file = context->newOutput(args); OwnedPtr diskRef = file->getOnDisk(File::WRITE); std::string path = diskRef->path(); cache.insert(std::make_pair(line, diskRef.get())); diskRefs.add(diskRef.release()); knownFiles.add(path, file.release()); responseStream->writeAll(path.data(), path.size()); responseStream->writeAll("\n", 1); } else if (command == "provide") { std::string filename = splitToken(&args); File* file = knownFiles.get(filename); if (file == NULL) { context->log("File passed to \"provide\" not created with \"newOutput\" nor noted as an " "input: " + filename + "\n"); context->failed(); } else { provisions.insert(std::make_pair(file, Tag::fromName(args))); } } else if (command == "install") { std::string filename = splitToken(&args); File* file = knownFiles.get(filename); if (file == NULL) { context->log("File passed to \"install\" not created with \"newOutput\" nor noted as an " "input: " + filename + "\n"); context->failed(); } else { std::string::size_type slashPos = args.find_first_of('/'); if (slashPos == std::string::npos || slashPos == args.size() - 1) { context->log("invalid install location: " + args); context->failed(); } else { std::string targetDir(args, 0, slashPos); std::string name(args, slashPos + 1); bool matched = false; BuildContext::InstallLocation location; for (int i = 0; i < BuildContext::INSTALL_LOCATION_COUNT; i++) { if (targetDir == BuildContext::INSTALL_LOCATION_NAMES[i]) { location = static_cast(i); matched = true; break; } } if (matched) { context->install(file, location, name); } else { context->log("invalid install location: " + args); } } } } else if (command == "passed") { context->passed(); } else { context->log("invalid command: " + command); context->failed(); } } void eof() { // Gather provisions and pass to context. std::vector tags; File* currentFile = NULL; for (ProvisionMap::iterator iter = provisions.begin(); iter != provisions.end(); ++iter) { if (iter->first != currentFile && !tags.empty()) { context->provide(currentFile, tags); tags.clear(); } currentFile = iter->first; tags.push_back(iter->second); } if (!tags.empty()) { context->provide(currentFile, tags); } // Also register new triggers. context->addActionType(newOwned( executable.release(), std::move(verb), silent, std::move(triggers))); } private: BuildContext* context; OwnedPtr executable; OwnedPtr input; // nullable OwnedPtr requestStream; OwnedPtr responseStream; LineReader lineReader; std::string verb; bool silent; std::vector triggers; OwnedPtrMap knownFiles; typedef std::unordered_map CacheMap; CacheMap cache; OwnedPtrVector diskRefs; typedef std::multimap ProvisionMap; ProvisionMap provisions; bool findInCache(const std::string& line) { CacheMap::const_iterator iter = cache.find(line); if (iter == cache.end()) { return false; } else { std::string path = iter->second->path(); responseStream->writeAll(path.data(), path.size()); responseStream->writeAll("\n", 1); return true; } } }; Promise PluginDerivedAction::start(EventManager* eventManager, BuildContext* context) { auto subprocess = newOwned(); subprocess->addArgument(executable.get(), File::READ); if (file != NULL) { subprocess->addArgument(file->canonicalName()); } OwnedPtr responseStream = subprocess->captureStdin(); OwnedPtr commandStream = subprocess->captureStdout(); OwnedPtr logStream = subprocess->captureStderr(); auto subprocessWaitOp = eventManager->when(subprocess->start(eventManager))( [context](ProcessExitCode exitCode) { if (exitCode.wasSignaled() || exitCode.getExitCode() != 0) { context->failed(); } }); auto commandReader = newOwned( context, commandStream.release(), responseStream.release(), executable.get(), file.get()); auto commandOp = commandReader->readAll(eventManager); OwnedPtr logger = newOwned(context, logStream.release()); auto logOp = logger->run(eventManager); return eventManager->when(subprocessWaitOp, commandOp, logOp, subprocess, commandReader, logger)( [](Void, Void, Void, OwnedPtr, OwnedPtr, OwnedPtr){}); } // ======================================================================================= PluginDerivedActionFactory::PluginDerivedActionFactory(OwnedPtr executable, std::string&& verb, bool silent, std::vector&& triggers) : executable(executable.release()), silent(silent) { this->verb.swap(verb); this->triggers.swap(triggers); } PluginDerivedActionFactory::~PluginDerivedActionFactory() {} void PluginDerivedActionFactory::enumerateTriggerTags( std::back_insert_iterator > iter) { for (unsigned int i = 0; i < triggers.size(); i++) { *iter++ = triggers[i]; } } OwnedPtr PluginDerivedActionFactory::tryMakeAction(const Tag& id, File* file) { return newOwned(executable.get(), verb, silent, file); } // ======================================================================================= ExecPluginActionFactory::ExecPluginActionFactory() {} ExecPluginActionFactory::~ExecPluginActionFactory() {} // implements ActionFactory -------------------------------------------------------------- void ExecPluginActionFactory::enumerateTriggerTags( std::back_insert_iterator > iter) { *iter++ = Tag::fromName("filetype:.ekam-rule"); } OwnedPtr ExecPluginActionFactory::tryMakeAction(const Tag& id, File* file) { return newOwned(file, "learn", false, (File*)NULL); } } // namespace ekam ================================================ FILE: src/ekam/ExecPluginActionFactory.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_EXECPLUGINACTIONFACTORY_H_ #define KENTONSCODE_EKAM_EXECPLUGINACTIONFACTORY_H_ #include "Action.h" namespace ekam { class ExecPluginActionFactory : public ActionFactory { public: ExecPluginActionFactory(); ~ExecPluginActionFactory(); // implements ActionFactory ------------------------------------------------------------ void enumerateTriggerTags(std::back_insert_iterator > iter); OwnedPtr tryMakeAction(const Tag& id, File* file); }; } // namespace ekam #endif // KENTONSCODE_EKAM_EXECPLUGINACTIONFACTORY_H_ ================================================ FILE: src/ekam/MuxDashboard.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "MuxDashboard.h" #include #include "base/Debug.h" namespace ekam { class MuxDashboard::TaskImpl : public Dashboard::Task { public: TaskImpl(MuxDashboard* mux, const std::string& verb, const std::string& noun, Silence silence); ~TaskImpl(); void attach(Dashboard* dashboard); void detach(Dashboard* dashboard); // implements Task --------------------------------------------------------------------- void setState(TaskState state); void addOutput(const std::string& text); private: MuxDashboard* mux; TaskState state; Silence silence; std::string verb; std::string noun; std::string outputText; typedef OwnedPtrMap WrappedTasksMap; WrappedTasksMap wrappedTasks; static const size_t OUTPUT_BUFER_LIMIT = 4096 - sizeof("\n...(log truncated)..."); }; const size_t MuxDashboard::TaskImpl::OUTPUT_BUFER_LIMIT; MuxDashboard::TaskImpl::TaskImpl(MuxDashboard* mux, const std::string& verb, const std::string& noun, Silence silence) : mux(mux), state(PENDING), silence(silence), verb(verb), noun(noun) { mux->tasks.insert(this); for (std::unordered_set::iterator iter = mux->wrappedDashboards.begin(); iter != mux->wrappedDashboards.end(); ++iter) { wrappedTasks.add(*iter, (*iter)->beginTask(verb, noun, silence)); } } MuxDashboard::TaskImpl::~TaskImpl() { mux->tasks.erase(this); } void MuxDashboard::TaskImpl::attach(Dashboard* dashboard) { OwnedPtr wrappedTask = dashboard->beginTask(verb, noun, silence); if (!outputText.empty()) { wrappedTask->addOutput(outputText); } if (state != PENDING) { wrappedTask->setState(state); } if (!wrappedTasks.addIfNew(dashboard, wrappedTask.release())) { DEBUG_ERROR << "Tried to attach task to a dashboard to which the task was already attached."; } } void MuxDashboard::TaskImpl::detach(Dashboard* dashboard) { if (wrappedTasks.erase(dashboard) == 0) { DEBUG_ERROR << "Tried to detach task from dashboard to which it was not attached."; } } void MuxDashboard::TaskImpl::setState(TaskState state) { if (state == PENDING || state == RUNNING) { outputText.clear(); } this->state = state; for (WrappedTasksMap::Iterator iter(wrappedTasks); iter.next();) { iter.value()->setState(state); } } void MuxDashboard::TaskImpl::addOutput(const std::string& text) { if (outputText.size() < OUTPUT_BUFER_LIMIT) { if (outputText.size() + text.size() < OUTPUT_BUFER_LIMIT) { outputText.append(text); } else { outputText.append(text, 0, OUTPUT_BUFER_LIMIT - outputText.size()); outputText.append("\n...(log truncated)..."); } } for (WrappedTasksMap::Iterator iter(wrappedTasks); iter.next();) { iter.value()->addOutput(text); } } // ======================================================================================= MuxDashboard::MuxDashboard() {} MuxDashboard::~MuxDashboard() {} OwnedPtr MuxDashboard::beginTask(const std::string& verb, const std::string& noun, Silence silence) { return newOwned(this, verb, noun, silence); } MuxDashboard::Connector::Connector(MuxDashboard* mux, Dashboard* dashboard) : mux(mux), dashboard(dashboard) { if (!mux->wrappedDashboards.insert(dashboard).second) { throw std::invalid_argument("Mux is already attached to this dashboard."); } for (std::unordered_set::iterator iter = mux->tasks.begin(); iter != mux->tasks.end(); ++iter) { (*iter)->attach(dashboard); } } MuxDashboard::Connector::~Connector() { if (mux->wrappedDashboards.erase(dashboard) == 0) { DEBUG_ERROR << "Deleting MuxDashboard connection that was never made?"; } for (std::unordered_set::iterator iter = mux->tasks.begin(); iter != mux->tasks.end(); ++iter) { (*iter)->detach(dashboard); } } } // namespace ekam ================================================ FILE: src/ekam/MuxDashboard.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_MUXDASHBOARD_H_ #define KENTONSCODE_EKAM_MUXDASHBOARD_H_ #include #include "Dashboard.h" namespace ekam { class MuxDashboard : public Dashboard { public: MuxDashboard(); ~MuxDashboard(); class Connector { public: Connector(MuxDashboard* mux, Dashboard* dashboard); ~Connector(); private: MuxDashboard* mux; Dashboard* dashboard; }; // implements Dashboard ---------------------------------------------------------------- OwnedPtr beginTask(const std::string& verb, const std::string& noun, Silence silence); private: class TaskImpl; std::unordered_set tasks; std::unordered_set wrappedDashboards; }; } // namespace ekam #endif // KENTONSCODE_EKAM_MUXDASHBOARD_H_ ================================================ FILE: src/ekam/ProtoDashboard.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "ProtoDashboard.h" #include #include #include #include #include #include "dashboard.capnp.h" #include "os/Socket.h" #include "MuxDashboard.h" namespace ekam { class ProtoDashboard::TaskImpl : public Dashboard::Task { public: TaskImpl(int id, const std::string& verb, const std::string& noun, Silence silence, WriteBuffer* output); ~TaskImpl(); // implements Task --------------------------------------------------------------------- void setState(TaskState state); void addOutput(const std::string& text); private: int id; WriteBuffer* output; static const proto::TaskUpdate::State STATE_CODES[]; }; const proto::TaskUpdate::State ProtoDashboard::TaskImpl::STATE_CODES[] = { proto::TaskUpdate::State::PENDING, proto::TaskUpdate::State::RUNNING, proto::TaskUpdate::State::DONE , proto::TaskUpdate::State::PASSED , proto::TaskUpdate::State::FAILED , proto::TaskUpdate::State::BLOCKED }; ProtoDashboard::TaskImpl::TaskImpl(int id, const std::string& verb, const std::string& noun, Silence silence, WriteBuffer* output) : id(id), output(output) { capnp::MallocMessageBuilder message; proto::TaskUpdate::Builder update = message.getRoot(); update.setId(id); update.setState(proto::TaskUpdate::State::PENDING); update.setVerb(verb); update.setNoun(noun); update.setSilent(silence == SILENT); output->write(message.getSegmentsForOutput()); } ProtoDashboard::TaskImpl::~TaskImpl() { capnp::MallocMessageBuilder message; proto::TaskUpdate::Builder update = message.getRoot(); update.setId(id); update.setState(proto::TaskUpdate::State::DELETED); output->write(message.getSegmentsForOutput()); } void ProtoDashboard::TaskImpl::setState(TaskState state) { capnp::MallocMessageBuilder message; proto::TaskUpdate::Builder update = message.getRoot(); update.setId(id); update.setState(STATE_CODES[state]); output->write(message.getSegmentsForOutput()); } void ProtoDashboard::TaskImpl::addOutput(const std::string& text) { capnp::MallocMessageBuilder message; proto::TaskUpdate::Builder update = message.getRoot(); update.setId(id); update.setLog(text); output->write(message.getSegmentsForOutput()); } // ======================================================================================= ProtoDashboard::ProtoDashboard(EventManager* eventManager, OwnedPtr stream) : idCounter(0), writeBuffer(eventManager, stream.release()) { capnp::MallocMessageBuilder message; proto::Header::Builder header = message.getRoot(); char* cwd = get_current_dir_name(); header.setProjectRoot(cwd); free(cwd); writeBuffer.write(message.getSegmentsForOutput()); } ProtoDashboard::~ProtoDashboard() {} OwnedPtr ProtoDashboard::beginTask( const std::string& verb, const std::string& noun, Silence silence) { return newOwned(++idCounter, verb, noun, silence, &writeBuffer); } // ======================================================================================= ProtoDashboard::WriteBuffer::WriteBuffer(EventManager* eventManager, OwnedPtr stream) : eventManager(eventManager), stream(stream.release()), ioWatcher(eventManager->watchFd(this->stream->getHandle()->get())), offset(0), disconnectFulfiller(NULL) {} ProtoDashboard::WriteBuffer::~WriteBuffer() {} void ProtoDashboard::WriteBuffer::write(kj::ArrayPtr> message) { if (stream == NULL) { // Already disconnected. return; } messages.push(capnp::messageToFlatArray(message)); ready(); } void ProtoDashboard::WriteBuffer::ready() { try { while (!messages.empty()) { const kj::Array& message = messages.front(); while (offset < message.size()) { offset += stream->write(message.asBytes().begin() + offset, message.asBytes().size() - offset); } offset = 0; messages.pop(); } } catch (const OsError& error) { if (error.getErrorNumber() == EAGAIN) { // Ran out of kernel buffer space. Wait until writable again. waitWritablePromise = eventManager->when(ioWatcher->onWritable())( [this](Void) { ready(); }); } else { stream.clear(); if (disconnectFulfiller != NULL) { disconnectFulfiller->disconnected(); } } } } // ======================================================================================= ProtoDashboard::WriteBuffer::DisconnectFulfiller::DisconnectFulfiller(Callback* callback, WriteBuffer* writeBuffer) : callback(callback), writeBuffer(writeBuffer) { if (writeBuffer->disconnectFulfiller != NULL) { throw std::logic_error("Can only register one disconnect callback at a time."); } writeBuffer->disconnectFulfiller = this; } ProtoDashboard::WriteBuffer::DisconnectFulfiller::~DisconnectFulfiller() { assert(writeBuffer->disconnectFulfiller == this); writeBuffer->disconnectFulfiller = NULL; } Promise ProtoDashboard::onDisconnect() { return writeBuffer.onDisconnect(); } Promise ProtoDashboard::WriteBuffer::onDisconnect() { return newPromise(this); } // ======================================================================================= class NetworkAcceptingDashboard : public Dashboard { public: NetworkAcceptingDashboard(EventManager* eventManager, const std::string& address, OwnedPtr baseDashboard) : eventManager(eventManager), base(baseDashboard.release()), baseConnector(newOwned(&mux, base.get())), socket(newOwned(eventManager, address)), acceptOp(doAccept()) {} ~NetworkAcceptingDashboard() {} Promise doAccept() { return eventManager->when(socket->accept())( [this](OwnedPtr stream){ accepted(stream.release()); return doAccept(); }); } void accepted(OwnedPtr stream); // implements Dashboard ---------------------------------------------------------------- OwnedPtr beginTask(const std::string& verb, const std::string& noun, Silence silence) { return mux.beginTask(verb, noun, silence); } private: EventManager* eventManager; OwnedPtr base; MuxDashboard mux; OwnedPtr baseConnector; OwnedPtr socket; Promise acceptOp; class ConnectedProtoDashboard { public: ConnectedProtoDashboard(NetworkAcceptingDashboard* owner, EventManager* eventManager, OwnedPtr stream) : protoDashboard(eventManager, stream.release()), connector(newOwned(&owner->mux, &protoDashboard)) { disconnectPromise = eventManager->when(protoDashboard.onDisconnect())( [this, owner](Void) { connector.clear(); owner->connectedDashboards.erase(this); }); } ~ConnectedProtoDashboard() {} private: ProtoDashboard protoDashboard; OwnedPtr connector; Promise disconnectPromise; }; OwnedPtrMap connectedDashboards; }; void NetworkAcceptingDashboard::accepted(OwnedPtr stream) { auto connectedDashboard = newOwned(this, eventManager, stream.release()); auto key = connectedDashboard.get(); // cannot inline due to undefined evaluation order connectedDashboards.add(key, connectedDashboard.release()); } OwnedPtr initNetworkDashboard(EventManager* eventManager, const std::string& address, OwnedPtr dashboardToWrap) { return newOwned(eventManager, address, dashboardToWrap.release()); } } // namespace ekam ================================================ FILE: src/ekam/ProtoDashboard.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_PROTODASHBOARD_H_ #define KENTONSCODE_EKAM_PROTODASHBOARD_H_ #include #include #include #include "Dashboard.h" #include "os/ByteStream.h" #include "os/EventManager.h" namespace ekam { class ProtoDashboard : public Dashboard { public: ProtoDashboard(EventManager* eventManager, OwnedPtr stream); ~ProtoDashboard(); Promise onDisconnect(); // implements Dashboard ---------------------------------------------------------------- OwnedPtr beginTask(const std::string& verb, const std::string& noun, Silence silence); private: class TaskImpl; class WriteBuffer { public: WriteBuffer(EventManager* eventManager, OwnedPtr stream); ~WriteBuffer(); void write(kj::ArrayPtr> data); Promise onDisconnect(); private: EventManager* eventManager; OwnedPtr stream; OwnedPtr ioWatcher; std::queue> messages; std::string::size_type offset; Promise waitWritablePromise; class DisconnectFulfiller : public PromiseFulfiller { public: DisconnectFulfiller(Callback* callback, WriteBuffer* writeBuffer); ~DisconnectFulfiller(); void disconnected() { callback->fulfill(); } private: Callback* callback; WriteBuffer* writeBuffer; }; DisconnectFulfiller* disconnectFulfiller; void ready(); }; int idCounter; WriteBuffer writeBuffer; }; } // namespace ekam #endif // KENTONSCODE_EKAM_PROTODASHBOARD_H_ ================================================ FILE: src/ekam/SimpleDashboard.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "SimpleDashboard.h" namespace ekam { class SimpleDashboard::TaskImpl : public Dashboard::Task { public: TaskImpl(const std::string& verb, const std::string& noun, Silence silence, FILE* outputStream); ~TaskImpl(); // implements Task --------------------------------------------------------------------- void setState(TaskState state); void addOutput(const std::string& text); private: TaskState state; Silence silence; std::string verb; std::string noun; std::string outputText; FILE* outputStream; static const char* const STATE_NAMES[]; }; const char* const SimpleDashboard::TaskImpl::STATE_NAMES[] = { "PENDING", "RUNNING", "DONE ", "PASSED ", "FAILED ", "BLOCKED" }; SimpleDashboard::TaskImpl::TaskImpl(const std::string& verb, const std::string& noun, Silence silence, FILE* outputStream) : state(PENDING), silence(silence), verb(verb), noun(noun), outputStream(outputStream) {} SimpleDashboard::TaskImpl::~TaskImpl() {} void SimpleDashboard::TaskImpl::setState(TaskState state) { // If state was previously BLOCKED, and we managed to un-block, then we don't care about the // reason why we were blocked, so clear the text. if (this->state == BLOCKED && (state == PENDING || state == RUNNING)) { outputText.clear(); } this->state = state; bool writeOutput = !outputText.empty() && state != BLOCKED; if (silence != SILENT || writeOutput) { // Write status update. fprintf(outputStream, "[%s] %s: %s\n", STATE_NAMES[state], verb.c_str(), noun.c_str()); // Write any output we have buffered, unless new state is BLOCKED in which case we save the // output for later. if (writeOutput) { fwrite(outputText.c_str(), sizeof(char), outputText.size(), outputStream); if (outputText[outputText.size() - 1] != '\n') { fputc('\n', outputStream); } outputText.clear(); } fflush(outputStream); } } void SimpleDashboard::TaskImpl::addOutput(const std::string& text) { outputText.append(text); } // ======================================================================================= SimpleDashboard::SimpleDashboard(FILE* outputStream) : outputStream(outputStream) {} SimpleDashboard::~SimpleDashboard() {} OwnedPtr SimpleDashboard::beginTask( const std::string& verb, const std::string& noun, Silence silence) { return newOwned(verb, noun, silence, outputStream); } } // namespace ekam ================================================ FILE: src/ekam/SimpleDashboard.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_SIMPLEDASHBOARD_H_ #define KENTONSCODE_EKAM_SIMPLEDASHBOARD_H_ #include #include "Dashboard.h" namespace ekam { class SimpleDashboard : public Dashboard { public: SimpleDashboard(FILE* outputStream); ~SimpleDashboard(); // implements Dashboard ---------------------------------------------------------------- OwnedPtr beginTask(const std::string& verb, const std::string& noun, Silence silence); private: class TaskImpl; FILE* outputStream; }; } // namespace ekam #endif // KENTONSCODE_EKAM_SIMPLEDASHBOARD_H_ ================================================ FILE: src/ekam/Tag.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Tag.h" #include "base/Debug.h" namespace ekam { namespace { std::string canonicalizePath(const std::string& path) { std::vector parts; std::string::size_type pos = 0; while (pos != std::string::npos) { std::string::size_type slashPos = path.find_first_of('/', pos); std::string part; if (slashPos == std::string::npos) { part.assign(path, pos, path.size() - pos); pos = slashPos; } else { part.assign(path, pos, slashPos - pos); pos = path.find_first_not_of('/', slashPos); } if (part.empty() || part == ".") { // skip } else if (part == "..") { if (parts.empty()) { // ignore } else { parts.pop_back(); } } else { parts.push_back(part); } } std::string result; result.reserve(path.size()); for (unsigned int i = 0; i < parts.size(); i++) { if (i > 0) { result.push_back('/'); } result.append(parts[i]); } return result; } } // namespace const Tag Tag::DEFAULT_TAG = Tag::fromName("file:*"); Tag Tag::fromFile(const std::string& path) { return fromName("file:" + canonicalizePath(path)); } } // namespace ekam ================================================ FILE: src/ekam/Tag.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_EKAM_TAG_H_ #define KENTONSCODE_EKAM_TAG_H_ #include #include #include #include #include "base/Hash.h" namespace ekam { class File; class Tag { public: Tag() {} // Every file has this tag. static const Tag DEFAULT_TAG; static inline Tag fromName(const std::string& name) { return Tag(name); } static Tag fromFile(const std::string& path); inline std::string toString() { return hash.toString(); } inline bool operator==(const Tag& other) const { return hash == other.hash; } inline bool operator!=(const Tag& other) const { return hash != other.hash; } inline bool operator< (const Tag& other) const { return hash < other.hash; } inline bool operator> (const Tag& other) const { return hash > other.hash; } inline bool operator<=(const Tag& other) const { return hash <= other.hash; } inline bool operator>=(const Tag& other) const { return hash >= other.hash; } class HashFunc { public: inline size_t operator()(const Tag& id) const { return inner(id.hash); } private: Hash::StlHashFunc inner; }; private: Hash hash; #ifdef EXTRA_DEBUG std::string name; inline explicit Tag(const std::string& name) : hash(Hash::of(name)), name(name) {} #else inline explicit Tag(const std::string& name) : hash(Hash::of(name)) {} #endif }; } // namespace ekam #endif // KENTONSCODE_EKAM_TAG_H_ ================================================ FILE: src/ekam/dashboard.capnp ================================================ # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. @0xa610fcb94ceea9cc; $import "/capnp/c++.capnp".namespace("ekam::proto"); struct Header { # The first message sent over the stream is the header. projectRoot @0 :Text; # The directory where ekam was run, containing "src", "tmp", etc. } struct TaskUpdate { # All subsequent messages are TaskUpdates. id @0 :UInt32; state @1 :State = unchanged; enum State { unchanged @0; deleted @1; pending @2; running @3; done @4; passed @5; failed @6; blocked @7; } verb @2 :Text; noun @3 :Text; silent @4 :Bool; log @5 :Text; } ================================================ FILE: src/ekam/ekam-client.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "dashboard.capnp.h" #include #include #include #include #include #include #include "ConsoleDashboard.h" #include "base/OwnedPtr.h" namespace ekam { void dump(proto::TaskUpdate::Reader message) { using std::cerr; using std::cout; using std::endl; cout << "================================================================================\n"; cout << message.getId() << ":"; if (message.getState() != proto::TaskUpdate::State::UNCHANGED) { cout << " " << kj::str(message.getState()).cStr(); } if (message.hasVerb()) { cout << " " << message.getVerb().cStr(); } if (message.hasNoun()) { cout << " " << message.getNoun().cStr(); } if (message.getSilent()) { cout << " (silent)"; } cout << '\n'; if (message.hasLog()) { auto log = message.getLog(); cout << log.cStr(); if (!log.endsWith("\n")) { cout << '\n'; } } cout.flush(); } Dashboard::TaskState toDashboardState(proto::TaskUpdate::State state) { switch (state) { case proto::TaskUpdate::State::PENDING: return Dashboard::PENDING; case proto::TaskUpdate::State::RUNNING: return Dashboard::RUNNING; case proto::TaskUpdate::State::DONE: return Dashboard::DONE; case proto::TaskUpdate::State::PASSED: return Dashboard::PASSED; case proto::TaskUpdate::State::FAILED: return Dashboard::FAILED; case proto::TaskUpdate::State::BLOCKED: return Dashboard::BLOCKED; default: throw std::invalid_argument(kj::str("Invalid state: ", state).cStr()); } } int main(int argc, char* argv[]) { int maxDisplayedLogLines = 30; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-l") == 0) { char* endptr; if (i + 1 >= argc || (maxDisplayedLogLines = strtoul(argv[++i], &endptr, 0), *endptr != '\0')) { fprintf(stderr, "Expected number after -l.\n"); return 1; } } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { printf( "usage: nc | %s [-l ]\n" "\n" "Connect to Ekam process at and display build status.\n" "\n" "options:\n" " -l Set max number of log lines to display per action. This is\n" " kept relatively short by default because it makes the build\n" " output noisy, but you may need to increase it if you need\n" " to see more of a particular error log.\n", argv[0]); return 0; } else { fprintf(stderr, "Unknown option: %s\n", argv[i]); return 1; } } kj::FdInputStream rawInput(STDIN_FILENO); kj::BufferedInputStreamWrapper bufferedInput(rawInput); { capnp::InputStreamMessageReader message(bufferedInput); proto::Header::Reader header = message.getRoot(); printf("Project root: %s\n", header.getProjectRoot().cStr()); } ConsoleDashboard dashboard(stdout, maxDisplayedLogLines); OwnedPtrMap tasks; while (bufferedInput.tryGetReadBuffer() != nullptr) { capnp::InputStreamMessageReader messageReader(bufferedInput); proto::TaskUpdate::Reader message = messageReader.getRoot(); if (message.getState() == proto::TaskUpdate::State::DELETED) { tasks.erase(message.getId()); } else if (Dashboard::Task* task = tasks.get(message.getId())) { if (message.hasLog()) { task->addOutput(message.getLog()); } if (message.getState() != proto::TaskUpdate::State::UNCHANGED) { task->setState(toDashboardState(message.getState())); } } else { OwnedPtr newTask = dashboard.beginTask( message.getVerb(), message.getNoun(), message.getSilent() ? Dashboard::SILENT : Dashboard::NORMAL); if (message.hasLog()) { newTask->addOutput(message.getLog()); } if (message.getState() != proto::TaskUpdate::State::UNCHANGED && message.getState() != proto::TaskUpdate::State::PENDING) { newTask->setState(toDashboardState(message.getState())); } tasks.add(message.getId(), newTask.release()); } } return 0; } } // namespace ekam int main(int argc, char* argv[]) { return ekam::main(argc, argv); } ================================================ FILE: src/ekam/ekam-langserve.c++ ================================================ #include #include #include #include #include #include #include #include #include #include #include #include namespace ekam { typedef unsigned int uint; class AsyncIoStreamPair final: public kj::AsyncIoStream { public: AsyncIoStreamPair(kj::Own input, kj::Own output) : input(kj::mv(input)), output(kj::mv(output)) {} kj::Promise read(void* buffer, size_t minBytes, size_t maxBytes) override { return input->read(buffer, minBytes, maxBytes); } kj::Promise tryRead(void* buffer, size_t minBytes, size_t maxBytes) override { return input->tryRead(buffer, minBytes, maxBytes); } kj::Maybe tryGetLength() override { return input->tryGetLength(); } kj::Promise pumpTo( AsyncOutputStream& output, uint64_t amount = kj::maxValue) override { return input->pumpTo(output, amount); } kj::Promise write(const void* buffer, size_t size) override { return output->write(buffer, size); } kj::Promise write(kj::ArrayPtr> pieces) override { return output->write(pieces); } kj::Maybe> tryPumpFrom( AsyncInputStream& input, uint64_t amount = kj::maxValue) override { return output->tryPumpFrom(input, amount); } kj::Promise whenWriteDisconnected() override { return output->whenWriteDisconnected(); } void shutdownWrite() override { output = nullptr; } void abortRead() override { input = nullptr; } private: kj::Own input; kj::Own output; }; class SourceFile; struct Message { SourceFile* file; uint line; uint column; uint endColumn; kj::String text; inline bool operator==(const Message& other) const { return file == other.file && line == other.line && column == other.column && endColumn == other.endColumn && text == other.text; } inline uint hashCode() const { return kj::hashCode(file, line, column, endColumn, text); } void exportRange(lsp::Range::Builder range) { auto start = range.initStart(); start.setLine(line == 0 ? 0 : line - 1); start.setCharacter(column == 0 ? 0 : column - 1); auto end = range.initEnd(); end.setLine(line == 0 ? 0 : line - 1); end.setCharacter(endColumn == 0 ? (column == 0 ? 65536 : column - 1) : endColumn - 1); } }; enum class Severity: uint8_t { NOTE = 0, ERROR = 1, WARNING = 2, INFO = 3, HINT = 4 }; struct Diagnostic { Severity severity; Message message; kj::Vector notes; inline bool operator==(const Diagnostic& other) const { return severity == other.severity && message == other.message; } inline uint hashCode() const { return kj::hashCode(static_cast(severity), message); } }; class SourceFile; class DirtySet { public: void add(SourceFile* file) { dirty.upsert(file, [](auto...) {}); KJ_IF_MAYBE(f, fulfiller) { f->get()->fulfill(); fulfiller = nullptr; } } template void forEach(Func&& func) { for (auto file: dirty) { func(*file); } dirty.clear(); } kj::Maybe> whenNonEmpty() { KJ_REQUIRE(fulfiller == nullptr, "can only call whenNonEmpty() once at a time"); if (dirty.size() > 0) { return kj::Promise(kj::READY_NOW); } else if (isShutdown) { return nullptr; } else { auto paf = kj::newPromiseAndFulfiller(); fulfiller = kj::mv(paf.fulfiller); return kj::mv(paf.promise); } } void shutdown() { // Make it so calling whenNotEmpty() when empty returns null rather than blocking. KJ_IF_MAYBE(f, fulfiller) { f->get()->fulfill(); fulfiller = nullptr; } isShutdown = true; } private: kj::HashSet dirty; kj::Maybe>> fulfiller; bool isShutdown = false; }; class SourceFile { public: SourceFile(DirtySet& dirtySet, kj::String realPath) : dirtySet(dirtySet), realPath(kj::mv(realPath)) {} inline kj::StringPtr getRealPath() { return realPath; } void markDirty() { dirtySet.add(this); } void markStale() { // We were informed the file has changed, which potentially invalidates all diagnostics. // Mark them all stale. bool dirty = false; for (auto& diagnostic: diagnostics) { if (!diagnostic.stale) { diagnostic.stale = true; dirty = true; } } if (dirty) { markDirty(); } } Diagnostic& add(Diagnostic&& diagnostic) { auto& result = diagnostics.findOrCreate(diagnostic, [&]() { dirtySet.add(this); return DiagnosticEntry { kj::heap(kj::mv(diagnostic)), 0 }; }); ++result.refcount; result.stale = false; return *result.diagnostic; } void remove(Diagnostic& diagnostic) { auto& entry = KJ_ASSERT_NONNULL(diagnostics.find(diagnostic)); KJ_ASSERT(entry.diagnostic.get() == &diagnostic); if (--entry.refcount == 0) { diagnostics.erase(entry); dirtySet.add(this); } } void exportDiagnostics(kj::StringPtr uriPrefix, lsp::LanguageClient::PublishDiagnosticsParams::Builder builder) { builder.setUri(kj::str(uriPrefix, realPath)); size_t count = 0; for (auto& diagnostic: diagnostics) { if (!diagnostic.stale) ++count; } auto list = builder.initDiagnostics(count); auto iter = diagnostics.begin(); for (auto out: list) { while (iter->stale) ++iter; auto& in = *(iter++)->diagnostic; out.setSeverity((uint)in.severity); in.message.exportRange(out.initRange()); out.setMessage(in.message.text); out.setSource("ekam"); if (!in.notes.empty()) { auto notes = out.initRelatedInformation(in.notes.size()); for (auto i: kj::indices(in.notes)) { auto outNote = notes[i]; auto& inNote = in.notes[i]; auto loc = outNote.initLocation(); loc.setUri(kj::str(uriPrefix, inNote.file->realPath)); inNote.exportRange(loc.initRange()); outNote.setMessage(inNote.text); } } } } private: DirtySet& dirtySet; kj::String realPath; struct DiagnosticEntry { kj::Own diagnostic; uint refcount; bool stale = false; bool operator==(const Diagnostic& other) const { return *diagnostic == other; } bool operator==(const DiagnosticEntry& other) const { return *diagnostic == *other.diagnostic; } uint hashCode() const { return kj::hashCode(*diagnostic); } }; kj::HashSet diagnostics; }; class SourceFileSet { public: SourceFileSet(const kj::ReadableDirectory& projectHome, DirtySet& dirtySet) : projectHome(projectHome), dirtySet(dirtySet) {} kj::Maybe get(kj::StringPtr name) { static constexpr kj::StringPtr STRIP_PREFIXES[] = { "/ekam-provider/canonical/"_kj, "/ekam-provider/c++header/"_kj }; for (auto prefix: STRIP_PREFIXES) { if (name.startsWith(prefix)) { name = name.slice(prefix.size()); break; } } auto deref = [](kj::Maybe>& maybe) { return maybe.map([](kj::Own& own) -> SourceFile& { return *own; }); }; KJ_IF_MAYBE(existing, files.find(name)) { return deref(*existing); } if (name.startsWith("/")) { // Absolute path, probably a header or something. return insert(name, kj::heap(dirtySet, kj::str(name))); } else { // Look for the file under `src` and `tmp`. auto canonical = kj::Path::parse(name); static constexpr kj::StringPtr DIRS[] = {"src"_kj, "tmp"_kj}; for (auto dir: DIRS) { auto path = kj::Path({dir}).append(canonical); if (projectHome.exists(path)) { // Found it. return insert(name, kj::heap(dirtySet, expandSymlinks(kj::mv(path)).toString())); } } if (projectHome.exists(canonical)) { // Maybe it was already non-canonical. return insert(name, kj::heap(dirtySet, expandSymlinks(kj::mv(canonical)).toString())); } // This path doesn't appear to be a file. Cache this fact. files.insert(kj::str(name), nullptr); return nullptr; } } kj::Maybe findByRealPath(kj::StringPtr path) { return filesByPath.find(path).map([](SourceFile* f) -> SourceFile& { return *f; }); } template void forEach(Func&& func) { for (auto& entry: files) { KJ_IF_MAYBE(f, entry.value) { func(**f); } } } private: const kj::ReadableDirectory& projectHome; DirtySet& dirtySet; kj::HashMap>> files; kj::HashMap filesByPath; SourceFile& insert(kj::StringPtr name, kj::Own file) { auto& result = *file; auto& realPathEntry = filesByPath.insert(file->getRealPath(), file.get()); KJ_ON_SCOPE_FAILURE(filesByPath.erase(realPathEntry)); files.insert(kj::str(name), kj::mv(file)); return result; } kj::Path expandSymlinks(kj::Path path) { retry: for (size_t i = 1; i < path.size(); i++) { auto parent = path.slice(0, i); KJ_IF_MAYBE(link, projectHome.tryReadlink(parent)) { if (!link->startsWith("/")) { try { path = parent.slice(0, i-1).eval(*link).append(path.slice(i, path.size())); goto retry; } catch (const kj::Exception& e) { KJ_LOG(WARNING, "bad symlink", *link, e.getDescription()); } } } } return kj::mv(path); } }; kj::Maybe tryConsumeNumberColon(kj::StringPtr& str) { if (str.startsWith(":")) return nullptr; for (size_t i = 0; i < str.size(); i++) { if (str[i] == ':') { uint result = strtoul(str.cStr(), nullptr, 10); str = str.slice(i + 1); return result; } else if (str[i] < '0' || str[i] > '9') { return nullptr; } } return nullptr; } struct ColumnRange { uint start; uint end; }; kj::Maybe tryConsumeRangeColon(kj::StringPtr& str) { if (str.startsWith(":")) return nullptr; kj::Maybe start; size_t startPos = 0; for (size_t i = 0; i < str.size(); i++) { if (str[i] == ':') { uint result = strtoul(str.cStr() + startPos, nullptr, 10); str = str.slice(i + 1); KJ_IF_MAYBE(s, start) { return ColumnRange { *s, result }; } else { return ColumnRange { result, result }; } } else if (str[i] == '-' && start == nullptr) { start = strtoul(str.cStr(), nullptr, 10); startPos = i + 1; } else if (str[i] < '0' || str[i] > '9') { return nullptr; } } return nullptr; } void trimLeadingSpace(kj::StringPtr& str) { while (str.startsWith(" ")) { str = str.slice(1); } } Severity consumeSeverity(kj::StringPtr& line) { if (line.startsWith("note:")) { line = line.slice(5); trimLeadingSpace(line); return Severity::NOTE; } else { struct SeverityPrefix { kj::StringPtr prefix; Severity severity; }; static constexpr SeverityPrefix SEVERITY_PREFIXES[] = { { "error:"_kj, Severity::ERROR }, { "warning:"_kj, Severity::WARNING }, { "warn:"_kj, Severity::WARNING }, { "info:"_kj, Severity::INFO }, { "hint:"_kj, Severity::HINT }, }; for (auto prefix: SEVERITY_PREFIXES) { if (line.startsWith(prefix.prefix)) { line = line.slice(prefix.prefix.size()); trimLeadingSpace(line); return prefix.severity; } } return Severity::ERROR; } } class Task { public: Task(proto::TaskUpdate::Reader update, SourceFileSet& files) {} ~Task() noexcept(false) { clearDiagnostics(); } void update(proto::TaskUpdate::Reader update, SourceFileSet& files) { // Invalidate log when the task is deleted or it is re-running or scheduled to re-run. if (update.getState() == proto::TaskUpdate::State::PENDING || update.getState() == proto::TaskUpdate::State::RUNNING || update.getState() == proto::TaskUpdate::State::DELETED) { clearDiagnostics(); } kj::StringPtr log = update.getLog(); kj::String ownLog; if (leftoverLog.size() > 0) { ownLog = kj::str(leftoverLog, log); log = ownLog; } for (;;) { kj::StringPtr line; kj::String ownLine; KJ_IF_MAYBE(eol, log.findFirst('\n')) { ownLine = kj::str(log.slice(0, *eol)); line = ownLine; log = log.slice(*eol + 1); } else { leftoverLog = kj::str(log); break; } trimLeadingSpace(line); if (line == nullptr) continue; static constexpr kj::StringPtr IGNORE_PREFIXES[] = { "In file included from "_kj }; bool ignore = false; for (auto prefix: IGNORE_PREFIXES) { if (line.startsWith(prefix)) { ignore = true; break; } } if (ignore) continue; static constexpr kj::StringPtr STRIP_PREFIXES[] = { // Linker errors start with this. "/usr/bin/ld: "_kj }; for (auto prefix: STRIP_PREFIXES) { if (line.startsWith(prefix)) { line = line.slice(prefix.size()); } } size_t spaceAt = line.findFirst(' ').orDefault(line.size()); if (line[spaceAt - 1] != ':') { // No file:line:column: to parse... skip. continue; } // parse file:line:column: size_t pos = KJ_ASSERT_NONNULL(line.findFirst(':')); auto filename = kj::str(line.slice(0, pos)); KJ_IF_MAYBE(file, files.get(filename)) { line = line.slice(pos + 1); kj::Maybe lineNo = tryConsumeNumberColon(line); kj::Maybe columnRange = tryConsumeRangeColon(line); trimLeadingSpace(line); Severity severity = consumeSeverity(line); Message message { file, lineNo.orDefault(0), columnRange.map([](ColumnRange c) { return c.start; }).orDefault(0), columnRange.map([](ColumnRange c) { return c.end; }).orDefault(0), kj::str(line) }; if (severity == Severity::NOTE) { // Append note to previous diagnostic. if (addNotesToBack) { diagnostics.back()->notes.add(kj::mv(message)); diagnostics.back()->message.file->markDirty(); } } else { Diagnostic diagnostic { severity, kj::mv(message), {} }; diagnostics.add(&file->add(kj::mv(diagnostic))); // If the notes aren't empty, then this must be a dupe diagnostic, and we don't want to // add duplicate notes. addNotesToBack = diagnostics.back()->notes.empty(); } } else { // Doesn't appear to start with a filename. Skip. continue; } } } private: kj::Vector diagnostics; kj::String leftoverLog; bool addNotesToBack = false; void clearDiagnostics() { for (auto diagnostic: diagnostics) { diagnostic->message.file->remove(*diagnostic); } diagnostics.clear(); addNotesToBack = false; } }; class LanguageServerImpl final: public lsp::LanguageServer::Server { public: LanguageServerImpl(kj::Own> initializedFulfiller) : initializedFulfiller(kj::mv(initializedFulfiller)) {} KJ_DISALLOW_COPY(LanguageServerImpl); class Scope { public: Scope(LanguageServerImpl& server, SourceFileSet& files, kj::StringPtr uriPrefix) : server(server), files(files), uriPrefix(uriPrefix) { server.scope = this; } ~Scope() noexcept(false) { server.scope = nullptr; } private: LanguageServerImpl& server; SourceFileSet& files; kj::StringPtr uriPrefix; friend class LanguageServerImpl; }; protected: kj::Promise initialize(InitializeContext context) override { initializedFulfiller->fulfill(); auto sync = context.initResults().initCapabilities().initTextDocumentSync(); sync.setOpenClose(true); sync.setChange(lsp::TextDocumentSyncKind::INCREMENTAL); sync.setDidSave(true); return kj::READY_NOW; } kj::Promise shutdown(ShutdownContext context) override { return kj::READY_NOW; } kj::Promise exit(ExitContext context) override { _exit(0); } kj::Promise didOpen(DidOpenContext context) override { // Ignore. return kj::READY_NOW; } kj::Promise didClose(DidCloseContext context) override { // Ignore. return kj::READY_NOW; } kj::Promise didChange(DidChangeContext context) override { KJ_IF_MAYBE(s, scope) { auto params = context.getParams(); auto uri = params.getTextDocument().getUri(); if (uri.startsWith(s->uriPrefix)) { auto path = kj::decodeUriComponent(uri.slice(s->uriPrefix.size())); KJ_IF_MAYBE(file, s->files.findByRealPath(path)) { file->markStale(); } } } return kj::READY_NOW; } kj::Promise didSave(DidSaveContext context) override { // Ignore for now. // TODO(someday): Start a new location map for this file and interpret future diagnostics // against that map rather than the current one. return kj::READY_NOW; } private: kj::Own> initializedFulfiller; kj::Maybe scope; }; class LanguageServerMain { public: LanguageServerMain(kj::ProcessContext& context): context(context) {} kj::MainFunc getMain() { return kj::MainBuilder(context, "Ekam Language Server", "Implements the VS Code Language Server Protocol to report errors from Ekam.") .expectArg("
", KJ_BIND_METHOD(*this, run)) .build(); } kj::MainBuilder::Validity run(kj::StringPtr addr) { auto io = kj::setupAsyncIo(); auto fs = kj::newDiskFilesystem(); auto parsedAddr = io.provider->getNetwork().parseAddress(addr).wait(io.waitScope); auto stream = kj::heap( io.lowLevelProvider->wrapInputFd(kj::AutoCloseFd(STDIN_FILENO)), io.lowLevelProvider->wrapOutputFd(kj::AutoCloseFd(STDOUT_FILENO))); auto initPaf = kj::newPromiseAndFulfiller(); auto ownServer = kj::heap(kj::mv(initPaf.fulfiller)); LanguageServerImpl& server = *ownServer; capnp::JsonRpc::ContentLengthTransport transport(*stream); capnp::JsonRpc jsonRpc(transport, capnp::toDynamic(kj::mv(ownServer))); auto errorTask = jsonRpc.onError() .then([]() { KJ_LOG(ERROR, "JsonRpc.onError() resolved normally?"); }, [](kj::Exception&& exception) { KJ_LOG(ERROR, "JSON-RPC connection failed", exception); }).attach(kj::defer([]() { exit(1); })).eagerlyEvaluate(nullptr); auto client = jsonRpc.getPeer(); // Wait until initialized, as vscode will ignore diagnostics delivered too soon. initPaf.promise.wait(io.waitScope); for (;;) { // Repeatedly try to connect to Ekam. kj::Own ekamConnection; for (;;) { KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() { ekamConnection = parsedAddr->connect().wait(io.waitScope); })) { if (exception->getType() == kj::Exception::Type::DISCONNECTED) { io.provider->getTimer().afterDelay(2 * kj::SECONDS).wait(io.waitScope); } else { kj::throwFatalException(kj::mv(*exception)); } } else { break; } } // Read first message from Ekam. kj::String homeUri; auto projectHome = ({ auto message = capnp::readMessage(*ekamConnection).wait(io.waitScope); auto header = message->getRoot(); auto path = kj::str(header.getProjectRoot()); homeUri = kj::str("file://", path, '/'); KJ_ASSERT(path.startsWith("/")); fs->getRoot().openSubdir(kj::Path::parse(path.slice(1))); }); DirtySet dirtySet; SourceFileSet files(*projectHome, dirtySet); kj::HashMap> tasks; LanguageServerImpl::Scope serverScope(server, files, homeUri); kj::Promise updateLoopTask = updateLoop(dirtySet, client, homeUri); for (;;) { kj::Own message; KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() { message = capnp::readMessage(*ekamConnection).wait(io.waitScope); })) { if (exception->getType() == kj::Exception::Type::DISCONNECTED) { // Disconnected, start over. break; } else { kj::throwFatalException(kj::mv(*exception)); } } auto update = message->getRoot(); if (update.getState() == proto::TaskUpdate::State::DELETED) { tasks.erase(update.getId()); } else { auto& task = *tasks.findOrCreate(update.getId(), [&]() { return kj::HashMap>::Entry { update.getId(), kj::heap(update, files) }; }); task.update(update, files); } } // Clear all diagnostics. tasks.clear(); dirtySet.shutdown(); updateLoopTask.wait(io.waitScope); } } private: kj::ProcessContext& context; kj::Promise updateLoop(DirtySet& dirtySet, lsp::LanguageClient::Client client, kj::StringPtr homeUri) { KJ_IF_MAYBE(p, dirtySet.whenNonEmpty()) { return p->then([]() { // Delay for other messages. return kj::evalLast([]() {}); }).then([&dirtySet, client, homeUri]() mutable { kj::Vector> promises; dirtySet.forEach([&](SourceFile& file) { auto req = client.publishDiagnosticsRequest(); file.exportDiagnostics(homeUri, req); promises.add(req.send().ignoreResult()); }); return kj::joinPromises(promises.releaseAsArray()); }).then([this, &dirtySet, client, homeUri]() mutable { return updateLoop(dirtySet, kj::mv(client), homeUri); }); } else { return kj::READY_NOW; } } }; } // namespace ekam KJ_MAIN(ekam::LanguageServerMain); ================================================ FILE: src/ekam/ekam.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Driver.h" #include "base/Debug.h" #include "os/DiskFile.h" #include "Action.h" #include "SimpleDashboard.h" #include "ConsoleDashboard.h" #include "CppActionFactory.h" #include "ExecPluginActionFactory.h" #include "os/OsHandle.h" namespace ekam { class ExtractTypeAction : public Action { public: ExtractTypeAction(File* file) : file(file->clone()) {} ~ExtractTypeAction() {} // implements Action ------------------------------------------------------------------- bool isSilent() { return true; } std::string getVerb() { return "scan"; } Promise start(EventManager* eventManager, BuildContext* context) { std::vector tags; std::string name = file->canonicalName(); tags.push_back(Tag::fromName("canonical:" + name)); while (true) { tags.push_back(Tag::fromFile(name)); std::string::size_type slashPos = name.find_first_of('/'); if (slashPos == std::string::npos) { break; } name.erase(0, slashPos + 1); } if (file->isDirectory()) { tags.push_back(Tag::fromName("directory:*")); } else { std::string base, ext; splitExtension(name, &base, &ext); if (!ext.empty()) tags.push_back(Tag::fromName("filetype:" + ext)); } context->provide(file.get(), tags); return newFulfilledPromise(); } private: OwnedPtr file; }; class ExtractTypeActionFactory : public ActionFactory { public: ExtractTypeActionFactory() {} ~ExtractTypeActionFactory() {} // implements ActionFactory ------------------------------------------------------------ void enumerateTriggerTags(std::back_insert_iterator > iter) { *iter++ = Tag::DEFAULT_TAG; } OwnedPtr tryMakeAction(const Tag& id, File* file) { return newOwned(file); } }; void usage(const char* command, FILE* out) { fprintf(out, "usage: %s [-hvc] [-j ] [-n []:] [-l ]\n" "\n" "Build code with Ekam. See https://github.io/sandstorm-io/ekam for details.\n" "\n" "options:\n" " -c Run in continuous mode: when there is nothing left to build,\n" " don't exit, but instead watch the source files for changes\n" " and rebuild as necessary.\n" " -j Run up to actions in parallel.\n" " -n []: Accept network connections on the given address/port\n" " and give real-time build status and logs to anyone who\n" " connects. This enables e.g. `ekam-client` and various IDE\n" " plugins.\n" " -l Set max number of log lines to display per action. This is\n" " kept relatively short by default because it makes the build\n" " output noisy, but you may need to increase it if you need\n" " to see more of a particular error log. NOTE: If you just\n" " need a one-off, you can use `ekam-client` rather than\n" " restarting Ekam.\n" " -h See this help\n" " -v Show debug logs.\n", command); } // ======================================================================================= // TODO: Move file-watching code to another module. class Watcher { public: Watcher(OwnedPtr file, EventManager* eventManager, Driver* driver, bool isDirectory) : eventManager(eventManager), driver(driver), isDirectory(isDirectory), file(file.release()) { resetWatch(); } virtual ~Watcher() {} EventManager* const eventManager; Driver* const driver; const bool isDirectory; OwnedPtr file; void resetWatch() { asyncOp.release(); diskRef = file->getOnDisk(File::READ); watcher = eventManager->watchFile(diskRef->path()); waitForEvent(); } void waitForEvent() { asyncOp = eventManager->when(watcher->onChange())( [this](EventManager::FileChangeType changeType) { switch (changeType) { case EventManager::FileChangeType::MODIFIED: modified(); break; case EventManager::FileChangeType::DELETED: deleted(); break; } waitForEvent(); }); } void clearWatch() { asyncOp.release(); } bool isDeleted() { return asyncOp == nullptr; } virtual void created() = 0; virtual void modified() = 0; virtual void deleted() = 0; virtual void reallyDeleted() = 0; private: OwnedPtr diskRef; OwnedPtr watcher; Promise asyncOp; }; class FileWatcher : public Watcher { public: FileWatcher(OwnedPtr file, EventManager* eventManager, Driver* driver) : Watcher(file.release(), eventManager, driver, false) {} ~FileWatcher() {} // implements FileChangeCallback ------------------------------------------------------- void created() { DEBUG_INFO << "Source file created: " << file->canonicalName(); modified(); } void modified() { DEBUG_INFO << "Source file modified: " << file->canonicalName(); driver->addSourceFile(file.get()); } void deleted() { if (file->isFile()) { // A new file was created in place of the old. Reset the watch. DEBUG_INFO << "Source file replaced: " << file->canonicalName(); resetWatch(); modified(); } else { reallyDeleted(); } } // implements Watcher ------------------------------------------------------------------ void reallyDeleted() { DEBUG_INFO << "Source file deleted: " << file->canonicalName(); clearWatch(); driver->removeSourceFile(file.get()); } }; class DirectoryWatcher : public Watcher { typedef OwnedPtrMap ChildMap; public: DirectoryWatcher(OwnedPtr file, EventManager* eventManager, Driver* driver) : Watcher(file.release(), eventManager, driver, true) {} ~DirectoryWatcher() {} // implements FileChangeCallback ------------------------------------------------------- void created() { driver->addSourceFile(file.get()); modified(); } void modified() { DEBUG_INFO << "Directory modified: " << file->canonicalName(); OwnedPtrVector list; try { file->list(list.appender()); } catch (OsError& e) { // Probably the directory has been deleted but we weren't yet notified. reallyDeleted(); return; } ChildMap newChildren; // Build new child list, copying over child watchers where possible. for (int i = 0; i < list.size(); i++) { OwnedPtr childFile = list.release(i); OwnedPtr child; bool childIsDirectory = childFile->isDirectory(); // When a file is deleted and replaced with a new one of the same type, we run into a lot // of awkward race conditions. There are three things that can happen in any order: // 1) Notification of file deletion. // 2) Notification of directory change. // 3) New file is created. // // Here is how we handle each possible ordering: // 1, 2, 3) File will not show up in directory list, so we won't transfer the watcher or // create a new one. It will be destroyed. // 1, 3, 2) child->isDeleted will be true so we'll create a new watcher to replace it. // 2, 1, 3) Like 1, 2, 3 except we directly call deleted() on the old child watcher from // this function (see below). We actually never receive the file deletion event from // the EventManager in this case. // 2, 3, 1) Same as 2, 1, 3. // 3, 1, 2) File watcher notices new file already exists and simply resumes watching. // Parent watcher thinks nothing happened. // 3, 2, 1) Same as 3, 1, 2. // // The last two are different if a file was replaced with a directory or vice versa: // 3, 1, 2) Child watcher notices replacement is a different type and so does not resume // watching. The parent notices child->isDeleted is true and replaces it. // 3, 2, 1) The parent notices that child->isDirectory does not match the type of the new // file, and so deletes the child watcher explicitly. if (!children.release(childFile.get(), &child) || child->isDeleted() || child->isDirectory != childIsDirectory) { if (childIsDirectory) { child = newOwned(childFile.release(), eventManager, driver); } else { child = newOwned(childFile.release(), eventManager, driver); } child->created(); } File* key = child->file.get(); // cannot inline due to undefined evaluation order newChildren.add(key, child.release()); } // Make sure remaining children have been notified of deletion before we destroy the objects. for (ChildMap::Iterator iter(children); iter.next();) { if (!iter.value()->isDeleted()) { iter.value()->reallyDeleted(); } } // Swap in new children. children.swap(&newChildren); } void deleted() { if (file->isDirectory()) { // A new directory was created in place of the old. Reset the watch. DEBUG_INFO << "Directory replaced: " << file->canonicalName(); resetWatch(); modified(); } else { reallyDeleted(); } } // implements Watcher ------------------------------------------------------------------ void reallyDeleted() { DEBUG_INFO << "Directory deleted: " << file->canonicalName(); clearWatch(); driver->removeSourceFile(file.get()); // Delete all children. for (ChildMap::Iterator iter(children); iter.next();) { if (!iter.value()->isDeleted()) { iter.value()->reallyDeleted(); } } children.clear(); } private: ChildMap children; }; // ======================================================================================= class EkamLocks final: public Driver::ActivityObserver { public: EkamLocks(File* tmp) : mainLock("tmp/.ekam-lock", openLockfile(tmp->relative(".ekam-lock").get())), activeLock("tmp/.ekam-lock-active", openLockfile(tmp->relative(".ekam-lock-active").get())) {} bool tryTakeMainLock() { while (flock(mainLock.get(), LOCK_EX | LOCK_NB) < 0) { if (errno == EWOULDBLOCK) { return false; } else if (errno == ENOLCK) { // Filesystem doesn't support locking. fprintf(stderr, "WARNING: Filesystem doesn't support locking. Do not run two Ekam instances concurrently.\n"); noLocking = true; return true; } else if (errno != EAGAIN) { throw OsError("flock(mainLock)", errno); } } return true; } void waitForOther() { wrapSyscall("flock(activeLock)", flock, activeLock.get(), LOCK_SH); char c = 0; ssize_t n = wrapSyscall("read(activeLock)", read, activeLock, &c, 1); if (n == 1 && c == 'p') { failed = false; } else if (n == 1 && c == 'f') { failed = true; } else { throw std::logic_error("lock file contents invalid"); } wrapSyscall("flock(activeLock)", flock, activeLock.get(), LOCK_UN); } bool hasFailures() { if (running) throw std::logic_error("can't check failures while still running"); return failed; } void startingAction() override { if (!running) { running = true; if (!noLocking) flock(activeLock.get(), LOCK_EX); } } void idle(bool hasFailures) override { if (running) { running = false; wrapSyscall("write(activeLock)", write, activeLock.get(), hasFailures ? "fail" : "pass", 4); if (!noLocking) wrapSyscall("flock(activeLock, LOCK_UN)", flock, activeLock.get(), LOCK_UN); } failed = hasFailures; } private: OsHandle mainLock; OsHandle activeLock; bool running = false; bool failed = false; bool noLocking = false; static int openLockfile(File* lockfile) { auto diskfile = lockfile->getOnDisk(File::UPDATE); return wrapSyscall("open(lockfile)", open, diskfile->path().c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); } }; // ======================================================================================= void scanSourceTree(File* src, Driver* driver) { OwnedPtrVector fileQueue; { fileQueue.add(src->clone()); } while (!fileQueue.empty()) { OwnedPtr current = fileQueue.releaseBack(); if (current->isDirectory()) { OwnedPtrVector list; current->list(list.appender()); for (int i = 0; i < list.size(); i++) { fileQueue.add(list.release(i)); } } driver->addSourceFile(current.get()); } } OwnedPtr getDashboard(int maxDisplayedLogLines) { if (!isatty(STDOUT_FILENO)) { return newOwned(stdout); } // Sanity check: make sure the window size is non-zero; otherwise something // is messed up about the terminal, and we should assume it doesn't work // correctly. // // See issue #29 struct winsize windowSize; if(ioctl(STDOUT_FILENO, TIOCGWINSZ, &windowSize) < 0) { const char *msg = strerror(errno); DEBUG_WARNING << "Error querying terminal size: " << msg << "; " << "falling back to simple output."; return newOwned(stdout); } if(windowSize.ws_row == 0 || windowSize.ws_col == 0) { DEBUG_WARNING << "Terminal size looks suspicious " << "(rows = " << windowSize.ws_row << ", columns = " << windowSize.ws_col << "); " << "falling back to simple output."; return newOwned(stdout); } return newOwned(stdout, maxDisplayedLogLines); } int main(int argc, char* argv[]) { signal(SIGPIPE, SIG_IGN); int maxDisplayedLogLines = 30; const char* command = argv[0]; int maxConcurrentActions = 1; bool continuous = false; std::string networkDashboardAddress; while (true) { int opt = getopt(argc, argv, "chvj:n:l:"); if (opt == -1) break; switch (opt) { case 'v': DebugMessage::setLogLevel(DebugMessage::INFO); break; case 'j': { char* endptr; maxConcurrentActions = strtoul(optarg, &endptr, 10); if (*endptr != '\0') { fprintf(stderr, "Expected number after -j.\n"); return 1; } break; } case 'h': usage(command, stdout); return 0; case 'c': continuous = true; break; case 'n': networkDashboardAddress = optarg; break; case 'l': { char* endptr; maxDisplayedLogLines = strtoul(optarg, &endptr, 0); if (*endptr != '\0') { fprintf(stderr, "Expected number after -l.\n"); return 1; } break; } default: usage(command, stderr); return 1; } } argc -= optind; argv += optind; if (argc > 0) { fprintf(stderr, "%s: unknown argument -- %s\n", command, argv[0]); return 1; } DiskFile src("src", NULL); DiskFile tmp("tmp", NULL); DiskFile bin("bin", NULL); DiskFile lib("lib", NULL); DiskFile nodeModules("node_modules", NULL); File* installDirs[BuildContext::INSTALL_LOCATION_COUNT] = { &bin, &lib, &nodeModules }; if (!tmp.isDirectory()) { tmp.createDirectory(); } EkamLocks locks(&tmp); if (!locks.tryTakeMainLock()) { if (continuous) { fprintf(stderr, "ERROR: Ekam is already running in this directory.\n"); return 1; } else { fprintf(stderr, "Another Ekam is already running in this directory.\n" "Waiting for build to complete...\n"); locks.waitForOther(); return locks.hasFailures() ? 1 : 0; } } OwnedPtr eventManager = newPreferredEventManager(); OwnedPtr dashboard = getDashboard(maxDisplayedLogLines); if (!networkDashboardAddress.empty()) { dashboard = initNetworkDashboard(eventManager.get(), networkDashboardAddress, dashboard.release()); } Driver driver(eventManager.get(), dashboard.get(), &tmp, installDirs, maxConcurrentActions, &locks); ExtractTypeActionFactory extractTypeActionFactcory; driver.addActionFactory(&extractTypeActionFactcory); CppActionFactory cppActionFactory; driver.addActionFactory(&cppActionFactory); ExecPluginActionFactory execPluginActionFactory; driver.addActionFactory(&execPluginActionFactory); OwnedPtr rootWatcher; if (continuous) { rootWatcher = newOwned(src.clone(), eventManager.get(), &driver); rootWatcher->modified(); } else { scanSourceTree(&src, &driver); } eventManager->loop(); // For debugging purposes, check for zombie processes. int zombieCount = 0; while (true) { int status; pid_t pid = wait(&status); if (pid < 0) { if (errno == ECHILD) { // No more children. break; } else { DEBUG_ERROR << "wait: " << strerror(errno); } } else { ++zombieCount; } } if (zombieCount > 0) { DEBUG_ERROR << "There were " << zombieCount << " zombie processes after the event loop stopped."; return 1; } else { DEBUG_INFO << "No zombie processes detected. Hooray."; return locks.hasFailures() ? 1 : 0; } } } // namespace ekam int main(int argc, char* argv[]) { return ekam::main(argc, argv); } ================================================ FILE: src/ekam/ekam.ekam-manifest ================================================ ekam bin ekam-client bin ekam-langserve bin ================================================ FILE: src/ekam/initNetworkDashboardStub.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Dashboard.h" #include namespace ekam { OwnedPtr initNetworkDashboard(EventManager* eventManager, const std::string& address, OwnedPtr dashboardToWrap) { throw std::logic_error("Network dashboard support not compiled in."); } } // namespace ekam ================================================ FILE: src/ekam/langserve.capnp ================================================ @0xdf85b05eff43f858; # Cap'n Proto definitions for a subset of the Language Server Protocol, usable with capnp::JsonRpc. # Currently this is the minimum subset needed for Ekam. using Json = import "/capnp/compat/json.capnp"; $import "/capnp/c++.capnp".namespace("ekam::lsp"); interface LanguageServer { initialize @0 InitializeParams -> (capabilities :ServerCapabilities); struct InitializeParams { processId @0 :UInt32; # Technically can be null. rootPath @1 :Text; rootUri @2 :DocumentUri; initializationOptions @3 :Json.Value; capabilities @4 :ClientCapabilities; trace @5 :Text = "off"; workspaceFolders @6 :List(WorkspaceFolder); } shutdown @1 (); exit @2 () $Json.notification; didOpen @3 () $Json.notification $Json.name("textDocument/didOpen"); didClose @4 () $Json.notification $Json.name("textDocument/didClose"); # Ignoring these for now. didChange @5 (textDocument :TextDocumentIdentifier, # Actually VersionedTextDocumentIdentifier but we don't care. contentChanges :List(TextDocumentContentChangeEvent)) $Json.notification $Json.name("textDocument/didChange"); didSave @6 (textDocument :TextDocumentIdentifier) $Json.notification $Json.name("textDocument/didSave"); # TODO(someday): Why is this never sent at present? } interface LanguageClient { publishDiagnostics @0 (uri :DocumentUri, diagnostics :List(Diagnostic)) $Json.notification $Json.name("textDocument/publishDiagnostics"); } struct ClientCapabilities { # TODO(someday): workspace, textDocument, experimental } struct ServerCapabilities { textDocumentSync @0 :TextDocumentSyncOptions; # TODO(someday): more } struct TextDocumentSyncOptions { openClose @0 :Bool = false; change @1 :UInt8 = TextDocumentSyncKind.none; didSave @2 :Bool = false; } struct TextDocumentSyncKind { const none :UInt32 = 0; const full :UInt32 = 1; const incremental :UInt32 = 2; } struct WorkspaceFolder { uri @0 :Text; name @1 :Text; } using DocumentUri = Text; struct Position { line @0 :UInt32; character @1 :UInt32; } struct Range { start @0 :Position; end @1 :Position; } struct Location { uri @0 :DocumentUri; range @1 :Range; } struct Diagnostic { range @0 :Range; severity @1 :UInt32; code @2 :Text; source @3 :Text; message @4 :Text; relatedInformation @5 :List(RelatedInformation); struct RelatedInformation { location @0 :Location; message @1 :Text; } const severityError :UInt32 = 1; const severityWarning :UInt32 = 2; const severityInformation :UInt32 = 3; const severityHint :UInt64 = 4; } struct Command { title @0 :Text; command @1 :Text; arguments @2 :List(Json.Value); } struct TextDocumentContentChangeEvent { range @0 :Range; text @1 :Text; } struct TextDocumentIdentifier { uri @0 :DocumentUri; } ================================================ FILE: src/ekam/rules/compile.ekam-rule ================================================ #! /bin/sh # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu if test $# = 0; then # Ekam is querying the script. Tell it that we like C++ source files. echo trigger filetype:.cpp echo trigger filetype:.cc echo trigger filetype:.C echo trigger filetype:.cxx echo trigger filetype:.c++ echo trigger filetype:.c exit 0 fi INPUT=$1 shift # Set defaults. We use -O2 -DNDEBUG as default flags because the users that are most likely not # to specify flags are people who are just compiling someone else's code to use it, and those # people do not want debug builds. CXX=${CXX:-c++} CXXFLAGS=${CXXFLAGS:--O2 -DNDEBUG} CC=${CC:-cc} CFLAGS=${CFLAGS:--O2 -DNDEBUG} # Look up modifiers. echo findModifiers compile.ekam-flags while true; do read MODIFIER if test -z "$MODIFIER"; then break fi . "$MODIFIER" 1>&2 done case "$INPUT" in *.cpp ) MODULE_NAME=${INPUT%.cpp} ;; *.cc ) MODULE_NAME=${INPUT%.cc} ;; *.C ) MODULE_NAME=${INPUT%.C} ;; *.cxx ) MODULE_NAME=${INPUT%.cxx} ;; *.c++ ) MODULE_NAME=${INPUT%.c++} ;; intercept.c | */intercept.c ) # Hack: Skip interceptor. It screws everything up since it appears to define syscalls like # write(). # TODO(cleanup): Do a better job detecting this. exit 0 ;; *.c ) MODULE_NAME=${INPUT%.c} CXX=${CC} CXXFLAGS=${CFLAGS} ;; * ) echo "Wrong file type: $INPUT" >&2 exit 1 ;; esac echo findProvider special:ekam-interceptor read INTERCEPTOR if test "$INTERCEPTOR" = ""; then echo "error: couldn't find intercept.so." >&2 exit 1 fi # Ask Ekam where to put the output file. Actually, the compiler will make the same request again # when it runs, but we need to know the location too. OUTPUT=${MODULE_NAME}.o echo newOutput "$OUTPUT" read OUTPUT_DISK_PATH compile() { local TARGET_CXX="$1" local SUFFIX="$2" local FLAGSVAR="CXXFLAGS_$3" shift 3 local FLAGS="$(eval "echo \"\${$FLAGSVAR:-\$CXXFLAGS}\"")" # Remove -Wglobal-constructors in tests because the test framework depends on registering global # objects, and we only care about global constructors in runtime code anyway. case "$FLAGS" in *-Wglobal-constructors* ) case "$MODULE_NAME" in *-test ) FLAGS="$FLAGS -Wno-global-constructors" ;; esac esac # Compile! We LD_PRELOAD intercept.so to intercept open() and other filesystem calls and convert # them into Ekam requests. intercept.so expects file descriptors 3 and 4 to be the Ekam request # and response streams, so we remap them to stdout and stdin, respectively. We also remap stdout # itself to stderr just to make sure that if the compiler prints anything to stdout (which # normally it shouldn't), that does not get misinterpreted as an Ekam request. # # The DYLD_ vars are the Mac OSX equivalent of LD_PRELOAD. We don't bother checking which OS # we're on since the other vars will just be ignored anyway. LD_PRELOAD=$INTERCEPTOR DYLD_FORCE_FLAT_NAMESPACE= DYLD_INSERT_LIBRARIES=$INTERCEPTOR \ ${CXX_WRAPPER:-} $TARGET_CXX -I/ekam-provider/c++header $FLAGS "$@" -c "/ekam-provider/canonical/$INPUT" \ -o "${MODULE_NAME}${SUFFIX}" 3>&1 4<&0 >&2 } compile "$CXX" .o host for TARGET in ${CROSS_TARGETS:-}; do SUFFIX="$(echo "$TARGET" | tr - _)" case "$(basename "$CXX")" in *clang* ) compile "$CXX" ".$TARGET.o" "$SUFFIX" -target "$TARGET" "-isystem/usr/$TARGET/include" ;; * ) compile "$TARGET-$CXX" ".$TARGET.o" "$SUFFIX" ;; esac done # TODO(someday): Generate symbols and deps separately for each target? Currently this is aimed at # architecture cross-compiling, not OS cross-compiling, so we expect the symbols are identical. # If they are not, the linker rule needs to change to understand this, too. Of course, what we # should really do is handle separate targets using separate build steps, but that seems to # require deep changes to Ekam. # Ask Ekam where to put the symbol and deps lists. echo newOutput "${MODULE_NAME}.o.syms" read SYMFILE echo newOutput "${MODULE_NAME}.o.deps" read DEPFILE # Generate the symbol list. # Additional flags to NM may be provided via NMFLAGS environment variables. # NMFLAGS is intentionally unquoted so that multiple arguments may be passed in. # The version of NM used can be supplied via the NM environment variable. # This can be useful, for example, if building with LTO & want to use llvm-nm. # TODO: Would be nice to use nm -C here to demangle names but it doesn't appear # to be supported on OSX. "${NM:-nm}" ${NMFLAGS:-} "$OUTPUT_DISK_PATH" > $SYMFILE # Function which reads the symbol list on stdin and writes all symbols matching # the given type pattern to stdout, optionally with a prefix. readsyms() { grep '[^ ]* *['$1'] ' | sed -e 's,^[^ ]* *. \(.*\)$,'"${2:-}"'\1,g' } # Construct the deps file by listing all undefined symbols. readsyms U < $SYMFILE > $DEPFILE # ======================================================================================== # Detect gtest-based tests and test support while we're here. # TODO(kenton): Probably should be a separate rule. IS_TEST=no case $OUTPUT in */gtest_main.o ) echo provide "$OUTPUT_DISK_PATH" gtest:main ;; */kj/test.o | kj/test.o ) echo provide "$OUTPUT_DISK_PATH" kjtest:main ;; *_test.o | *_unittest.o | *_regtest.o | *-test.o ) # Is this a gtest test that needs to link against gtest_main? if grep -q 7testing8internal23MakeAndRegisterTestInfo $DEPFILE && \ ! egrep -q '[^U] _?main$' $SYMFILE; then echo provide "$OUTPUT_DISK_PATH" gtest:test IS_TEST=yes fi # Is this a KJ test that needs to link against kj/test.o? if grep -q N2kj8TestCaseC $DEPFILE; then echo provide "$OUTPUT_DISK_PATH" kjtest:test IS_TEST=yes fi ;; * ) # Node v0.10 exports a symbol like so: # NODE_MODULE_EXPORT node::node_module_struct modname ## _module = ... # # The HandleScope constructor is v8::HandleScope::HandleScope(). if egrep -q ' D [a-z0-9]+_module' $SYMFILE && \ grep -q _ZN2v811HandleScopeC1Ev $DEPFILE; then echo provide "$OUTPUT_DISK_PATH" nodejs:module fi # Node v4 exports a symbol like so: # static node::node_module _module = ... # # Symbols may bear an "L" prefix to indicate constness, but not all compiler versions # mangle this way, so we tolerate the presence or absence of the qualifier. # # The HandleScope constructor is v8::HandleScope::HandleScope(v8::Isolate*) if grep -q -E ' d _ZL?7_module' $SYMFILE && \ grep -q _ZN2v811HandleScopeC1EPNS_7IsolateE $DEPFILE; then echo provide "$OUTPUT_DISK_PATH" nodejs:module fi ;; esac if [ "$IS_TEST" = no ]; then # Tell Ekam about the symbols provided by this file. But not for tests, because we don't want # other things to accidentally link against tests. readsyms ABCDGRSTV "provide $OUTPUT_DISK_PATH c++symbol:" < $SYMFILE fi ================================================ FILE: src/ekam/rules/include.ekam-rule ================================================ #! /bin/sh # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu if test $# = 0; then # Ekam is querying the script. Tell it that we care about headers. echo trigger filetype:.h echo trigger 'directory:*' echo silent exit 0 fi INPUT=$1 INCLUDE_NAME=$INPUT INCLUDE_NAME=${INCLUDE_NAME##*/src/} INCLUDE_NAME=${INCLUDE_NAME#src/} INCLUDE_NAME=${INCLUDE_NAME##*/include/} INCLUDE_NAME=${INCLUDE_NAME#include/} echo provide "$INPUT" "c++header:$INCLUDE_NAME" # HACK: gtest likes to include things from its top-level directory. # TODO: Come up with a more general way for dealing with this. INCLUDE_NAME=${INPUT##*/gtest/} INCLUDE_NAME=${INCLUDE_NAME#gtest/} if test "$INCLUDE_NAME" != "$INPUT"; then echo provide "$INPUT" "c++header:$INCLUDE_NAME" fi ================================================ FILE: src/ekam/rules/install.ekam-rule ================================================ #! /bin/sh # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu if test $# = 0; then # Ekam is querying the script. Tell it that we like C++ source files. echo trigger filetype:.ekam-manifest exit 0 fi INPUT=$1 echo findProvider file:$INPUT read INPUT_DISK_PATH exec 3<$INPUT_DISK_PATH DIRNAME=$(dirname $INPUT) while read FILE TARGET <&3; do FILE=$DIRNAME/$FILE echo findInput $FILE read FILE_DISK_PATH if test "$FILE_DISK_PATH" = ""; then echo "$FILE: not found" >&2 exit 1 fi case $TARGET in */* ) echo install $FILE_DISK_PATH $TARGET ;; * ) echo install $FILE_DISK_PATH $TARGET/$(basename $FILE) ;; esac done ================================================ FILE: src/ekam/rules/intercept.c ================================================ /* Ekam Build System * Author: Kenton Varda (kenton@sandstorm.io) * Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define _GNU_SOURCE #define _DARWIN_NO_64_BIT_INODE /* See stat madness, below. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if __linux__ #include #include #include #include #endif static const int EKAM_DEBUG = 0; #define likely(x) __builtin_expect((x),1) typedef int open_t(const char * pathname, int flags, ...); void __attribute__((constructor)) start_interceptor() { if (EKAM_DEBUG) { static open_t* real_open; if (real_open == NULL) { real_open = (open_t*) dlsym(RTLD_NEXT, "open"); assert(real_open != NULL); } int fd = real_open("/proc/self/cmdline", O_RDONLY); char buffer[1024]; int n = read(fd, buffer, 1024); close(fd); write(STDERR_FILENO, buffer, n); write(STDERR_FILENO, "\n", 1); } #if __linux__ /* Block the statx syscall which node 12 uses which we're unable to intercept because it doesn't * actually have a glibc wrapper, ugh. */ #if 0 /* source of the seccomp filter below */ ld [0] /* offsetof(struct seccomp_data, nr) */ jeq #332, nosys /* __NR_statx */ jmp good nosys: ret #0x00050026 /* SECCOMP_RET_ERRNO | ENOSYS */ good: ret #0x7fff0000 /* SECCOMP_RET_ALLOW */ #endif static struct sock_filter filter[] = { { 0x20, 0, 0, 0000000000 }, { 0x15, 1, 0, 0x0000014c }, { 0x05, 0, 0, 0x00000001 }, { 0x06, 0, 0, 0x00050026 }, { 0x06, 0, 0, 0x7fff0000 }, }; static struct sock_fprog prog = { sizeof(filter) / sizeof(filter[0]), filter }; prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); syscall(__NR_seccomp, SECCOMP_SET_MODE_FILTER, 0, &prog); #endif } /****************************************************************************************/ /* Bootstrap pthreads without linking agaist libpthread. */ typedef int pthread_once_func(pthread_once_t*, void (*)(void)); typedef int pthread_mutex_lock_func(pthread_mutex_t*); typedef int pthread_mutex_unlock_func(pthread_mutex_t*); static pthread_once_func* dynamic_pthread_once = NULL; static pthread_mutex_lock_func* dynamic_pthread_mutex_lock = NULL; static pthread_mutex_unlock_func* dynamic_pthread_mutex_unlock = NULL; int fake_pthread_once(pthread_once_t* once_control, void (*init_func)(void)) { static pthread_mutex_t fake_once_mutex = PTHREAD_MUTEX_INITIALIZER; static int ONCE_INPROGRESS = 1; static int ONCE_DONE = 2; int initialized = __atomic_load_n(once_control, __ATOMIC_ACQUIRE); if (likely(initialized & ONCE_DONE)) return 0; // TODO(soon): Remove the dynamic_pthread_mutex usage once dynamic_pthread_once isn't forced to // unconditionally use fake_pthread_once. See init_pthreads. // The real implementation is typically more complex to try to make sure that concurrent // initialization of different pthread_once doesn't have a false dependency on a global mutex. // However, we only have a single such usage here & that would be overkill to try to implement on // MacOS and Linux. Use a single mutex anyway because it would have largely the same effect. It // also has additional complexities to handle things like forking and that too isn't a concern // here. dynamic_pthread_mutex_lock(&fake_once_mutex); pthread_once_t expected = PTHREAD_ONCE_INIT; bool updated =__atomic_compare_exchange_n(once_control, &expected, ONCE_INPROGRESS, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); if (likely(!updated)) { assert(__atomic_load_n(&expected, __ATOMIC_RELAXED) == ONCE_DONE); dynamic_pthread_mutex_unlock(&fake_once_mutex); return 0; } init_func(); __atomic_store_n(once_control, ONCE_DONE, __ATOMIC_RELEASE); dynamic_pthread_mutex_unlock(&fake_once_mutex); return 0; } int fake_pthread_mutex_lock(pthread_mutex_t* mutex) { return 0; } int fake_pthread_mutex_unlock(pthread_mutex_t* mutex) { return 0; } void init_pthreads() { if (dynamic_pthread_once == NULL) { dynamic_pthread_once = (pthread_once_func*) dlsym(RTLD_DEFAULT, "pthread_once"); } /* TODO: For some reason the pthread_once returned by dlsym() doesn't do anything? */ // Update 7/13/2021 - on Arch Linux pthread_once seems to work fine. Not tested on Buster so // unclear if this was a glibc bug that had been fixed at some point or there's something subtle // about when this breaks. There's an assert added in init_streams that ensures the functor // invoked. if (dynamic_pthread_once == NULL || 1) { dynamic_pthread_once = &fake_pthread_once; } if (dynamic_pthread_mutex_lock == NULL) { dynamic_pthread_mutex_lock = (pthread_mutex_lock_func*) dlsym(RTLD_DEFAULT, "pthread_mutex_lock"); } if (dynamic_pthread_mutex_lock == NULL) { dynamic_pthread_mutex_lock = &fake_pthread_mutex_lock; } if (dynamic_pthread_mutex_unlock == NULL) { dynamic_pthread_mutex_unlock = (pthread_mutex_unlock_func*) dlsym(RTLD_DEFAULT, "pthread_mutex_unlock"); } if (dynamic_pthread_mutex_lock == NULL) { dynamic_pthread_mutex_lock = &fake_pthread_mutex_unlock; } } /****************************************************************************************/ #define EKAM_CALL_FILENO 3 #define EKAM_RETURN_FILENO 4 static FILE* ekam_call_stream; static FILE* ekam_return_stream; static char current_dir[PATH_MAX + 1]; static pthread_once_t init_once_control = PTHREAD_ONCE_INIT; typedef ssize_t readlink_t(const char *path, char *buf, size_t bufsiz); static readlink_t* real_readlink = NULL; static bool bypass_interceptor = false; static void init_bypass_interceptor() { static const char* sanitizer_env_vars[] = { "ASAN_SYMBOLIZER_PATH", "MSAN_SYMBOLIZER_PATH", "LLVM_SYMBOLIZER_PATH", NULL, }; char real_exe[PATH_MAX + 1]; bool real_exe_initialized = false; assert(real_readlink != NULL); for (size_t i = 0; sanitizer_env_vars[i]; i++) { const char* sanitizer_path = getenv(sanitizer_env_vars[i]); if (sanitizer_path != NULL) { if (!real_exe_initialized) { ssize_t result = real_readlink("/proc/self/exe", real_exe, PATH_MAX); if (result == -1) { fprintf(stderr, "Failed to readlink(/proc/self/exe): %s (%d)\n", strerror(errno), errno); abort(); } real_exe_initialized = true; } if (strcmp(real_exe, sanitizer_path) == 0) { bypass_interceptor = true; break; } } } } static void init_streams_once() { static bool initialized = false; if (__atomic_exchange_n(&initialized, true, __ATOMIC_RELEASE)) { fprintf(stderr, "pthread_once is broken\n"); abort(); } assert(ekam_call_stream == NULL); assert(ekam_return_stream == NULL); assert(real_readlink == NULL); real_readlink = (readlink_t*) dlsym(RTLD_NEXT, "readlink"); assert(real_readlink != NULL); init_bypass_interceptor(); if (bypass_interceptor) { /* Bypass any further initialization as it will fail. */ return; } ekam_call_stream = fdopen(EKAM_CALL_FILENO, "w"); if (ekam_call_stream == NULL) { fprintf(stderr, "fdopen(EKAM_CALL_FILENO): error %d\n", errno); abort(); } ekam_return_stream = fdopen(EKAM_RETURN_FILENO, "r"); if (ekam_return_stream == NULL) { fprintf(stderr, "fdopen(EKAM_CALL_FILENO): error %d\n", errno); abort(); } if (getcwd(current_dir, PATH_MAX) == NULL) { fprintf(stderr, "getcwd(): error %d\n", errno); abort(); } strcat(current_dir, "/"); } static void init_streams() { init_pthreads(); dynamic_pthread_once(&init_once_control, &init_streams_once); // This assert makes sure the pthread_once call actually invokes the initializer. assert(bypass_interceptor || (ekam_call_stream != NULL && ekam_return_stream != NULL)); } /****************************************************************************************/ typedef enum usage { READ, WRITE } usage_t; static const char TAG_PROVIDER_PREFIX[] = "/ekam-provider/"; static const char TMP[] = "/tmp"; static const char VAR_TMP[] = "/var/tmp"; static const char TMP_PREFIX[] = "/tmp/"; static const char VAR_TMP_PREFIX[] = "/var/tmp/"; static const char PROC_PREFIX[] = "/proc/"; static usage_t last_usage = READ; static char last_path[PATH_MAX] = ""; static char last_result[PATH_MAX] = ""; pthread_mutex_t cache_mutex = PTHREAD_MUTEX_INITIALIZER; static void cache_result(const char* input, const char* output, usage_t usage) { dynamic_pthread_mutex_lock(&cache_mutex); last_usage = usage; strcpy(last_path, input); strcpy(last_result, output); dynamic_pthread_mutex_unlock(&cache_mutex); } static int get_cached_result(const char* pathname, char* buffer, usage_t usage) { int result = 0; dynamic_pthread_mutex_lock(&cache_mutex); if (usage == last_usage && strcmp(pathname, last_path) == 0) { strcpy(buffer, last_result); result = 1; } dynamic_pthread_mutex_unlock(&cache_mutex); return result; } static void canonicalizePath(char* path) { /* Preconditions: * - path has already been determined to be relative, perhaps because the pointer actually points * into the middle of some larger path string, in which case it must point to the character * immediately after a '/'. */ /* Invariants: * - src points to the beginning of a path component. * - dst points to the location where the path component should end up, if it is not special. * - src == path or src[-1] == '/'. * - dst == path or dst[-1] == '/'. */ char* src = path; char* dst = path; char* locked = dst; /* dst cannot backtrack past this */ char* partEnd; int hasMore; for (;;) { while (*src == '/') { /* Skip duplicate slash. */ ++src; } partEnd = strchr(src, '/'); hasMore = partEnd != NULL; if (hasMore) { *partEnd = '\0'; } else { partEnd = src + strlen(src); } if (strcmp(src, ".") == 0) { /* Skip it. */ } else if (strcmp(src, "..") == 0) { if (dst > locked) { /* Backtrack over last path component. */ --dst; while (dst > locked && dst[-1] != '/') --dst; } else { locked += 3; goto copy; } } else { /* Copy if needed. */ copy: if (dst < src) { memmove(dst, src, partEnd - src); dst += partEnd - src; } else { dst = partEnd; } *dst++ = '/'; } if (hasMore) { src = partEnd + 1; } else { /* Oops, we have to remove the trailing '/'. */ if (dst == path) { /* Oops, there is no trailing '/'. We have to return ".". */ strcpy(path, "."); } else { /* Remove the trailing '/'. Note that this means that opening the file will work even * if it is not a directory, where normally it should fail on non-directories when a * trailing '/' is present. If this is a problem, we need to add some sort of special * handling for this case where we stat() it separately to check if it is a directory, * because Ekam findInput will not accept a trailing '/'. */ --dst; *dst = '\0'; } return; } } } static bool path_has_prefix(const char* pathname, const char* prefix, size_t prefix_length) { // Returns true if pathname is a child within prefix or if it matches prefix directly. Prefix must // always end with a trailing '/'. assert(prefix_length > 1); assert(prefix_length <= PATH_MAX); assert(prefix[prefix_length - 1] == '/'); if (strncmp(pathname, prefix, prefix_length - 1) == 0) { return pathname[prefix_length - 1] == '\0' || pathname[prefix_length - 1] == '/'; } return false; } static bool bypass_remap(const char *pathname) { int debug = EKAM_DEBUG; if (pathname[0] != '/') { // Only absolute paths are allowed to bypass the remap. return false; } const char* bypass_dirs = getenv("EKAM_REMAP_BYPASS_DIRS"); if (bypass_dirs == NULL) { return false; } bool found = false; while (bypass_dirs && *bypass_dirs) { char* separator_location = strchr(bypass_dirs, ':'); size_t bypass_dir_length; if (separator_location) { bypass_dir_length = separator_location - bypass_dirs; } else { bypass_dir_length = strlen(bypass_dirs); } if (bypass_dirs[0] != '/') { fprintf(stderr, "Bypass directory '%s' isn't an absolute path. Skipping.\n", bypass_dirs); } else if (bypass_dirs[bypass_dir_length - 1] != '/') { fprintf(stderr, "Bypass directory '%s' doesn't end in trailing '/'. Skipping.\n", bypass_dirs); } else if (path_has_prefix(pathname, bypass_dirs, bypass_dir_length)) { if (debug) { fprintf(stderr, "Bypass found for %s\n", pathname); } found = true; break; } if (separator_location) { bypass_dirs = separator_location + 1; } else { bypass_dirs = NULL; } } return found; } static bool is_temporary_dir(const char *pathname) { size_t cwd_len = strlen(current_dir) - 1; if (strncmp(pathname, current_dir, cwd_len) == 0 && (pathname[cwd_len] == '/' || pathname[cwd_len] == '\0')) { // Somewhere under our working directory. If our working directory happens // to be inside one of the temporary dirs, then we need to be careful to // *not* treat this as a temporary file, otherwise this will apply to basicaly // everything, and strange things will happen. return false; } // TODO(soon): Simplify the logic to use `path_has_prefix`. if (strcmp(pathname, TMP) == 0 || strcmp(pathname, VAR_TMP) == 0 || strncmp(pathname, TMP_PREFIX, strlen(TMP_PREFIX)) == 0 || strncmp(pathname, VAR_TMP_PREFIX, strlen(VAR_TMP_PREFIX)) == 0 || strncmp(pathname, PROC_PREFIX, strlen(PROC_PREFIX)) == 0) { return true; } #if defined(__linux__) #define SYSTEMD_TMPDIR_PREFIX "/run/user/" if (path_has_prefix(pathname, SYSTEMD_TMPDIR_PREFIX, strlen(SYSTEMD_TMPDIR_PREFIX))) { return true; } #else (void) pathname; #endif return false; } static const char* remap_file(const char* syscall_name, const char* pathname, char* buffer, usage_t usage) { char* pos; int debug = EKAM_DEBUG; /* Ad-hoc debugging can be accomplished by setting debug = 1 when a particular file pattern * is matched. */ if (debug) { fprintf(stderr, "remap for %s (%s): %s\n", syscall_name, (usage == READ ? "read" : "write"), pathname); } init_streams(); if (bypass_interceptor) { if (debug) fprintf(stderr, "Bypassing interceptor for %s on %s\n", syscall_name, pathname); return pathname; } if (strlen(pathname) >= PATH_MAX) { /* Too long. */ if (debug) fprintf(stderr, " name too long\n"); errno = ENAMETOOLONG; return NULL; } if (get_cached_result(pathname, buffer, usage)) { if (debug) fprintf(stderr, " cached: %s\n", buffer); return buffer; } flockfile(ekam_call_stream); if (strncmp(pathname, TAG_PROVIDER_PREFIX, strlen(TAG_PROVIDER_PREFIX)) == 0) { /* A tag reference. Construct the tag name in |buffer|. */ strcpy(buffer, pathname + strlen(TAG_PROVIDER_PREFIX)); if (usage == READ) { /* Change first slash to a colon to form a tag. E.g. "header/foo.h" becomes * "header:foo.h". */ pos = strchr(buffer, '/'); if (pos == NULL) { /* This appears to be a tag type without a name, so it should look like a directory. * We can use the current directory. TODO: Return some fake empty directory instead. */ funlockfile(ekam_call_stream); strcpy(buffer, "."); if (debug) fprintf(stderr, " is directory\n"); return buffer; } *pos = ':'; canonicalizePath(pos + 1); if (strcmp(buffer, "canonical:.") == 0) { /* HACK: Don't try to remap top directory. */ funlockfile(ekam_call_stream); if (debug) fprintf(stderr, " current directory\n"); return "src"; } } /* Ask ekam to remap the file name. */ fputs(usage == READ ? "findProvider " : "newProvider ", ekam_call_stream); fputs(buffer, ekam_call_stream); fputs("\n", ekam_call_stream); } else if (is_temporary_dir(pathname)) { /* Temp file or /proc. Ignore. */ funlockfile(ekam_call_stream); if (debug) fprintf(stderr, " temp file: %s\n", pathname); return pathname; } else if (bypass_remap(pathname)) { funlockfile(ekam_call_stream); if (debug) fprintf(stderr, " bypassed file: %s\n", pathname); return pathname; } else { if (strncmp(pathname, current_dir, strlen(current_dir)) == 0 && strncmp(pathname + strlen(current_dir), "deps/", 5) != 0) { /* The app is trying to open files in the current directory by absolute path. Treat it * exactly as if it had used a relative path. We make a special exception for the directory * `deps`, to allow e.g. executing binaries found there without mapping them into the * common source tree. */ pathname = pathname + strlen(current_dir); } else if (pathname[0] == '/' || strncmp(pathname, "deps/", 5) == 0) { /* Absolute path or under `deps`. Note the access but don't remap. */ if (usage == WRITE) { /* Cannot write to absolute paths. */ funlockfile(ekam_call_stream); errno = EACCES; if (debug) fprintf(stderr, " absolute path, can't write\n"); return NULL; } fputs("noteInput ", ekam_call_stream); fputs(pathname, ekam_call_stream); fputs("\n", ekam_call_stream); fflush(ekam_call_stream); if (ferror_unlocked(ekam_call_stream)) { funlockfile(ekam_call_stream); fprintf(stderr, "error: Ekam call stream broken.\n"); abort(); } cache_result(pathname, pathname, usage); funlockfile(ekam_call_stream); if (debug) fprintf(stderr, " absolute path: %s\n", pathname); return pathname; } /* Path in current directory. */ strcpy(buffer, pathname); canonicalizePath(buffer); if (strcmp(buffer, ".") == 0) { /* HACK: Don't try to remap current directory. */ funlockfile(ekam_call_stream); if (debug) fprintf(stderr, " current directory\n"); return "."; } else { /* Ask ekam to remap the file name. */ fputs(usage == READ ? "findInput " : "newOutput ", ekam_call_stream); fputs(buffer, ekam_call_stream); fputs("\n", ekam_call_stream); } } fflush(ekam_call_stream); if (ferror_unlocked(ekam_call_stream)) { funlockfile(ekam_call_stream); fprintf(stderr, "error: Ekam call stream broken.\n"); abort(); } /* Carefully lock the return stream then unlock the call stream, so that we know that * responses will be received in the correct order. */ flockfile(ekam_return_stream); funlockfile(ekam_call_stream); /* Read response from Ekam. */ if (fgets(buffer, PATH_MAX, ekam_return_stream) == NULL) { funlockfile(ekam_return_stream); fprintf(stderr, "error: Ekam return stream broken.\n"); abort(); } /* Done reading. */ funlockfile(ekam_return_stream); /* Remove the trailing newline. */ pos = strchr(buffer, '\n'); if (pos == NULL) { fprintf(stderr, "error: Path returned from Ekam was too long.\n"); abort(); } *pos = '\0'; if (*buffer == '\0') { /* Not found. */ errno = ENOENT; if (debug) fprintf(stderr, " ekam says no such file\n"); return NULL; } cache_result(pathname, buffer, usage); if (debug) fprintf(stderr, " remapped to: %s\n", buffer); return buffer; } /****************************************************************************************/ #define WRAP(RETURNTYPE, NAME, PARAMTYPES, PARAMS, USAGE, ERROR_RESULT) \ typedef RETURNTYPE NAME##_t PARAMTYPES; \ RETURNTYPE NAME PARAMTYPES { \ static NAME##_t* real_##NAME = NULL; \ char buffer[PATH_MAX]; \ \ if (real_##NAME == NULL) { \ real_##NAME = (NAME##_t*) dlsym(RTLD_NEXT, #NAME); \ assert(real_##NAME != NULL); \ } \ \ path = remap_file(#NAME, path, buffer, USAGE); \ if (path == NULL) return ERROR_RESULT; \ return real_##NAME PARAMS; \ } \ RETURNTYPE _##NAME PARAMTYPES { \ return NAME PARAMS; \ } WRAP(int, chdir, (const char* path), (path), READ, -1) WRAP(int, chmod, (const char* path, mode_t mode), (path, mode), WRITE, -1) WRAP(int, lchmod, (const char* path, mode_t mode), (path, mode), WRITE, -1) WRAP(int, unlink, (const char* path), (path), WRITE, -1) WRAP(int, link, (const char* target, const char* path), (target, path), WRITE, -1) WRAP(int, symlink, (const char* target, const char* path), (target, path), WRITE, -1) WRAP(int, execve, (const char* path, char* const argv[], char* const envp[]), (path, argv, envp), READ, -1); #ifdef __APPLE__ WRAP(int, stat, (const char* path, struct stat* sb), (path, sb), READ, -1) WRAP(int, lstat, (const char* path, struct stat* sb), (path, sb), READ, -1) /* OSX defines an alternate version of stat with 64-bit inodes. */ WRAP(int, stat64, (const char* path, struct stat64* sb), (path, sb), READ, -1) /* In some crazy attempt to transition the regular "stat" call to use 64-bit inodes, Apple * resorted to some sort of linker magic in which calls to stat() in newly-compiled code * actually go to _stat$INODE64(), which appears to be identical to stat64(). We disabled * this by defining _DARWIN_NO_64_BIT_INODE, above, but we need to intercept all versions * of stat, including the $INODE64 version. Let's avoid any dependency on stat64, though, * since it is "deprecated". So, we just make the stat buf pointer opaque. */ typedef int stat_inode64_t(const char* path, void* sb); int stat_inode64(const char* path, void* sb) __asm("_stat$INODE64"); int stat_inode64(const char* path, void* sb) { static stat_inode64_t* real_stat_inode64 = NULL; char buffer[PATH_MAX]; if (real_stat_inode64 == NULL) { real_stat_inode64 = (stat_inode64_t*) dlsym(RTLD_NEXT, "stat$INODE64"); assert(real_stat_inode64 != NULL); } path = remap_file("_stat$INODE64", path, buffer, READ); if (path == NULL) return -1; return real_stat_inode64(path, sb); } /* Cannot intercept intra-libc function calls on OSX, so we must intercept fopen() as well. */ WRAP(FILE*, fopen, (const char* path, const char* mode), (path, mode), mode[0] == 'w' || mode[0] == 'a' ? WRITE : READ, NULL) WRAP(FILE*, freopen, (const char* path, const char* mode, FILE* file), (path, mode, file), mode[0] == 'w' || mode[0] == 'a' ? WRITE : READ, NULL) /* And remove(). */ WRAP(int, remove, (const char* path), (path), WRITE, -1) /* Called by access(), below. */ static int direct_stat(const char* path, struct stat* sb) { static stat_t* real_stat = NULL; if (real_stat == NULL) { real_stat = (stat_t*) dlsym(RTLD_NEXT, "stat"); assert(real_stat != NULL); } return real_stat(path, sb); } #elif defined(__linux__) WRAP(int, __xstat, (int ver, const char* path, struct stat* sb), (ver, path, sb), READ, -1) //WRAP(int, __lxstat, (int ver, const char* path, struct stat* sb), (ver, path, sb), READ, -1) WRAP(int, __xstat64, (int ver, const char* path, struct stat64* sb), (ver, path, sb), READ, -1) //WRAP(int, __lxstat64, (int ver, const char* path, struct stat64* sb), (ver, path, sb), READ, -1) /* Within the source tree, we make all symbolic links look like hard links. Otherwise, if a * symlink in the source tree pointed outside of it, and if a tool decided to read back that link * and follow it manually, we'd no longer know that it was accessing something that was meant to * be treated as source. * * To implement this, we redirect lstat -> stat when the file is in the current directory. */ typedef int __lxstat_t (int ver, const char* path, struct stat* sb); int __lxstat (int ver, const char* path, struct stat* sb) { static __lxstat_t* real___lxstat = NULL; static __xstat_t* real___xstat = NULL; char buffer[PATH_MAX]; if (real___lxstat == NULL) { real___lxstat = (__lxstat_t*) dlsym(RTLD_NEXT, "__lxstat"); assert(real___lxstat != NULL); } if (real___xstat == NULL) { real___xstat = (__lxstat_t*) dlsym(RTLD_NEXT, "__xstat"); assert(real___xstat != NULL); } path = remap_file("__lxstat", path, buffer, READ); if (path == NULL) return -1; if (path[0] == '/') { return real___lxstat(ver, path, sb); } else { return real___xstat(ver, path, sb); } } int ___lxstat (int ver, const char* path, struct stat* sb) { return __lxstat (ver, path, sb); } typedef int __lxstat64_t (int ver, const char* path, struct stat64* sb); int __lxstat64 (int ver, const char* path, struct stat64* sb) { static __lxstat64_t* real___lxstat64 = NULL; static __xstat64_t* real___xstat64 = NULL; char buffer[PATH_MAX]; if (real___lxstat64 == NULL) { real___lxstat64 = (__lxstat64_t*) dlsym(RTLD_NEXT, "__lxstat64"); assert(real___lxstat64 != NULL); } if (real___xstat64 == NULL) { real___xstat64 = (__lxstat64_t*) dlsym(RTLD_NEXT, "__xstat64"); assert(real___xstat64 != NULL); } path = remap_file("__lxstat64", path, buffer, READ); if (path == NULL) return -1; if (path[0] == '/') { return real___lxstat64(ver, path, sb); } else { return real___xstat64(ver, path, sb); } } int ___lxstat64 (int ver, const char* path, struct stat64* sb) { return __lxstat64 (ver, path, sb); } #ifndef _STAT_VER /* glibc 2.33+ no longer defines `stat()` as an inline wrapper in the header, but instead exports * it as a regular symbol. So, we need to intercept it. And, we need to give `lstat()` special * treatment just like `__lxstat()` above. */ WRAP(int, stat, (const char* path, struct stat* sb), (path, sb), READ, -1) WRAP(int, stat64, (const char* path, struct stat64* sb), (path, sb), READ, -1) //WRAP(int, lstat, (const char* path, struct stat* sb), (path, sb), READ, -1) typedef int lstat_t (const char* path, struct stat* sb); int lstat(const char* path, struct stat* sb) { static lstat_t* real_lstat = NULL; static stat_t* real_stat = NULL; char buffer[PATH_MAX]; if (real_lstat == NULL) { real_lstat = (lstat_t*) dlsym(RTLD_NEXT, "lstat"); assert(real_lstat != NULL); } if (real_stat == NULL) { real_stat = (lstat_t*) dlsym(RTLD_NEXT, "stat"); assert(real_stat != NULL); } path = remap_file("lstat", path, buffer, READ); if (path == NULL) return -1; if (path[0] == '/') { return real_lstat(path, sb); } else { return real_stat(path, sb); } } int _lstat(const char* path, struct stat* sb) { return lstat(path, sb); } #endif /* defined(_STAT_VER) */ /* Cannot intercept intra-libc function calls on Linux, so we must intercept fopen() as well. */ WRAP(FILE*, fopen, (const char* path, const char* mode), (path, mode), mode[0] == 'w' || mode[0] == 'a' ? WRITE : READ, NULL) WRAP(FILE*, fopen64, (const char* path, const char* mode), (path, mode), mode[0] == 'w' || mode[0] == 'a' ? WRITE : READ, NULL) WRAP(FILE*, freopen, (const char* path, const char* mode, FILE* file), (path, mode, file), mode[0] == 'w' || mode[0] == 'a' ? WRITE : READ, NULL) WRAP(FILE*, freopen64, (const char* path, const char* mode, FILE* file), (path, mode, file), mode[0] == 'w' || mode[0] == 'a' ? WRITE : READ, NULL) /* And remove(). */ WRAP(int, remove, (const char* path), (path), WRITE, -1) /* And dlopen(). */ WRAP(void*, dlopen, (const char* path, int flag), (path, flag), READ, NULL) /* And exec*(). The PATH-checking versions need special handling. */ WRAP(int, execv, (const char* path, char* const argv[]), (path, argv), READ, -1) typedef int execvpe_t(const char* path, char* const argv[], char* const envp[]); int execvpe(const char* path, char* const argv[], char* const envp[]) { static execvpe_t* real_execvpe = NULL; char buffer[PATH_MAX]; if (real_execvpe == NULL) { real_execvpe = (execvpe_t*) dlsym(RTLD_NEXT, "execvpe"); assert(real_execvpe != NULL); } /* If the path does not contain a '/', PATH resolution will be done, so we don't want to * remap. */ if (strchr(path, '/') != NULL) { path = remap_file("execvpe", path, buffer, READ); if (path == NULL) return -1; } return real_execvpe(path, argv, envp); } int _execvpe (const char* path, char* const argv[], char* const envp[]) { return execvpe(path, argv, envp); } typedef int execvp_t(const char* path, char* const argv[]); int execvp(const char* path, char* const argv[]) { static execvp_t* real_execvp = NULL; char buffer[PATH_MAX]; if (real_execvp == NULL) { real_execvp = (execvp_t*) dlsym(RTLD_NEXT, "execvp"); assert(real_execvp != NULL); } /* If the path does not contain a '/', PATH resolution will be done, so we don't want to * remap. */ if (strchr(path, '/') != NULL) { path = remap_file("execvp", path, buffer, READ); if (path == NULL) return -1; } return real_execvp(path, argv); } int _execvp (const char* path, char* const argv[]) { return execvp(path, argv); } /* Called by access(), below. */ static int direct_stat(const char* path, struct stat* sb) { #ifdef _STAT_VER // glibc <2.33 defines `stat()` as an inline wrapper in the header file, which calls __xstat(). static __xstat_t* real_xstat = NULL; if (real_xstat == NULL) { real_xstat = (__xstat_t*) dlsym(RTLD_NEXT, "__xstat"); assert(real_xstat != NULL); } return real_xstat(_STAT_VER, path, sb); #else /* defined(_STAT_VER) */ // glibc 2.33+ no longer defines stat() as an inline wrapper in the header, but instead exports // it as a regular symbol. static stat_t* real_stat = NULL; if (real_stat == NULL) { real_stat = (stat_t*) dlsym(RTLD_NEXT, "stat"); assert(real_stat != NULL); } return real_stat(path, sb); #endif /* defined(_STAT_VER), #else */ } ssize_t readlink(const char *path, char *buf, size_t bufsiz) { char buffer[PATH_MAX]; path = remap_file("readlink", path, buffer, READ); if (path == NULL) return -1; if (path[0] != '/') return EINVAL; // no links in source directory return real_readlink(path, buf, bufsiz); } #else WRAP(int, stat, (const char* path, struct stat* sb), (path, sb), READ, -1) WRAP(int, lstat, (const char* path, struct stat* sb), (path, sb), READ, -1) /* Called by access(), below. */ static int direct_stat(const char* path, struct stat* sb) { static stat_t* real_stat = NULL; if (real_stat == NULL) { real_stat = (stat_t*) dlsym(RTLD_NEXT, "stat"); assert(real_stat != NULL); } return real_stat(path, sb); } #endif /* platform */ static int intercepted_open(const char * pathname, int flags, va_list args) { static open_t* real_open; char buffer[PATH_MAX]; const char* remapped; #if __linux__ // The new TCMalloc will attempt to open various files in /sys/ to read information about the CPU // (this is implemented within Abseil). If we actually do the dlsym below, we'll end up causing // a deadlock within TCMalloc because `dlsym` will cause TCMalloc's constructor to try to // initialize again, but the open call is being done within that. I thought perhaps this could be // resolved upstream (https://github.com/google/tcmalloc/issues/78) but after looking into it // more, there's a fundamental issue with calls to open these files causing trying to read these // files again. Since Abseil has "call_once" semantics to read the files in /sys/, even if you // tried to pre-cache it in TCMalloc, Abseil would end up deadlocking when you reentrantly tried // to initialize the CPU frequency. // Technically this only needs to be done if `real_open` isn't resolved, but my thought was that // there's no real use-case where Ekam would want to intercept /sys/ & thus this simplifies me // having to do more thorough testing. if (strncmp("/sys/", pathname, strlen("/sys/")) == 0) { mode_t mode = 0; if (flags & O_CREAT) { mode = va_arg(args, int); } if (EKAM_DEBUG) { fprintf(stderr, "TCMalloc workaround. Bypassing Ekam for %s\n", pathname); } return syscall(SYS_openat, AT_FDCWD, pathname, flags, mode); } #endif if (real_open == NULL) { real_open = (open_t*) dlsym(RTLD_NEXT, "open"); assert(real_open != NULL); } remapped = remap_file("open", pathname, buffer, (flags & O_ACCMODE) == O_RDONLY ? READ : WRITE); if (remapped == NULL) return -1; if (flags & O_CREAT) { mode_t mode = va_arg(args, int); return real_open(remapped, flags, mode); } else { return real_open(remapped, flags, 0); } } int open(const char * pathname, int flags, ...) { va_list args; va_start(args, flags); return intercepted_open(pathname, flags, args); } int _open(const char * pathname, int flags, ...) { va_list args; va_start(args, flags); return intercepted_open(pathname, flags, args); } static int intercepted_open64(const char * pathname, int flags, va_list args) { static open_t* real_open64; char buffer[PATH_MAX]; const char* remapped; if (real_open64 == NULL) { real_open64 = (open_t*) dlsym(RTLD_NEXT, "open64"); assert(real_open64 != NULL); } remapped = remap_file("open64", pathname, buffer, (flags & O_ACCMODE) == O_RDONLY ? READ : WRITE); if (remapped == NULL) return -1; if(flags & O_CREAT) { mode_t mode = va_arg(args, int); return real_open64(remapped, flags, mode); } else { return real_open64(remapped, flags, 0); } } int open64(const char * pathname, int flags, ...) { va_list args; va_start(args, flags); return intercepted_open64(pathname, flags, args); } int _open64(const char * pathname, int flags, ...) { va_list args; va_start(args, flags); return intercepted_open64(pathname, flags, args); } static int intercepted_openat(int dirfd, const char * pathname, int flags, va_list args) { #if __linux__ /* We're going to emulate openat() on top of open() by finding out the full path of the directory * via /proc/self/fd. */ char procfile[128]; char buffer[PATH_MAX]; ssize_t n; if (pathname[0] == '/') { return intercepted_open(pathname, flags, args); } sprintf(procfile, "/proc/self/fd/%d", dirfd); n = readlink(procfile, buffer, sizeof(buffer)); if (n < 0) { return n; } if (n + strlen(pathname) + 2 >= sizeof(buffer)) { errno = ENAMETOOLONG; return -1; } buffer[n] = '/'; strcpy(buffer + n + 1, pathname); return intercepted_open(buffer, flags, args); #else fprintf(stderr, "openat(%s) not intercepted\n", pathname); errno = ENOSYS; return -1; #endif } int openat(int dirfd, const char * pathname, int flags, ...) { va_list args; va_start(args, flags); return intercepted_openat(dirfd, pathname, flags, args); } int _openat(int dirfd, const char * pathname, int flags, ...) { va_list args; va_start(args, flags); return intercepted_openat(dirfd, pathname, flags, args); } static int intercepted_openat64(int dirfd, const char * pathname, int flags, va_list args) { #if __linux__ /* We're going to emulate openat() on top of open() by finding out the full path of the directory * via /proc/self/fd. */ char procfile[128]; char buffer[PATH_MAX]; ssize_t n; if (pathname[0] == '/') { return intercepted_open64(pathname, flags, args); } sprintf(procfile, "/proc/self/fd/%d", dirfd); n = readlink(procfile, buffer, sizeof(buffer)); if (n < 0) { return n; } if (n + strlen(pathname) + 2 >= sizeof(buffer)) { errno = ENAMETOOLONG; return -1; } buffer[n] = '/'; strcpy(buffer + n + 1, pathname); return intercepted_open64(buffer, flags, args); #else fprintf(stderr, "openat(%s) not intercepted\n", pathname); errno = ENOSYS; return -1; #endif } int openat64(int dirfd, const char * pathname, int flags, ...) { va_list args; va_start(args, flags); return intercepted_openat64(dirfd, pathname, flags, args); } int _openat64(int dirfd, const char * pathname, int flags, ...) { va_list args; va_start(args, flags); return intercepted_openat64(dirfd, pathname, flags, args); } /* For rename(), we consider both locations to be outputs, since both are modified. */ typedef int rename_t(const char* from, const char* to); int rename(const char* from, const char* to) { static rename_t* real_rename = NULL; char buffer[PATH_MAX]; char buffer2[PATH_MAX]; if (real_rename == NULL) { real_rename = (rename_t*) dlsym(RTLD_NEXT, "rename"); assert(real_rename != NULL); } from = remap_file("rename", from, buffer, WRITE); if (from == NULL) return -1; to = remap_file("rename", to, buffer2, WRITE); if (to == NULL) return -1; return real_rename (from, to); } int _rename(const char* from, const char* to) { return rename(from, to); } /* We override mkdir to just make the directory under tmp/. Ekam doesn't really care to track * directory creations. */ typedef int mkdir_t(const char* path, mode_t mode); int mkdir(const char* path, mode_t mode) { static mkdir_t* real_mkdir = NULL; char buffer[PATH_MAX]; char* slash_pos; if (real_mkdir == NULL) { real_mkdir = (mkdir_t*) dlsym(RTLD_NEXT, "mkdir"); assert(real_mkdir != NULL); } if (*path == '/') { /* Absolute path. Use the regular remap logic (which as of this writing will only allow * writes to /tmp). */ path = remap_file("mkdir", path, buffer, WRITE); if (path == NULL) return -1; return real_mkdir(path, mode); } else { strcpy(buffer, "tmp/"); if (strlen(path) + strlen(buffer) + 1 > PATH_MAX) { errno = ENAMETOOLONG; return -1; } strcat(buffer, path); /* Attempt to create parent directories. */ slash_pos = buffer; while ((slash_pos = strchr(slash_pos, '/')) != NULL) { *slash_pos = '\0'; /* We don't care if this fails -- we'll find out when we make the final mkdir call below. */ real_mkdir(buffer, mode); *slash_pos = '/'; ++slash_pos; } /* Finally create the requested directory. */ return real_mkdir(buffer, mode); } } int _mkdir(const char* path, mode_t mode) { return mkdir(path, mode); } /* HACK: Ekam doesn't really track directories right now, but some tools call access() on * directories. Here, we check if the path exists as a directory in src or tmp and, if so, * go ahead and return success. Otherwise, we remap as normal. * TODO: Add some sort of directory handling to Ekam? */ typedef int access_t(const char* path, int mode); int access(const char* path, int mode) { static access_t* real_access = NULL; char buffer[PATH_MAX]; struct stat stats; if (real_access == NULL) { real_access = (access_t*) dlsym(RTLD_NEXT, "access"); assert(real_access != NULL); } if (*path != '/') { strcpy(buffer, "src/"); if (strlen(path) + strlen(buffer) + 1 > PATH_MAX) { errno = ENAMETOOLONG; return -1; } strcat(buffer, path); if (direct_stat(buffer, &stats) == 0 && S_ISDIR(stats.st_mode)) { /* Directory exists in src. */ return 0; } memcpy(buffer, "tmp", 3); if (direct_stat(buffer, &stats) == 0 && S_ISDIR(stats.st_mode)) { /* Directory exists in tmp. */ return 0; } path = remap_file("access", path, buffer, READ); if (path == NULL) return -1; /* TODO: If checking W_OK, we should probably only succeed if this program created the file * in the first place. For the moment we simply disallow writing to source files, as a * compromise. */ if ((mode & W_OK) && strncmp(path, "src/", 4) == 0) { /* Cannot write to source files. */ errno = EACCES; return -1; } return real_access(path, mode); } else { /* Absolute path. Don't do anything fancy. */ path = remap_file("access", path, buffer, (mode & W_OK) ? WRITE : READ); if (path == NULL) return -1; return real_access(path, mode); } } ================================================ FILE: src/ekam/rules/intercept.ekam-rule ================================================ #! /bin/sh # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This is a one-off rule that builds intercept.so from intercept.c. set -eu INPUT=intercept.c echo findProvider file:intercept.c read INPUT_DISK_PATH handle_target() { OUTPUT="${INPUT%.c}$1.so" echo newOutput "$OUTPUT" read OUTPUT_DISK_PATH if test `uname` = Linux; then ${2}gcc -g -shared -Wall -Wl,-no-undefined -fPIC -o "$OUTPUT_DISK_PATH" "$INPUT_DISK_PATH" -ldl else ${2}gcc -g -shared -Wall -fPIC -o "$OUTPUT_DISK_PATH" "$INPUT_DISK_PATH" fi echo provide $OUTPUT_DISK_PATH special:ekam-interceptor$3 } handle_target "" "" "" for TARGET in ${CROSS_TARGETS:-}; do handle_target .${TARGET} ${TARGET}- -${TARGET} done ================================================ FILE: src/ekam/rules/proto.ekam-rule ================================================ #! /bin/sh # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu if test $# = 0; then echo trigger filetype:.proto exit 0 fi INPUT=$1 PROTO_NAME=$INPUT PROTO_NAME=${PROTO_NAME##*/src/} PROTO_NAME=${PROTO_NAME#src/} PROTO_NAME=${PROTO_NAME##*/include/} PROTO_NAME=${PROTO_NAME#include/} if test "$PROTO_NAME" = "google/protobuf/descriptor.proto" -o \ "$PROTO_NAME" = "google/protobuf/compiler/plugin.proto"; then # HACK: The generated code for this proto is checked into the repository because protoc itself # depends on it. # TODO: This could be made more general by checking if the output already exists. Or maybe # Ekam itself should detect when the same file exists from two sources and produce an error # if and only if they are not identical. That would actually be ideal here because it would # mean an error is produced if the checked-in generated code is out-of-date. exit 0 fi if test "$PROTO_NAME" = "$INPUT"; then SOURCE_ROOT=. else SOURCE_ROOT=${INPUT%/$PROTO_NAME} fi echo findProvider special:ekam-interceptor read INTERCEPTOR if test "$INTERCEPTOR" = ""; then echo "error: couldn't find intercept.so." >&2 exit 1 fi if test "${PROTOC:-}" = ""; then # No PROTOC specified; try to find one built from the source tree. echo findProvider file:google/protobuf/compiler/main read PROTOC if test "$PROTOC" = ""; then echo "error: couldn't find protoc." >&2 exit 1 fi fi LD_PRELOAD=$INTERCEPTOR DYLD_FORCE_FLAT_NAMESPACE= DYLD_INSERT_LIBRARIES=$INTERCEPTOR \ $PROTOC -I"$SOURCE_ROOT" -I/ekam-provider/protobuf --cpp_out="$SOURCE_ROOT" "$INPUT" 3>&1 4<&0 >&2 echo findInput $INPUT read INPUT_DISK_PATH echo provide "$INPUT_DISK_PATH" protobuf:"$PROTO_NAME" ================================================ FILE: src/ekam/rules/test.ekam-rule ================================================ #! /bin/sh # Ekam Build System # Author: Kenton Varda (kenton@sandstorm.io) # Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -eu if test $# = 0; then echo trigger test:executable exit 0 fi echo findInput "$1" read TEST_PROG INTERCEPTOR_TAG=special:ekam-interceptor INTERPRETER= for TARGET in ${CROSS_TARGETS:-}; do if test "${1%.$TARGET}" != "$1"; then INTERCEPTOR_TAG="special:ekam-interceptor-$TARGET" export QEMU_LD_PREFIX=/usr/$TARGET INTERPRETER="qemu-${TARGET%-linux-gnu}" fi done if grep -q EKAM_TEST_DISABLE_INTERCEPTOR "$TEST_PROG"; then # Test requested no interceptor. INTERCEPTOR= else echo findProvider "$INTERCEPTOR_TAG" read INTERCEPTOR if test "$INTERCEPTOR" = ""; then echo "error: couldn't find intercept.so." >&2 exit 1 fi fi # TODO: Run in sandbox. echo newOutput "${1}.log" read TEST_LOG if ${TEST_WRAPPER:-} env LD_PRELOAD=$INTERCEPTOR DYLD_FORCE_FLAT_NAMESPACE= DYLD_INSERT_LIBRARIES=$INTERCEPTOR \ $INTERPRETER $TEST_PROG 3>&1 4<&0 2>"$TEST_LOG" >&2; then echo passed exit 0 else echo "full log: $TEST_LOG" >&2 egrep 'FAIL|ERROR|FATAL|ekam-provider' "$TEST_LOG" >&2 exit 1 fi ================================================ FILE: src/os/ByteStream.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "ByteStream.h" #include #include #include #include #include #include #include "base/Debug.h" namespace ekam { ByteStream::ByteStream(const std::string& path, int flags) : handle(path, WRAP_SYSCALL(open, path.c_str(), flags, 0666)) {} ByteStream::ByteStream(const std::string& path, int flags, int mode) : handle(path, WRAP_SYSCALL(open, path.c_str(), flags, mode)) {} ByteStream::ByteStream(int fd, const std::string& name) : handle(name, fd) {} ByteStream::~ByteStream() noexcept(false) {} size_t ByteStream::read(void* buffer, size_t size) { return WRAP_SYSCALL(read, handle, buffer, size); } Promise ByteStream::readAsync(EventManager* eventManager, void* buffer, size_t size) { if (watcher == nullptr) { watcher = eventManager->watchFd(handle.get()); } return eventManager->when(watcher->onReadable())( [this, buffer, size](Void) -> size_t { return read(buffer, size); }); } size_t ByteStream::write(const void* buffer, size_t size) { return WRAP_SYSCALL(write, handle, buffer, size); } void ByteStream::writeAll(const void* buffer, size_t size) { const char* cbuffer = reinterpret_cast(buffer); while (size > 0) { ssize_t n = write(cbuffer, size); cbuffer += n; size -= n; } } void ByteStream::stat(struct stat* stats) { WRAP_SYSCALL(fstat, handle, stats); } // ======================================================================================= Pipe::Pipe() { if (pipe(fds) != 0) { throw OsError("", "pipe", errno); } fcntl(fds[0], F_SETFD, FD_CLOEXEC); fcntl(fds[1], F_SETFD, FD_CLOEXEC); } Pipe::~Pipe() { closeReadEnd(); closeWriteEnd(); } OwnedPtr Pipe::releaseReadEnd() { auto result = newOwned(fds[0], "pipe.readEnd"); fds[0] = -1; return result.release(); } OwnedPtr Pipe::releaseWriteEnd() { auto result = newOwned(fds[1], "pipe.writeEnd"); fds[1] = -1; return result.release(); } void Pipe::attachReadEndForExec(int target) { dup2(fds[0], target); closeReadEnd(); closeWriteEnd(); } void Pipe::attachWriteEndForExec(int target) { dup2(fds[1], target); closeReadEnd(); closeWriteEnd(); } void Pipe::closeReadEnd() { if (fds[0] != -1) { if (close(fds[0]) != 0) { DEBUG_ERROR << "close(pipe): " << strerror(errno); } fds[0] = -1; } } void Pipe::closeWriteEnd() { if (fds[1] != -1) { if (close(fds[1]) != 0) { DEBUG_ERROR << "close(pipe): " << strerror(errno); } fds[1] = -1; } } } // namespace ekam ================================================ FILE: src/os/ByteStream.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_BYTESTREAM_H_ #define KENTONSCODE_OS_BYTESTREAM_H_ #include #include #include #include "base/OwnedPtr.h" #include "OsHandle.h" #include "EventManager.h" namespace ekam { class EventManager; class ByteStream { public: // TODO(kenton): Make class abstract. ByteStream(const std::string& path, int flags); ByteStream(const std::string& path, int flags, int mode); ByteStream(int fd, const std::string& name); ~ByteStream() noexcept(false); inline OsHandle* getHandle() { return &handle; } size_t read(void* buffer, size_t size); Promise readAsync(EventManager* eventManager, void* buffer, size_t size); size_t write(const void* buffer, size_t size); void writeAll(const void* buffer, size_t size); void stat(struct stat* stats); private: class ReadEventCallback; OsHandle handle; OwnedPtr watcher; }; class Pipe { public: Pipe(); ~Pipe(); OwnedPtr releaseReadEnd(); OwnedPtr releaseWriteEnd(); void attachReadEndForExec(int target); void attachWriteEndForExec(int target); private: int fds[2]; void closeReadEnd(); void closeWriteEnd(); }; } // namespace ekam #endif // KENTONSCODE_OS_BYTESTREAM_H_ ================================================ FILE: src/os/DiskFile.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "DiskFile.h" #include #include #include #include #include #include #include #include #include #include #include "base/Debug.h" #include "os/OsHandle.h" #include "os/ByteStream.h" #include "base/Hash.h" namespace ekam { // TODO: Use new openat() and friends? Very capability-like! namespace { class DirectoryReader { public: DirectoryReader(const std::string& path) : path(path), dir(opendir(path.c_str())) { if (dir == NULL) { // TODO: Better exception type. throw OsError(path, "opendir", ENOTDIR); } } ~DirectoryReader() { if (closedir(dir) < 0) { DEBUG_ERROR << path << ": closedir: " << strerror(errno); } } bool next(std::string* output) { errno = 0; struct dirent *entryPointer = readdir(dir); if (errno != 0) { throw OsError(path, "readdir", errno); } if (entryPointer == NULL) { return false; } else { *output = entryPointer->d_name; return true; } } private: const std::string& path; DIR* dir; }; bool statIfExists(const std::string& path, struct stat* output) { int result; do { result = stat(path.c_str(), output); } while (result < 0 && errno == EINTR); if (result == 0) { return true; } else if (errno == ENOENT) { return false; } else { throw OsError(path, "stat", errno); } } } // anonymous namespace DiskFile::DiskFile(const std::string& path, File* parent) : path(path) { if (parent != NULL) { this->parentRef = parent->clone(); } } DiskFile::~DiskFile() {} std::string DiskFile::basename() { if (path.empty()) { return "."; } std::string::size_type slashPos = path.find_last_of('/'); if (slashPos == std::string::npos) { return path; } else { return path.substr(slashPos + 1); } } std::string DiskFile::canonicalName() { if (parentRef == NULL) { return "."; } std::string result = parentRef->canonicalName(); if (result == ".") { result.clear(); } else { result.push_back('/'); } result.append(basename()); return result; } OwnedPtr DiskFile::clone() { return newOwned(path, parentRef.get()); } bool DiskFile::hasParent() { return parentRef != NULL; } OwnedPtr DiskFile::parent() { if (parentRef == NULL) { throw std::runtime_error("Tried to get parent of top-level directory: " + canonicalName()); } return parentRef->clone(); } bool DiskFile::equals(File* other) { DiskFile* otherDiskFile = dynamic_cast(other); return otherDiskFile != NULL && otherDiskFile->path == path; } size_t DiskFile::identityHash() { return std::hash()(path); } class DiskFile::DiskRefImpl : public File::DiskRef { public: DiskRefImpl(const std::string& path) : pathName(path) {} ~DiskRefImpl() {} // implements DiskRef ------------------------------------------------------------------ virtual const std::string& path() { return pathName; } private: std::string pathName; }; OwnedPtr DiskFile::getOnDisk(Usage usage) { return newOwned(path); } bool DiskFile::exists() { struct stat stats; return statIfExists(path.c_str(), &stats) && (S_ISREG(stats.st_mode) || S_ISDIR(stats.st_mode)); } bool DiskFile::isFile() { struct stat stats; return statIfExists(path.c_str(), &stats) && S_ISREG(stats.st_mode); } bool DiskFile::isDirectory() { struct stat stats; return statIfExists(path.c_str(), &stats) && S_ISDIR(stats.st_mode); } // File only. Hash DiskFile::contentHash() { try { Hash::Builder hasher; ByteStream fd(path, O_RDONLY); char buffer[8192]; while (true) { size_t n = fd.read(buffer, sizeof(buffer)); if (n == 0) { return hasher.build(); } hasher.add(buffer, n); } } catch (const OsError& e) { if (e.getErrorNumber() == ENOENT || e.getErrorNumber() == EACCES || e.getErrorNumber() == EISDIR) { return Hash::NULL_HASH; } throw; } } std::string DiskFile::readAll() { ByteStream fd(path, O_RDONLY); struct stat stats; fd.stat(&stats); size_t size = stats.st_size; std::string result; if (size > 0) { result.resize(size); char* buffer = &*result.begin(); size_t bytesRead = 0; while (size > bytesRead) { ssize_t n = fd.read(buffer + bytesRead, size - bytesRead); if (n == 0) { result.resize(bytesRead); size = bytesRead; } else { bytesRead += n; } } } return result; } void DiskFile::writeAll(const std::string& content) { ByteStream fd(path, O_WRONLY | O_TRUNC | O_CREAT); std::string::size_type pos = 0; while (pos < content.size()) { pos += fd.write(content.data() + pos, content.size() - pos); } } void DiskFile::writeAll(const void* data, int size) { ByteStream fd(path, O_WRONLY | O_TRUNC | O_CREAT); const char* pos = reinterpret_cast(data); while (size > 0) { int n = fd.write(pos, size); pos += n; size -= n; } } // Directory only. void DiskFile::list(OwnedPtrVector::Appender output) { std::string prefix; if (!path.empty()) { prefix = path + "/"; } DirectoryReader reader(path); std::string filename; while (reader.next(&filename)) { if (filename.empty()) { DEBUG_ERROR << "DirectoryReader returned empty file name."; } else if (filename[0] != '.') { // skip hidden files output.add(newOwned(prefix + filename, this)); } } } OwnedPtr DiskFile::relative(const std::string& path) { if (path.empty()) { throw std::invalid_argument("File::relative(): path cannot be empty."); } else if (path[0] == '/') { throw std::invalid_argument("File::relative(): path cannot start with a slash."); } std::string::size_type slash_pos = path.find_first_of('/'); std::string first_part; std::string rest; if (slash_pos == std::string::npos) { if (path == ".") { return clone(); } else if (path == "..") { return parent(); } else if (this->path.empty()) { return newOwned(path, this); } else { return newOwned(this->path + "/" + path, this); } } else { first_part.assign(path, 0, slash_pos); std::string::size_type after_slash_pos = path.find_first_not_of('/', slash_pos); if (after_slash_pos == std::string::npos) { // Trailing slash. Bah. return relative(first_part); } else { rest.assign(path, after_slash_pos, std::string::npos); if (first_part == ".") { return relative(rest); } else if (first_part == "..") { if (parentRef == NULL) { throw std::runtime_error( "Tried to get parent of top-level directory: " + canonicalName()); } return parentRef->relative(rest); } else { OwnedPtr temp; if (this->path.empty()) { temp = newOwned(first_part, this); } else { temp = newOwned(this->path + "/" + first_part, this); } return temp->relative(rest); } } } } void DiskFile::createDirectory() { while (true) { if (mkdir(path.c_str(), 0777) == 0) { return; } else if (errno != EINTR) { throw OsError(path, "mkdir", errno); } } } void DiskFile::link(File* target) { DiskFile* diskTarget = dynamic_cast(target); if (diskTarget == NULL) { throw new std::invalid_argument("Cannot link disk file to non-disk file: " + path); } WRAP_SYSCALL(link, diskTarget->path.c_str(), path.c_str()); } void DiskFile::unlink() { WRAP_SYSCALL(unlink, path.c_str()); } } // namespace ekam ================================================ FILE: src/os/DiskFile.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_DISKFILE_H_ #define KENTONSCODE_OS_DISKFILE_H_ #include "File.h" #include namespace ekam { class DiskFile: public File { public: DiskFile(const std::string& path, File* parent); ~DiskFile(); // implements File --------------------------------------------------------------------- std::string basename(); std::string canonicalName(); OwnedPtr clone(); bool hasParent(); OwnedPtr parent(); bool equals(File* other); size_t identityHash(); OwnedPtr getOnDisk(Usage usage); bool exists(); bool isFile(); bool isDirectory(); // File only. Hash contentHash(); std::string readAll(); void writeAll(const std::string& content); void writeAll(const void* data, int size); // Directory only. void list(OwnedPtrVector::Appender output); OwnedPtr relative(const std::string& path); // Methods that create or delete objects. void createDirectory(); void link(File* target); void unlink(); private: class DiskRefImpl; std::string path; OwnedPtr parentRef; }; } // namespace ekam #endif // KENTONSCODE_OS_DISKFILE_H_ ================================================ FILE: src/os/EpollEventManager.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "EpollEventManager.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "base/Debug.h" #include "base/Table.h" namespace ekam { namespace { std::string epollEventsToString(uint32_t events) { std::string result; if (events == 0) { result.append(" (none)"); } if (events & EPOLLIN) { result.append(" EPOLLIN"); } if (events & EPOLLOUT) { result.append(" EPOLLOUT"); } if (events & EPOLLRDHUP) { result.append(" EPOLLRDHUP"); } if (events & EPOLLPRI) { result.append(" EPOLLPRI"); } if (events & EPOLLERR) { result.append(" EPOLLERR"); } if (events & EPOLLHUP) { result.append(" EPOLLHUP"); } if (events & EPOLLET) { result.append(" EPOLLET"); } if (events & EPOLLONESHOT) { result.append(" EPOLLONESHOT"); } if (events & ~(EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLPRI | EPOLLERR | EPOLLHUP | EPOLLET | EPOLLONESHOT)) { result.append(" (others)"); } return result; } // TODO: Copied from DiskFile.cpp. Share code somehow? bool statIfExists(const std::string& path, struct stat* output) { int result; do { result = stat(path.c_str(), output); } while (result < 0 && errno == EINTR); if (result == 0) { return true; } else if (errno == ENOENT) { return false; } else { throw OsError(path, "stat", errno); } } bool isDirectory(const std::string& path) { struct stat stats; return statIfExists(path.c_str(), &stats) && S_ISDIR(stats.st_mode); } } // namespace EpollEventManager::Epoller::Epoller() : epollHandle("epoll", WRAP_SYSCALL(epoll_create1, (int)EPOLL_CLOEXEC)), watchCount(0) {} EpollEventManager::Epoller::~Epoller() { if (watchCount > 0) { DEBUG_ERROR << "Epoller destroyed before all Watches destroyed."; } } EpollEventManager::Epoller::Watch::Watch(Epoller* epoller, OsHandle* handle, uint32_t events, IoHandler* handler) : epoller(epoller), events(0), registeredEvents(0), fd(handle->get()), name(handle->getName()), handler(handler) { addEvents(events); } EpollEventManager::Epoller::Watch::Watch(Epoller* epoller, int fd, uint32_t events, IoHandler* handler) : epoller(epoller), events(0), registeredEvents(0), fd(fd), name(toString(fd)), handler(handler) { addEvents(events); } EpollEventManager::Epoller::Watch::~Watch() { removeEvents(events); if (epoller->watchesNeedingUpdate.erase(this) > 0) { updateRegistration(); } } void EpollEventManager::Epoller::Watch::addEvents(uint32_t eventsToAdd) { DEBUG_INFO << "Adding events for " << fd << ":" << epollEventsToString(eventsToAdd); uint32_t newEvents = events | eventsToAdd; if (newEvents == events) { return; } events = newEvents; if (events == registeredEvents) { epoller->watchesNeedingUpdate.erase(this); } else { epoller->watchesNeedingUpdate.insert(this); } } void EpollEventManager::Epoller::Watch::removeEvents(uint32_t eventsToRemove) { DEBUG_INFO << "Removing events for " << fd << ":" << epollEventsToString(eventsToRemove); uint32_t newEvents = events & ~eventsToRemove; if (newEvents == events) { return; } events = newEvents; if (events == registeredEvents) { epoller->watchesNeedingUpdate.erase(this); } else { epoller->watchesNeedingUpdate.insert(this); } } void EpollEventManager::Epoller::Watch::updateRegistration() { if (registeredEvents == events) { DEBUG_ERROR << "Watch does not need updating."; return; } DEBUG_INFO << "Updating event registrations for " << fd << ":" << epollEventsToString(events); int op = EPOLL_CTL_MOD; if (registeredEvents == 0) { ++epoller->watchCount; op = EPOLL_CTL_ADD; } else if (events == 0) { --epoller->watchCount; op = EPOLL_CTL_DEL; } registeredEvents = events; struct epoll_event event; event.events = registeredEvents; event.data.ptr = this; WRAP_SYSCALL(epoll_ctl, epoller->epollHandle, op, fd, &event); } bool EpollEventManager::Epoller::handleEvent() { // Run pending updates. for (Watch* watch : watchesNeedingUpdate) { watch->updateRegistration(); } watchesNeedingUpdate.clear(); if (watchCount == 0) { DEBUG_INFO << "No more events."; return false; } DEBUG_INFO << "Waiting for " << watchCount << " events..."; struct epoll_event event; int result = WRAP_SYSCALL(epoll_wait, epollHandle, &event, 1, -1); if (result == 0) { throw std::logic_error("epoll_wait() returned zero despite infinite timeout."); } else if (result > 1) { throw std::logic_error("epoll_wait() returned more than one event when only one requested."); } Watch* watch = reinterpret_cast(event.data.ptr); DEBUG_INFO << "epoll event: " << watch->name << ":" << epollEventsToString(event.events); watch->handler->handle(event.events); return true; } // ============================================================================= namespace { sigset_t getHandledSignals() { sigset_t result; sigemptyset(&result); sigaddset(&result, SIGCHLD); return result; } const sigset_t HANDLED_SIGNALS = getHandledSignals(); } // namespace EpollEventManager::SignalHandler::SignalHandler(Epoller* epoller) : signalStream(WRAP_SYSCALL(signalfd, -1, &HANDLED_SIGNALS, SFD_NONBLOCK | SFD_CLOEXEC), "signalfd"), watch(epoller, signalStream.getHandle(), 0, this) { sigprocmask(SIG_BLOCK, &HANDLED_SIGNALS, NULL); } EpollEventManager::SignalHandler::~SignalHandler() { sigprocmask(SIG_UNBLOCK, &HANDLED_SIGNALS, NULL); } void EpollEventManager::SignalHandler::handle(uint32_t events) { DEBUG_INFO << "Received signal on signalfd."; struct signalfd_siginfo signalEvent; if (signalStream.read(&signalEvent, sizeof(signalEvent)) != sizeof(signalEvent)) { DEBUG_ERROR << "read(signalfd) returned wrong size."; return; } if (signalEvent.ssi_signo == SIGCHLD) { // Alas, the contents of signalEvent are useless, as the signal queue only holds one signal // per signal number. Therefore, if two SIGCHLDs are received before we have a chance to // handle the first one, one of the two is discarded. But we have to wait() to avoid zombies // anyway so I guess it's no big deal. handleProcessExit(); } else { DEBUG_ERROR << "Unexpected signal number: " << signalEvent.ssi_signo; } } void EpollEventManager::SignalHandler::maybeStopExpecting() { if (processExitHandlerMap.empty()) { watch.removeEvents(EPOLLIN); } } // ======================================================================================= class EpollEventManager::AsyncCallbackHandler : public PendingRunnable { public: AsyncCallbackHandler(EpollEventManager* eventManager, OwnedPtr runnable) : eventManager(eventManager), called(false), runnable(runnable.release()) { eventManager->asyncCallbacks.push_back(this); } ~AsyncCallbackHandler() { if (!called) { for (std::deque::iterator iter = eventManager->asyncCallbacks.begin(); iter != eventManager->asyncCallbacks.end(); ++iter) { if (*iter == this) { eventManager->asyncCallbacks.erase(iter); return; } } DEBUG_ERROR << "AsyncCallbackHandler not called but not in asyncCallbacks."; } } void run() { called = true; runnable->run(); } private: EpollEventManager* eventManager; bool called; OwnedPtr runnable; }; OwnedPtr EpollEventManager::runLater(OwnedPtr runnable) { return newOwned(this, runnable.release()); } // ======================================================================================= class EpollEventManager::SignalHandler::ProcessExitHandler : public PromiseFulfiller { public: ProcessExitHandler(Callback* callback, SignalHandler* signalHandler, pid_t pid) : callback(callback), signalHandler(signalHandler), pid(pid) { if (!signalHandler->processExitHandlerMap.insert( std::make_pair(pid, this)).second) { throw std::runtime_error("Already waiting on this process."); } signalHandler->watch.addEvents(EPOLLIN); } ~ProcessExitHandler() { if (pid != -1) { signalHandler->processExitHandlerMap.erase(pid); signalHandler->maybeStopExpecting(); } } void handle(int waitStatus) { DEBUG_INFO << "Process " << pid << " exited with status: " << waitStatus; signalHandler->processExitHandlerMap.erase(pid); signalHandler->maybeStopExpecting(); pid = -1; if (WIFEXITED(waitStatus)) { callback->fulfill(ProcessExitCode(WEXITSTATUS(waitStatus))); } else if (WIFSIGNALED(waitStatus)) { callback->fulfill(ProcessExitCode(ProcessExitCode::SIGNALED, WTERMSIG(waitStatus))); } else { DEBUG_ERROR << "Didn't understand process exit status."; callback->fulfill(ProcessExitCode(-1)); } } private: Callback* callback; SignalHandler* signalHandler; pid_t pid; }; void EpollEventManager::SignalHandler::handleProcessExit() { // If multiple signals with the same signal number are delivered while signals are blocked, // only one of them is actually delivered once un-blocked. The others are cast into the // void. Therefore, the contents of the siginfo structure are effectively useless for // SIGCHLD. We must instead call waitpid() repeatedly until there are no more completed // children. Signals suck so much. while (true) { int waitStatus; pid_t pid = waitpid(-1, &waitStatus, WNOHANG); if (pid < 0) { // ECHILD indicates there are no child processes. Anything else is a real error. if (errno != ECHILD) { DEBUG_ERROR << "waitpid: " << strerror(errno); } break; } else if (pid == 0) { // There are child processes, but they are still running. break; } // Get the handler associated with this PID. std::unordered_map::iterator iter = processExitHandlerMap.find(pid); if (iter == processExitHandlerMap.end()) { // It is actually important that any code creating a subprocess call onProcessExit() // to receive notification of completion even if it doesn't care about completion, // because otherwise the sub-process may be stuck as a zombie. This is actually // NOT the case with EpollEventManager because it waits on all sub-processes whether // onProcessExit() was called or not, but we should warn if we encounter a process for // which onProcessExit() was never called so that the code can be fixed. DEBUG_ERROR << "Got SIGCHLD for PID we weren't waiting for: " << pid; return; } iter->second->handle(waitStatus); } } Promise EpollEventManager::SignalHandler::onProcessExit(pid_t pid) { return newPromise(this, pid); } Promise EpollEventManager::onProcessExit(pid_t pid) { return signalHandler.onProcessExit(pid); } // ======================================================================================= class EpollEventManager::IoWatcherImpl: public IoWatcher, public IoHandler { public: IoWatcherImpl(Epoller* epoller, int fd) : watch(epoller, fd, 0, this), readFulfiller(nullptr), writeFulfiller(nullptr) {} ~IoWatcherImpl() { if (readFulfiller != nullptr) { readFulfiller->abandon(); } if (writeFulfiller != nullptr) { writeFulfiller->abandon(); } } // implements IoWatcher -------------------------------------------------------------- Promise onReadable() { if (readFulfiller != nullptr) { throw std::logic_error("Already waiting for readability on this fd."); } return newPromise(&watch, EPOLLIN, &readFulfiller); } Promise onWritable() { if (readFulfiller != nullptr) { throw std::logic_error("Already waiting for writability on this fd."); } return newPromise(&watch, EPOLLOUT, &writeFulfiller); } // implements IoHandler -------------------------------------------------------------- void handle(uint32_t events) { if (events & (EPOLLIN | EPOLLERR | EPOLLHUP)) { if (readFulfiller != nullptr) { readFulfiller->ready(); } } if (events & (EPOLLOUT | EPOLLERR | EPOLLHUP)) { if (writeFulfiller != nullptr) { writeFulfiller->ready(); } } } private: class Fulfiller: public PromiseFulfiller { public: Fulfiller(Callback* callback, Epoller::Watch* watch, uint32_t events, Fulfiller** ptr) : callback(callback), watch(watch), events(events), ptr(ptr) { *ptr = this; watch->addEvents(events); } ~Fulfiller() { if (ptr != nullptr) { *ptr = nullptr; watch->removeEvents(events); } } void ready() { *ptr = nullptr; ptr = nullptr; watch->removeEvents(events); callback->fulfill(); } void abandon() { *ptr = nullptr; ptr = nullptr; watch->removeEvents(events); try { throw std::logic_error("IoWatcher deleted while waiting for I/O."); } catch (...) { callback->propagateCurrentException(); } } private: Callback* callback; Epoller::Watch* watch; uint32_t events; Fulfiller** ptr; }; Epoller::Watch watch; Fulfiller* readFulfiller; Fulfiller* writeFulfiller; }; OwnedPtr EpollEventManager::watchFd(int fd) { return newOwned(&epoller, fd); } // ======================================================================================= class EpollEventManager::InotifyHandler::WatchedDirectory { class CallbackTable : public Table, UniqueColumn > { public: static const int BASENAME = 0; static const int WATCH_OP = 1; }; public: WatchedDirectory(InotifyHandler* inotifyHandler, const std::string& path) : inotifyHandler(inotifyHandler), path(path) { wd = WRAP_SYSCALL(inotify_add_watch, *inotifyHandler->inotifyStream.getHandle(), path.c_str(), IN_ATTRIB | IN_CLOSE_WRITE | IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVE_SELF | IN_MOVED_FROM | IN_MOVED_TO); DEBUG_INFO << "inotify_add_watch(" << path << ") [" << wd << "]"; inotifyHandler->watchMap[wd] = this; inotifyHandler->watchByNameMap[path] = this; inotifyHandler->watch.addEvents(EPOLLIN); } ~WatchedDirectory() { DEBUG_INFO << "~WatchedDirectory(): " << path; if (callbackTable.size() > 0) { DEBUG_ERROR << "Deleting WatchedDirectory before all FileWatcherImpls were removed."; } if (wd >= 0) { DEBUG_INFO << "inotify_rm_watch(" << path << ") [" << wd << "]"; if (WRAP_SYSCALL(inotify_rm_watch, *inotifyHandler->inotifyStream.getHandle(), wd) < 0) { DEBUG_ERROR << "inotify_rm_watch(" << path << "): " << strerror(errno); } } invalidate(); } void addWatch(const std::string& basename, FileWatcherImpl* op) { DEBUG_INFO << "Watch directory " << path << " now covering: " << basename; callbackTable.add(basename, op); } void removeWatch(FileWatcherImpl* op) { DEBUG_INFO << "Watch directory " << path << " no longer covering: " << basenameForOp(op); if (callbackTable.erase(op) == 0) { DEBUG_ERROR << "Trying to remove watch that was never added."; } deleteSelfIfEmpty(); } void invalidate() { if (wd >= 0) { inotifyHandler->watchMap.erase(wd); inotifyHandler->watchByNameMap.erase(path); wd = -1; } } void handle(struct inotify_event* event); private: InotifyHandler* inotifyHandler; int wd; std::string path; CallbackTable callbackTable; void deleteSelfIfEmpty() { if (callbackTable.size() == 0) { auto inotifyHandler = this->inotifyHandler; inotifyHandler->ownedWatchDirectories.erase(this); if (inotifyHandler->ownedWatchDirectories.empty()) { inotifyHandler->watch.removeEvents(EPOLLIN); } } } std::string basenameForOp(FileWatcherImpl* op) { const CallbackTable::Row* row = callbackTable.find(op); if (row == NULL) { return "(invalid)"; } else { return row->cell(); } } }; class EpollEventManager::InotifyHandler::FileWatcherImpl: public FileWatcher { public: FileWatcherImpl(InotifyHandler* inotifyHandler, const std::string& filename) : watchedDirectory(nullptr), modified(false), deleted(false), fulfiller(nullptr) { // Split directory and basename. std::string directory; std::string basename; if (isDirectory(filename)) { directory = filename; } else { std::string::size_type slashPos = filename.find_last_of('/'); if (slashPos == std::string::npos) { directory.assign("."); basename.assign(filename); } else { directory.assign(filename, 0, slashPos); basename.assign(filename, slashPos + 1, std::string::npos); } } // Find or create WatchedDirectory object. WatchByNameMap::iterator iter = inotifyHandler->watchByNameMap.find(directory); if (iter == inotifyHandler->watchByNameMap.end()) { auto newWatchedDirectory = newOwned(inotifyHandler, directory); watchedDirectory = newWatchedDirectory.get(); inotifyHandler->ownedWatchDirectories.add(watchedDirectory, newWatchedDirectory.release()); } else { watchedDirectory = iter->second; } // Add to WatchedDirectory. watchedDirectory->addWatch(basename, this); } ~FileWatcherImpl() { if (watchedDirectory != NULL) { watchedDirectory->removeWatch(this); } if (fulfiller != nullptr) { fulfiller->abandon(); } } void flagAsModified() { modified = true; maybeFulfill(); } void flagAsDeleted() { deleted = true; maybeFulfill(); } // implements FileWatcher -------------------------------------------------------------- Promise onChange() { if (fulfiller != nullptr) { fulfiller->abandon(); } auto result = newPromise(&fulfiller); maybeFulfill(); return result; } private: class Fulfiller: public PromiseFulfiller { public: Fulfiller(Callback* callback, Fulfiller** ptr) : callback(callback), ptr(ptr) { *ptr = this; } ~Fulfiller() { if (ptr != nullptr) { *ptr = nullptr; } } void fulfill(FileChangeType type) { *ptr = nullptr; ptr = nullptr; callback->fulfill(type); } void abandon() { *ptr = nullptr; ptr = nullptr; try { throw std::logic_error("FileWatcher deleted while waiting for changes."); } catch (...) { callback->propagateCurrentException(); } } private: Callback* callback; Fulfiller** ptr; }; WatchedDirectory* watchedDirectory; bool modified; bool deleted; Fulfiller* fulfiller; void maybeFulfill() { if (fulfiller != nullptr) { if (deleted) { fulfiller->fulfill(FileChangeType::DELETED); } else if (modified) { fulfiller->fulfill(FileChangeType::MODIFIED); } deleted = false; modified = false; } } }; EpollEventManager::InotifyHandler::InotifyHandler(Epoller* epoller) : inotifyStream(WRAP_SYSCALL(inotify_init1, IN_NONBLOCK | IN_CLOEXEC), "inotify"), watch(epoller, inotifyStream.getHandle(), 0, this) {} EpollEventManager::InotifyHandler::~InotifyHandler() {} void EpollEventManager::InotifyHandler::WatchedDirectory::handle(struct inotify_event* event) { std::string basename; if (event->len > 0) { basename = event->name; } DEBUG_INFO << "inotify event on: " << path << "\n basename: " << basename << "\n flags:" << ((event->mask & IN_ATTRIB ) ? " IN_ATTRIB" : "") << ((event->mask & IN_CLOSE_WRITE) ? " IN_CLOSE_WRITE" : "") << ((event->mask & IN_CREATE ) ? " IN_CREATE" : "") << ((event->mask & IN_DELETE ) ? " IN_DELETE" : "") << ((event->mask & IN_DELETE_SELF) ? " IN_DELETE_SELF" : "") << ((event->mask & IN_MODIFY ) ? " IN_MODIFY" : "") << ((event->mask & IN_MOVE_SELF ) ? " IN_MOVE_SELF" : "") << ((event->mask & IN_MOVED_FROM ) ? " IN_MOVED_FROM" : "") << ((event->mask & IN_MOVED_TO ) ? " IN_MOVED_TO" : ""); // Some events implicitly remove the watch descriptor (because the watched directory no longer // exists). Such descriptors are now invalid and may be reused the next time // inotify_add_watch() is called. But, the corresponding WatchedDirectories still have // FileWatcherImpls pointing at them, so we can't just delete them. So, invalidate them so // that no future FileWatcherImpls will use them. // // inotify has a special kind of event called IN_IGNORED which is supposed to signal that the // watch descriptor was removed, either implicitly or explicitly. However, in my testing, // this event does not seem to be generated in the case of IN_MOVE_SELF, even though this // case *does* implicitly remove the watch descriptor (which I know because the next call // to inotify_add_watch() typically reuses it). This appears to be a bug in Linux (observed // in 2.6.35-22-generic). Will file a bug report if I get time to write a demo program. if (event->mask & (IN_DELETE_SELF | IN_MOVE_SELF)) { DEBUG_INFO << "Watch descriptor implicitly removed: " << event->wd; invalidate(); } for (CallbackTable::SearchIterator iter(callbackTable, basename); iter.next();) { FileWatcherImpl* op = iter.cell(); if (event->mask & (IN_DELETE | IN_DELETE_SELF | IN_MOVED_FROM | IN_MOVE_SELF)) { op->flagAsDeleted(); } else { op->flagAsModified(); } } // If this event is indicating creation or deletion of a file in the directory, then call the // directory's modified() callback as well. if (!basename.empty() && (event->mask & (IN_CREATE | IN_DELETE | IN_MOVED_FROM | IN_MOVED_TO))) { for (CallbackTable::SearchIterator iter(callbackTable, ""); iter.next();) { FileWatcherImpl* op = iter.cell(); op->flagAsModified(); } } } void EpollEventManager::InotifyHandler::handle(uint32_t epollEvents) { char buffer[sizeof(struct inotify_event) + PATH_MAX]; ssize_t n = inotifyStream.read(buffer, sizeof(buffer)); char* pos = buffer; char* end = buffer + n; // Annoyingly, inotify() provides no way to read a single event at a time. If we were using // traditional callbacks for each event, then the callback for an earlier event could invalidate // later events. E.g. handling the first event might cause the watch descriptor for the second // event to be unregistered. // // As if that weren't bad enough, any particular event in the stream might indicate that a // particular watch descriptor has been automatically removed because the thing it was watching // no longer exists. This removal takes place at the time of the read(). So if the second // event in the buffer has the "watch descriptor removed" flag, and then while handling the // *first* event we create a new watch descriptor, that new descriptor may have the same // number as the one associated with the second event THAT WE HAVEN'T EVEN HANDLED YET. // // Luckily, when a promise is fulfilled, no callback is executed immediately; the callback // is queued on the event queue. Therefore, we don't have to worry about our caller coming // back and messing with our state while we're still going through the event list. while (pos < end) { if (end - pos < (signed)sizeof(struct inotify_event)) { DEBUG_ERROR << "read(inotifyFd) returned too few bytes to be an inotify_event."; break; } struct inotify_event* event = reinterpret_cast(pos); if (end - pos - sizeof(struct inotify_event) < event->len) { DEBUG_ERROR << "read(inotifyFd) returned inotify_event with 'len' that overruns the buffer."; break; } pos += sizeof(struct inotify_event) + event->len; DEBUG_INFO << "inotify " << event->wd << ":" << ((event->mask & IN_ATTRIB ) ? " IN_ATTRIB" : "") << ((event->mask & IN_CLOSE_WRITE) ? " IN_CLOSE_WRITE" : "") << ((event->mask & IN_CREATE ) ? " IN_CREATE" : "") << ((event->mask & IN_DELETE ) ? " IN_DELETE" : "") << ((event->mask & IN_DELETE_SELF) ? " IN_DELETE_SELF" : "") << ((event->mask & IN_MODIFY ) ? " IN_MODIFY" : "") << ((event->mask & IN_MOVE_SELF ) ? " IN_MOVE_SELF" : "") << ((event->mask & IN_MOVED_FROM ) ? " IN_MOVED_FROM" : "") << ((event->mask & IN_MOVED_TO ) ? " IN_MOVED_TO" : "") << ((event->mask & IN_IGNORED ) ? " IN_IGNORED" : ""); WatchMap::iterator iter = watchMap.find(event->wd); if (iter == watchMap.end()) { if (event->mask != IN_IGNORED) { DEBUG_ERROR << "inotify event had unknown watch descriptor? " << event->wd; } } else { iter->second->handle(event); } } } OwnedPtr EpollEventManager::InotifyHandler::watchFile( const std::string& filename) { return newOwned(this, filename); } OwnedPtr EpollEventManager::watchFile(const std::string& filename) { return inotifyHandler.watchFile(filename); } // ======================================================================================= EpollEventManager::EpollEventManager() : signalHandler(&epoller), inotifyHandler(&epoller) {} EpollEventManager::~EpollEventManager() {} void EpollEventManager::loop() { while (handleEvent()) {} } bool EpollEventManager::handleEvent() { // Run any async callbacks first. // TODO: Avoid starvation of I/O? Probably doesn't matter. if (!asyncCallbacks.empty()) { AsyncCallbackHandler* handler = asyncCallbacks.front(); asyncCallbacks.pop_front(); handler->run(); return true; } return epoller.handleEvent(); } // ======================================================================================= OwnedPtr newPreferredEventManager() { return newOwned(); } } // namespace ekam ================================================ FILE: src/os/EpollEventManager.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_EPOLLEVENTMANAGER_H_ #define KENTONSCODE_OS_EPOLLEVENTMANAGER_H_ #include #include #include #include #include #include #include #include "EventManager.h" #include "base/OwnedPtr.h" #include "OsHandle.h" #include "ByteStream.h" typedef struct pollfd PollFd; namespace ekam { class EpollEventManager : public RunnableEventManager { public: EpollEventManager(); ~EpollEventManager(); // implements RunnableEventManager ----------------------------------------------------- void loop(); // implements Executor ----------------------------------------------------------------- OwnedPtr runLater(OwnedPtr runnable); // implements EventManager ------------------------------------------------------------- Promise onProcessExit(pid_t pid); OwnedPtr watchFd(int fd); OwnedPtr watchFile(const std::string& filename); private: class AsyncCallbackHandler; class IoWatcherImpl; class IoHandler { public: virtual ~IoHandler() noexcept(false) {} virtual void handle(uint32_t events) = 0; }; class Epoller { public: Epoller(); ~Epoller(); bool handleEvent(); class Watch { public: Watch(Epoller* epoller, OsHandle* handle, uint32_t events, IoHandler* handler); Watch(Epoller* epoller, int fd, uint32_t events, IoHandler* handler); ~Watch(); // Add or remove events being watched. void addEvents(uint32_t eventsToAdd); void removeEvents(uint32_t eventsToRemove); private: friend class Epoller; Epoller* epoller; uint32_t events; uint32_t registeredEvents; int fd; std::string name; IoHandler* handler; void updateRegistration(); }; private: OsHandle epollHandle; int watchCount; std::unordered_set watchesNeedingUpdate; }; class SignalHandler : public IoHandler { public: SignalHandler(Epoller* epoller); ~SignalHandler(); Promise onProcessExit(pid_t pid); // implements IoHandler -------------------------------------------------------------- void handle(uint32_t events); private: class ProcessExitHandler; ByteStream signalStream; Epoller::Watch watch; std::unordered_map processExitHandlerMap; void handleProcessExit(); void maybeStopExpecting(); }; class InotifyHandler : public IoHandler { public: InotifyHandler(Epoller* epoller); ~InotifyHandler(); OwnedPtr watchFile(const std::string& filename); // implements IoHandler -------------------------------------------------------------- void handle(uint32_t events); private: class WatchedDirectory; class FileWatcherImpl; ByteStream inotifyStream; Epoller::Watch watch; OwnedPtrMap ownedWatchDirectories; typedef std::unordered_map WatchMap; WatchMap watchMap; typedef std::unordered_map WatchByNameMap; WatchByNameMap watchByNameMap; }; Epoller epoller; SignalHandler signalHandler; InotifyHandler inotifyHandler; std::deque asyncCallbacks; bool handleEvent(); }; } // namespace ekam #endif // KENTONSCODE_OS_EPOLLEVENTMANAGER_H_ ================================================ FILE: src/os/EventGroup.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "EventGroup.h" #include "base/Debug.h" namespace ekam { EventGroup::ExceptionHandler::~ExceptionHandler() noexcept(false) {} class EventGroup::PendingEvent { public: PendingEvent(EventGroup* group): group(group) { ++group->eventCount; } ~PendingEvent() { if (--group->eventCount == 0) { group->callNoMoreEventsLater(); } } private: EventGroup* group; }; #define HANDLE_EXCEPTIONS(STATEMENT) \ EventGroup* group = this->group; \ try { \ STATEMENT; \ } catch (const std::exception& exception) { \ group->exceptionHandler->threwException(exception); \ } catch (...) { \ group->exceptionHandler->threwUnknownException(); \ } class EventGroup::RunnableWrapper : public Runnable { public: RunnableWrapper(EventGroup* group, OwnedPtr wrapped) : group(group), pendingEvent(group), wrapped(wrapped.release()) {} ~RunnableWrapper() {} // implements Runnable ----------------------------------------------------------------- void run() { HANDLE_EXCEPTIONS(wrapped->run()); } private: EventGroup* group; PendingEvent pendingEvent; OwnedPtr wrapped; }; #undef HANDLE_EXCEPTIONS // ======================================================================================= EventGroup::EventGroup(EventManager* inner, ExceptionHandler* exceptionHandler) : inner(inner), exceptionHandler(exceptionHandler), eventCount(0) {} EventGroup::~EventGroup() {} OwnedPtr EventGroup::runLater(OwnedPtr runnable) { auto wrappedCallback = newOwned(this, runnable.release()); return inner->runLater(wrappedCallback.release()); } Promise EventGroup::onProcessExit(pid_t pid) { Promise innerPromise = inner->onProcessExit(pid); return when(innerPromise, newPendingEvent())( [](ProcessExitCode exitCode, OwnedPtr) -> ProcessExitCode { return exitCode; }); } class EventGroup::IoWatcherWrapper: public EventManager::IoWatcher { public: IoWatcherWrapper(EventGroup* group, OwnedPtr inner) : group(group), inner(inner.release()) {} ~IoWatcherWrapper() {} // implements IoWatcher ---------------------------------------------------------------- Promise onReadable() { Promise innerPromise = inner->onReadable(); return group->when(innerPromise, group->newPendingEvent())( [](Void, OwnedPtr) { // Let PendingEvent die. }); } Promise onWritable() { Promise innerPromise = inner->onWritable(); return group->when(innerPromise, group->newPendingEvent())( [](Void, OwnedPtr) { // Let PendingEvent die. }); } private: EventGroup* group; OwnedPtr inner; }; OwnedPtr EventGroup::watchFd(int fd) { return newOwned(this, inner->watchFd(fd)); } class EventGroup::FileWatcherWrapper: public EventManager::FileWatcher { public: FileWatcherWrapper(EventGroup* group, OwnedPtr inner) : group(group), inner(inner.release()) {} ~FileWatcherWrapper() {} // implements IoWatcher ---------------------------------------------------------------- Promise onChange() { Promise innerPromise = inner->onChange(); return group->when(innerPromise, group->newPendingEvent())( [](FileChangeType changeType, OwnedPtr) -> FileChangeType { // Let PendingEvent die. return changeType; }); } private: EventGroup* group; OwnedPtr inner; }; OwnedPtr EventGroup::watchFile(const std::string& filename) { return newOwned(this, inner->watchFile(filename)); } OwnedPtr EventGroup::newPendingEvent() { return newOwned(this); } void EventGroup::callNoMoreEventsLater() { pendingNoMoreEvents = inner->when()( [this]() { pendingNoMoreEvents.release(); if (eventCount == 0) { DEBUG_INFO << "No more events on EventGroup."; exceptionHandler->noMoreEvents(); } }); } } // namespace ekam ================================================ FILE: src/os/EventGroup.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_EVENTGROUP_H_ #define KENTONSCODE_OS_EVENTGROUP_H_ #include #include "EventManager.h" namespace ekam { // A wrapper around an EventManager which keeps track of all the events being waited on // through it and calls a callback when there is nothing left to do. Additionally, exceptions // thrown by callbacks are caught and reported. // // TODO: Better name? class EventGroup: public EventManager { public: class ExceptionHandler { public: virtual ~ExceptionHandler() noexcept(false); // An event callback threw an exception. The ExceptionHandler is expected to immediately // cancel the event group such that no further events are received by it. virtual void threwException(const std::exception& e) = 0; virtual void threwUnknownException() = 0; // Indicates that this group completed successfully -- it is no longer waiting for anything // and no exception was thrown. virtual void noMoreEvents() = 0; }; EventGroup(EventManager* inner, ExceptionHandler* exceptionHandler); ~EventGroup(); // implements Executor ----------------------------------------------------------------- OwnedPtr runLater(OwnedPtr runnable); // implements EventManager ------------------------------------------------------------- Promise onProcessExit(pid_t pid); OwnedPtr watchFd(int fd); OwnedPtr watchFile(const std::string& filename); private: class PendingEvent; class RunnableWrapper; class IoWatcherWrapper; class FileWatcherWrapper; EventManager* inner; ExceptionHandler* exceptionHandler; int eventCount; Promise pendingNoMoreEvents; OwnedPtr newPendingEvent(); void callNoMoreEventsLater(); }; } // namespace ekam #endif // KENTONSCODE_OS_EVENTGROUP_H_ ================================================ FILE: src/os/EventManager.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "EventManager.h" #include #include "OsHandle.h" // temporary, for toString() namespace ekam { EventManager::~EventManager() noexcept(false) {} EventManager::IoWatcher::~IoWatcher() noexcept(false) {} EventManager::FileWatcher::~FileWatcher() {} RunnableEventManager::~RunnableEventManager() noexcept(false) {} void ProcessExitCode::throwError() { if (signaled) { throw std::logic_error("Process was signaled: " + toString(exitCodeOrSignal)); } else { throw std::logic_error("Process was not signaled. Exit code: " + toString(exitCodeOrSignal)); } } } // namespace ekam ================================================ FILE: src/os/EventManager.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_EVENTMANAGER_H_ #define KENTONSCODE_OS_EVENTMANAGER_H_ #include #include #include #include "base/OwnedPtr.h" #include "base/Promise.h" namespace ekam { class ProcessExitCode { public: ProcessExitCode(): signaled(false), exitCodeOrSignal(0) {} ProcessExitCode(int exitCode) : signaled(false), exitCodeOrSignal(exitCode) {} enum Signaled { SIGNALED }; ProcessExitCode(Signaled, int signalNumber) : signaled(true), exitCodeOrSignal(signalNumber) {} bool wasSignaled() { return signaled; } int getExitCode() { if (signaled) { throwError(); } return exitCodeOrSignal; } int getSignalNumber() { if (!signaled) { throwError(); } return exitCodeOrSignal; } private: bool signaled; int exitCodeOrSignal; void throwError(); }; class EventManager : public Executor { public: virtual ~EventManager() noexcept(false); // Fulfills the promise when the process exits. virtual Promise onProcessExit(pid_t pid) = 0; class IoWatcher { public: virtual ~IoWatcher() noexcept(false); // Fulfills the promise when the file descriptor is readable. virtual Promise onReadable() = 0; // Fulfills the promise when the file descriptor is writable. virtual Promise onWritable() = 0; }; // Watch the file descriptor for readability and writability. virtual OwnedPtr watchFd(int fd) = 0; enum class FileChangeType { MODIFIED, DELETED }; class FileWatcher { public: virtual ~FileWatcher(); virtual Promise onChange() = 0; }; // Watch a file (on disk) for changes or deletion. virtual OwnedPtr watchFile(const std::string& filename) = 0; }; class RunnableEventManager : public EventManager { public: virtual ~RunnableEventManager() noexcept(false); virtual void loop() = 0; }; OwnedPtr newPreferredEventManager(); } // namespace ekam #endif // KENTONSCODE_OS_EVENTMANAGER_H_ ================================================ FILE: src/os/File.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "File.h" #include namespace ekam { File::~File() {} File::DiskRef::~DiskRef() {}; void splitExtension(const std::string& name, std::string* base, std::string* ext) { std::string::size_type pos = name.find_last_of('.'); std::string::size_type slashpos = name.find_last_of('/'); if (pos == std::string::npos || (slashpos != std::string::npos && pos < slashpos)) { base->assign(name); ext->clear(); } else { base->assign(name, 0, pos); ext->assign(name, pos, std::string::npos); } } void recursivelyCreateDirectory(File* location) { if (!location->isDirectory()) { recursivelyCreateDirectory(location->parent().get()); location->createDirectory(); } } } // namespace ekam ================================================ FILE: src/os/File.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_FILE_H_ #define KENTONSCODE_OS_FILE_H_ #include #include #include "base/OwnedPtr.h" namespace ekam { class Hash; class File { public: virtual ~File(); virtual std::string basename() = 0; virtual std::string canonicalName() = 0; virtual OwnedPtr clone() = 0; virtual bool hasParent() = 0; virtual OwnedPtr parent() = 0; virtual bool equals(File* other) = 0; virtual size_t identityHash() = 0; class HashFunc { public: inline size_t operator()(File* file) const { return file->identityHash(); } }; struct EqualFunc { inline bool operator()(File* a, File* b) const { return a->equals(b); } }; class DiskRef { public: virtual ~DiskRef(); virtual const std::string& path() = 0; }; enum Usage { READ, WRITE, UPDATE }; virtual OwnedPtr getOnDisk(Usage usage) = 0; virtual bool exists() = 0; virtual bool isFile() = 0; virtual bool isDirectory() = 0; // File only. virtual Hash contentHash() = 0; virtual std::string readAll() = 0; virtual void writeAll(const std::string& content) = 0; virtual void writeAll(const void* data, int size) = 0; // Directory only. virtual void list(OwnedPtrVector::Appender output) = 0; virtual OwnedPtr relative(const std::string& path) = 0; // Methods that create or delete objects. virtual void createDirectory() = 0; virtual void link(File* target) = 0; virtual void unlink() = 0; }; void splitExtension(const std::string& name, std::string* base, std::string* ext); void recursivelyCreateDirectory(File* location); } // namespace ekam #endif // KENTONSCODE_OS_FILE_H_ ================================================ FILE: src/os/KqueueEventManager.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_KQUEUEEVENTMANAGER_H_ #define KENTONSCODE_OS_KQUEUEEVENTMANAGER_H_ #include #include #include #include "EventManager.h" #include "base/OwnedPtr.h" typedef struct kevent KEvent; namespace ekam { class KqueueEventManager: public RunnableEventManager { public: KqueueEventManager(); ~KqueueEventManager(); // implements RunnableEventManager ----------------------------------------------------- void loop(); // implements EventManager ------------------------------------------------------------- OwnedPtr runAsynchronously(Callback* callback); OwnedPtr onProcessExit(pid_t pid, ProcessExitCallback* callback); OwnedPtr onReadable(int fd, IoCallback* callback); OwnedPtr onWritable(int fd, IoCallback* callback); OwnedPtr onFileChange(const std::string& filename, FileChangeCallback* callback); private: class KEventHandler; class AsyncCallbackHandler; class ProcessExitHandler; class ReadHandler; class WriteHandler; class FileChangeHandler; struct IntptrShortPairHash { inline bool operator()(const std::pair& p) const { return p.first * 65537 + p.second; } }; int kqueueFd; std::deque asyncCallbacks; std::deque fakeEvents; int handlerCount; bool handleEvent(); void updateKqueue(const KEvent& event); void updateKqueue(uintptr_t ident, short filter, u_short flags, KEventHandler* handler = NULL, u_int fflags = 0, intptr_t data = 0); }; } // namespace ekam #endif // KENTONSCODE_OS_KQUEUEEVENTMANAGER_H_ ================================================ FILE: src/os/OsHandle.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "OsHandle.h" #include #include #include #include #include #include #include #include "base/Debug.h" namespace ekam { OsHandle::OsHandle(const std::string& name, int fd) : name(name), fd(fd) { if (fd < 0) { throw std::invalid_argument("Negative file descriptor given for: " + name); } fcntl(fd, F_SETFD, FD_CLOEXEC); } OsHandle::~OsHandle() { if (close(fd) < 0) { DEBUG_ERROR << "close(" << name << "): " << strerror(errno); } } std::string toString(const char* arg) { return arg; } std::string toString(int arg) { std::stringstream sout(std::stringstream::out); sout << arg; return sout.str(); } std::string toString(const OsHandle& arg) { return arg.getName(); } OsError::OsError(const std::string& path, const char* function, int errorNumber) : errorNumber(errorNumber) { if (function != NULL && *function != '\0') { description.append(function); if (!path.empty()) { description.append("("); description.append(path); description.append(")"); } description.append(": "); } else if (!path.empty()) { description.append(path); description.append(": "); } description.append(strerror(errorNumber)); } OsError::OsError(const char* function, int errorNumber) : errorNumber(errorNumber) { if (function != NULL && *function != '\0') { description.append(function); description.append(": "); } description.append(strerror(errorNumber)); } OsError::~OsError() throw() {} const char* OsError::what() const throw() { return description.c_str(); } } // namespace ekam ================================================ FILE: src/os/OsHandle.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_OSHANDLE_H_ #define KENTONSCODE_OS_OSHANDLE_H_ #include #include #include #include "base/OwnedPtr.h" namespace ekam { class EventManager; class OsHandle { public: OsHandle(const std::string& name, int fd); ~OsHandle(); const std::string& getName() const { return name; } int get() const { return fd; } private: const std::string name; const int fd; }; class OsError : public std::exception { public: OsError(const std::string& path, const char* function, int errorNumber); OsError(const char* function, int errorNumber); virtual ~OsError() throw(); virtual const char* what() const throw(); inline int getErrorNumber() const { return errorNumber; } private: std::string description; int errorNumber; }; // TODO(kenton): Factor out toString() module. std::string toString(const char* arg); std::string toString(int arg); std::string toString(const OsHandle& arg); template inline const T& toSyscallArg(const T& value) { return value; } inline int toSyscallArg(const OsHandle& value) { return value.get(); } template long wrapSyscall(const char* name, const Func& func) { long result; do { result = func(); } while (result < 0 && errno == EINTR); if (result < 0) { throw OsError(name, errno); } return result; } template long wrapSyscall(const char* name, const Func& func, Arg1&& arg1, Args&&... args) { long result; do { result = func(toSyscallArg(arg1), toSyscallArg(args)...); } while (result < 0 && errno == EINTR); if (result < 0) { throw OsError(toString(arg1), name, errno); } return result; } // The ## tells GCC to omit the preceding comma if __VA_ARGS__ is empty. This is non-standard, // but apparently MSVC will do the same. #define WRAP_SYSCALL(FUNC, ...) (::ekam::wrapSyscall(#FUNC, ::FUNC, ##__VA_ARGS__)) } // namespace ekam #endif // KENTONSCODE_OS_OSHANDLE_H_ ================================================ FILE: src/os/PollEventManager.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_POLLEVENTMANAGER_H_ #define KENTONSCODE_OS_POLLEVENTMANAGER_H_ #include #include #include #include #include #include "EventManager.h" #include "base/OwnedPtr.h" typedef struct pollfd PollFd; namespace ekam { class PollEventManager : public RunnableEventManager { public: PollEventManager(); ~PollEventManager(); // implements RunnableEventManager ----------------------------------------------------- void loop(); // implements EventManager ------------------------------------------------------------- OwnedPtr runAsynchronously(Callback* callback); OwnedPtr onProcessExit(pid_t pid, ProcessExitCallback* callback); OwnedPtr onReadable(int fd, IoCallback* callback); OwnedPtr onWritable(int fd, IoCallback* callback); OwnedPtr onFileChange(const std::string& filename, FileChangeCallback* callback); private: class IoHandler; class AsyncCallbackHandler; class ProcessExitHandler; class ReadHandler; class WriteHandler; std::deque asyncCallbacks; std::unordered_map processExitHandlerMap; std::unordered_map readHandlerMap; std::unordered_map writeHandlerMap; bool handleEvent(); void handleSignal(const siginfo_t& siginfo); }; } // namespace ekam #endif // KENTONSCODE_OS_POLLEVENTMANAGER_H_ ================================================ FILE: src/os/Socket.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Socket.h" #include #include #include #include #include #include #include #include #include "base/Debug.h" namespace ekam { namespace { std::string splitFirst(std::string* str, char delim) { std::string::size_type pos = str->find_first_of(delim); std::string result; if (pos == std::string::npos) { result = *str; str->clear(); } else { result.assign(*str, 0, pos); str->erase(0, pos + 1); } return result; } bool parseIpAddr(std::string text, struct sockaddr_in* addr) { // TODO: This code sucks. Find a library to call. addr->sin_family = AF_INET; std::string address = splitFirst(&text, ':'); if (text.empty()) return false; std::vector parts; while (!address.empty()) { std::string part = splitFirst(&address, '.'); char* end; parts.push_back(strtoul(part.c_str(), &end, 0)); if (end != part.data() + part.size()) return false; } if (parts.size() > 4) return false; addr->sin_addr.s_addr = 0; if (!parts.empty()) { for (size_t i = 0; i < parts.size() - 1; i++) { if (parts[i] > 0xFFu) return false; addr->sin_addr.s_addr |= parts[i] << ((3 - i) * 8); } if (parts.back() > (0xFFFFFFFFu >> ((parts.size() - 1) * 8))) return false; addr->sin_addr.s_addr |= parts.back(); addr->sin_addr.s_addr = htonl(addr->sin_addr.s_addr); } char* end; unsigned long port = strtoul(text.c_str(), &end, 0); if (end != text.data() + text.size() || port > 0xFFFFu) return false; addr->sin_port = htons(port); return true; } } // namespace ServerSocket::ServerSocket(EventManager* eventManager, const std::string& bindAddress, int backlog) : eventManager(eventManager), handle(bindAddress, WRAP_SYSCALL(socket, AF_INET, SOCK_STREAM, 0)), watcher(eventManager->watchFd(handle.get())) { WRAP_SYSCALL(fcntl, handle, F_SETFL, O_NONBLOCK); int optval = 1; WRAP_SYSCALL(setsockopt, handle, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); struct sockaddr_in addr; if (!parseIpAddr(bindAddress, &addr)) { throw std::invalid_argument("Invalid bind address: " + bindAddress); } WRAP_SYSCALL(bind, handle, reinterpret_cast(&addr), sizeof(addr)); WRAP_SYSCALL(listen, handle, (backlog == 0) ? SOMAXCONN : backlog); } ServerSocket::~ServerSocket() {} Promise> ServerSocket::accept() { return eventManager->when(watcher->onReadable())( [this](Void) -> Promise> { int fd = ::accept(handle.get(), NULL, NULL); if (fd < 0) { switch (errno) { case EINTR: case ECONNABORTED: case EAGAIN: #if EAGAIN != EWOULDBLOCK case EWOULDBLOCK: #endif // This are "normal". Try again. DEBUG_INFO << "accept: " << strerror(errno); return accept(); default: throw OsError("accept", errno); break; } } else { // TODO: Use peer address as name. return newFulfilledPromise(newOwned(fd, "accepted connection")); } }); } } // namespace ekam ================================================ FILE: src/os/Socket.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_SOCKET_H_ #define KENTONSCODE_OS_SOCKET_H_ #include "base/OwnedPtr.h" #include "OsHandle.h" #include "ByteStream.h" #include "EventManager.h" namespace ekam { class ServerSocket { public: ServerSocket(EventManager* eventManager, const std::string& bindAddress, int backlog = 0); ~ServerSocket(); Promise> accept(); private: class AcceptOp; EventManager* eventManager; OsHandle handle; OwnedPtr watcher; }; } // namespace ekam #endif // KENTONSCODE_OS_SOCKET_H_ ================================================ FILE: src/os/Subprocess.cpp ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "Subprocess.h" #include #include #include #include #include #include #include #include #include "OsHandle.h" #include "base/Debug.h" namespace ekam { Subprocess::Subprocess() : doPathLookup(false), pid(-1) {} Subprocess::~Subprocess() { if (pid >= 0) { DEBUG_INFO << "Killing pid: " << pid; // Kill entire progress group. kill(-pid, SIGKILL); int dummy; waitpid(pid, &dummy, 0); } } void Subprocess::addArgument(const std::string& arg) { if (args.empty()) { executableName = arg; doPathLookup = true; } args.push_back(arg); } File::DiskRef* Subprocess::addArgument(File* file, File::Usage usage) { OwnedPtr diskRef = file->getOnDisk(usage); if (args.empty()) { executableName = diskRef->path(); doPathLookup = false; } args.push_back(diskRef->path()); File::DiskRef* result = diskRef.get(); diskRefs.add(diskRef.release()); return result; } OwnedPtr Subprocess::captureStdin() { stdinPipe = newOwned(); return stdinPipe->releaseWriteEnd(); } OwnedPtr Subprocess::captureStdout() { stdoutPipe = newOwned(); stdoutAndStderrPipe.clear(); return stdoutPipe->releaseReadEnd(); } OwnedPtr Subprocess::captureStderr() { stderrPipe = newOwned(); stdoutAndStderrPipe.clear(); return stderrPipe->releaseReadEnd(); } OwnedPtr Subprocess::captureStdoutAndStderr() { stdoutAndStderrPipe = newOwned(); stdoutPipe.clear(); stderrPipe.clear(); return stdoutAndStderrPipe->releaseReadEnd(); } Promise Subprocess::start(EventManager* eventManager) { pid = fork(); if (pid < 0) { throw OsError("", "fork", errno); } else if (pid == 0) { // In child. std::vector argv; std::string command; for (unsigned int i = 0; i < args.size(); i++) { argv.push_back(strdup(args[i].c_str())); if (i > 0) command.push_back(' '); command.append(args[i]); } argv.push_back(NULL); DEBUG_INFO << "exec: " << command; if (stdinPipe != NULL) { stdinPipe->attachReadEndForExec(STDIN_FILENO); } if (stdoutPipe != NULL) { stdoutPipe->attachWriteEndForExec(STDOUT_FILENO); } if (stderrPipe != NULL) { stderrPipe->attachWriteEndForExec(STDERR_FILENO); } if (stdoutAndStderrPipe != NULL) { stdoutAndStderrPipe->attachWriteEndForExec(STDOUT_FILENO); dup2(STDOUT_FILENO, STDERR_FILENO); } // Start a new progress group so that we can kill it all at once. // TODO(someday): This means if you ctrl+C ekam itself, the SIGINT is not distributed to jobs // running under it. Can we fix that? Another thing we could do is put the job into a PID // namespace but that's a lot more work and requires user namespaces and only works on Linux. // Probably what we have to do is handle sigint ourselves and redistribute it to all // children, bleh. setpgid(0, 0); if (doPathLookup) { execvp(executableName.c_str(), &argv[0]); } else { execv(executableName.c_str(), &argv[0]); } perror("exec"); exit(1); } else { if (stdoutPipe != NULL) { stdoutPipe.clear(); } if (stdinPipe != NULL) { stdinPipe.clear(); } if (stderrPipe != NULL) { stderrPipe.clear(); } if (stdoutAndStderrPipe != NULL) { stdoutAndStderrPipe.clear(); } // Set the child's process group ID. The child also does this to itself (see above), but we // need to do it in the parent as well to prevent a race condition in which we end up killing // the child before it manages to call setpgid(). If that happens, then the child will keep // going and the parent process will end up blockend on waitpid(). But the child process will // almost certainly make an RPC to the parent process and wait for a reply, leading to // deadlock. setpgid(pid, 0); return eventManager->when(eventManager->onProcessExit(pid))( [this](ProcessExitCode exitCode) -> ProcessExitCode { pid = -1; return exitCode; }); } } } // namespace ekam ================================================ FILE: src/os/Subprocess.h ================================================ // Ekam Build System // Author: Kenton Varda (kenton@sandstorm.io) // Copyright (c) 2010-2015 Kenton Varda, Google Inc., and contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #ifndef KENTONSCODE_OS_SUBPROCESS_H_ #define KENTONSCODE_OS_SUBPROCESS_H_ #include #include #include "base/OwnedPtr.h" #include "EventManager.h" #include "ByteStream.h" #include "File.h" namespace ekam { class Subprocess { public: Subprocess(); ~Subprocess(); void addArgument(const std::string& arg); File::DiskRef* addArgument(File* file, File::Usage usage); OwnedPtr captureStdin(); OwnedPtr captureStdout(); OwnedPtr captureStderr(); OwnedPtr captureStdoutAndStderr(); Promise start(EventManager* eventManager); private: class CallbackWrapper; std::string executableName; bool doPathLookup; std::vector args; OwnedPtrVector diskRefs; OwnedPtr stdinPipe; OwnedPtr stdoutPipe; OwnedPtr stderrPipe; OwnedPtr stdoutAndStderrPipe; pid_t pid; }; } // namespace ekam #endif // KENTONSCODE_OS_SUBPROCESS_H_ ================================================ FILE: vscode/.gitignore ================================================ out node_modules package-lock.json *.vsix ================================================ FILE: vscode/LICENSE ================================================ The MIT License (MIT) Copyright (c) 2018 Kenton Varda Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: vscode/README.md ================================================ # Ekam VS Code Plugin Brings error reports from Ekam into Visual Studio Code. ## Usage The extension will look for `ekam-langserve` in your `PATH`, or you can specify its location in your config like: ```json { "ekam.path": "/absolute/path/to/ekam-langserve", "ekam.args": [ "localhost:41315" ] } ``` To obtain `ekam-langserve` binary, build [Ekam](https://github.com/capnproto/ekam). The binary will end up in `bin/ekam-langserve` after the build completes. The language server expects to connect to a local Ekam run. You'll need to tell Ekam to publish logs on a local port by running it like: ekam -c -n :41315 ## Building from source ```bash npm install npm run postinstall npm run package ``` This builds `vscode-ekam.vsix`, which you can then install into VS Code. ================================================ FILE: vscode/package.json ================================================ { "name": "vscode-ekam", "displayName": "vscode-ekam", "description": "Ekam Language Server", "version": "0.2.0", "publisher": "kentonv", "homepage": "https://github.com/sandstorm-io/ekam", "engines": { "vscode": "^1.27.0" }, "keywords": [ "LSP", "Ekam" ], "activationEvents": [ "onLanguage:cpp" ], "main": "./out/src/extension", "scripts": { "vscode:prepublish": "tsc -p ./", "compile": "tsc -watch -p ./", "package": "vsce package" }, "dependencies": { "vscode-languageclient": "6.x", "vscode-languageserver": "6.x" }, "devDependencies": { "typescript": "^4.9.4", "@types/vscode": "^1.1.0", "@types/node": "^6.0.40", "vsce": "^1.51.0" }, "repository": { "type": "git", "url": "http://github.com/sandstorm-io/ekam" }, "contributes": { "configuration": { "type": "object", "title": "ekam configuration", "properties": { "ekam.path": { "type": "string", "default": "ekam-langserve", "description": "The path to ekam-langserve executable, e.g.: /usr/bin/ekam-langserve" }, "ekam.arguments": { "type": "array", "default": [ "localhost:41315" ], "items": { "type": "string" }, "description": "Arguments for ekam-lsp server; should specify host:port of Ekam" } } } } } ================================================ FILE: vscode/src/extension.ts ================================================ import * as vscode from 'vscode'; import * as vscodelc from 'vscode-languageclient'; /** * Method to get workspace configuration option * @param option name of the option (e.g. for ekam.path should be path) * @param defaultValue default value to return if option is not set */ function getConfig(option: string, defaultValue?: any): T { const config = vscode.workspace.getConfiguration('ekam'); return config.get(option, defaultValue); } /** * this method is called when your extension is activate * your extension is activated the very first time the command is executed */ export function activate(context: vscode.ExtensionContext) { const syncFileEvents = getConfig('syncFileEvents', true); const options: vscodelc.Executable = { command: getConfig('path'), args: getConfig('arguments') }; const serverOptions: vscodelc.ServerOptions = options; const clientOptions: vscodelc.LanguageClientOptions = { documentSelector: [{ scheme: 'file' }] }; const ekamClient = new vscodelc.LanguageClient('Ekam Language Server', serverOptions, clientOptions); console.log('Ekam Language Server is now active!'); const disposable = ekamClient.start(); context.subscriptions.push(disposable); } ================================================ FILE: vscode/tsconfig.json ================================================ { "compilerOptions": { "module": "commonjs", "target": "es6", "outDir": "out", "lib": [ "es6", "es2015.core", "es2015.collection", "es2015.generator", "es2015.iterable", "es2015.promise", "es2015.symbol", "es2016.array.include" ], "sourceMap": true, "rootDir": ".", "alwaysStrict": true, "noEmitOnError": true, "noFallthroughCasesInSwitch": true, "noImplicitAny": true, "noImplicitReturns": true, "noImplicitThis": true }, "exclude": [ "node_modules", ".vscode-test" ] }