Repository: src-d/sourced-ce Branch: master Commit: 0399d8cbd987 Files: 63 Total size: 213.5 KB Directory structure: gitextract_wj_qjlin/ ├── .gitbook.yaml ├── .github/ │ ├── CODE_OF_CONDUCT.md │ └── pull_request_template.md ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── DCO ├── LICENSE.md ├── MAINTAINERS ├── Makefile ├── NOTICE.md ├── README.md ├── cmd/ │ └── sourced/ │ ├── cmd/ │ │ ├── compose.go │ │ ├── init.go │ │ ├── logs.go │ │ ├── prune.go │ │ ├── restart.go │ │ ├── root.go │ │ ├── sql.go │ │ ├── start.go │ │ ├── status.go │ │ ├── stop.go │ │ └── web.go │ ├── compose/ │ │ ├── compose.go │ │ ├── file/ │ │ │ └── file.go │ │ └── workdir/ │ │ ├── env_file_test.go │ │ ├── factory.go │ │ ├── factory_test.go │ │ ├── handler.go │ │ ├── handler_test.go │ │ ├── workdir.go │ │ └── workdir_test.go │ ├── dir/ │ │ ├── dir.go │ │ └── dir_test.go │ ├── format/ │ │ └── colors.go │ ├── main.go │ └── release/ │ ├── release.go │ └── release_test.go ├── docker-compose.yml ├── docs/ │ ├── CONTRIBUTING.md │ ├── README.md │ ├── learn-more/ │ │ ├── architecture.md │ │ ├── faq.md │ │ └── troubleshooting.md │ ├── quickstart/ │ │ ├── 1-install-requirements.md │ │ ├── 2-install-sourced.md │ │ ├── 3-init-sourced.md │ │ ├── 4-explore-sourced.md │ │ └── README.md │ └── usage/ │ ├── README.md │ ├── bblfsh.md │ ├── commands.md │ ├── examples.md │ └── multiple-datasets.md ├── go.mod ├── go.sum ├── run-integration-tests.bat └── test/ ├── commander.go ├── common.go ├── compose_test.go ├── init_local_test.go ├── init_orgs_test.go └── superset.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitbook.yaml ================================================ root: ./docs structure: readme: ../README.md summary: README.md ================================================ FILE: .github/CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at conduct@sourced.tech. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org ================================================ FILE: .github/pull_request_template.md ================================================ --- * [ ] I have updated the CHANGELOG file according to the conventions in [keepachangelog.com](https://keepachangelog.com) * [ ] This PR contains changes that do not require a mention in the CHANGELOG file ================================================ FILE: .gitignore ================================================ .ci/ build/ ================================================ FILE: .travis.yml ================================================ branches: only: - master - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ dist: xenial sudo: required language: go go_import_path: github.com/src-d/sourced-ce go: - 1.13.x env: global: - SOURCED_GITHUB_TOKEN=$GITHUB_TOKEN matrix: fast_finish: true services: - docker stages: - name: tests - name: release if: tag IS present jobs: include: - stage: tests name: 'Go Unit Tests' script: - make packages - make test-coverage codecov - stage: tests name: 'Integration Tests Linux' script: # cannot use 'make test-integration' because 'make clean' fails with # GO111MODULE, see https://github.com/golang/go/issues/31002 - make build - make test-integration-no-build - stage: release name: 'Release to GitHub' script: - make packages deploy: provider: releases api_key: $GITHUB_TOKEN file_glob: true file: - build/*.tar.gz skip_cleanup: true on: all_branches: true ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). The changes listed under `Unreleased` section have landed in master but are not yet released. ## [Unreleased] ### Components - `bblfsh/bblfshd` has been updated to [v2.15.0](https://github.com/bblfsh/bblfshd/releases/tag/v2.15.0). - `bblfsh/web` has been updated to [v0.11.4](https://github.com/bblfsh/web/releases/tag/v0.11.4). - Use the same logging level as the other components reading `LOG_LEVEL` enviroment value (default: `info`) (([#263](https://github.com/src-d/sourced-ce/pull/263)). - `srcd/sourced-ui` has been updated to [v0.8.1](https://github.com/src-d/sourced-ui/releases/tag/v0.8.1). ### Fixed - Identify and show errors for old unsupported version of docker/docker-compose ([#253](https://github.com/src-d/sourced-ce/issues/253)) ## [v0.17.0](https://github.com/src-d/sourced-ce/releases/tag/v0.17.0) - 2019-10-01 ### Components - `srcd/sourced-ui` has been updated to [v0.7.0](https://github.com/src-d/sourced-ui/releases/tag/v0.7.0). - `srcd/gitcollector` has been updated to [v0.0.4](https://github.com/src-d/gitcollector/releases/tag/v0.0.4). ### Fixed - More detailed error messages for file downloads ([#245](https://github.com/src-d/sourced-ce/pull/245)). ### Changed - Make `sourced-ui` Superset celery workers run as separate containers ([#269](https://github.com/src-d/sourced-ui/issues/269)). - Remove need for `docker-compose.override.yml` ([#252](https://github.com/src-d/sourced-ui/issues/252)). ### Internal - Development and building of source{d} CE now requires `go 1.13` ([#242](https://github.com/src-d/sourced-ce/pull/242)). ### Upgrading Install the new `v0.17.0` binary, then run `sourced compose download`. Because of a change in the `docker-compose.yml` file version, you must delete the file `~/.sourced/compose-files/__active__/docker-compose.override.yml` manually. If you had a deployment running, you must re-deploy the containers with `sourced restart`. All your existing data will continue to work after the upgrade. ```shell $ sourced version sourced version v0.16.0 $ rm ~/.sourced/compose-files/__active__/docker-compose.override.yml $ sourced compose download Docker compose file successfully downloaded to your ~/.sourced/compose-files directory. It is now the active compose file. To update your current installation use `sourced restart` $ sourced restart ``` ## [v0.16.0](https://github.com/src-d/sourced-ce/releases/tag/v0.16.0) - 2019-09-16 ### Components - `srcd/sourced-ui` has been updated to [v0.6.0](https://github.com/src-d/sourced-ui/releases/tag/v0.6.0). - `bblfsh/web` has been updated to [v0.11.3](https://github.com/bblfsh/web/releases/tag/v0.11.3). ### Fixed - Increase the timeout for the `start` command ([#219](https://github.com/src-d/sourced-ce/pull/219)). ### Changed - `sourced compose list` shows an index number for each compose entry, and `sourced compose set` now accepts both the name or the index number (@cmbahadir) ([#199](https://github.com/src-d/sourced-ce/issues/199)). ### Upgrading Install the new `v0.16.0` binary, then run `sourced compose download`. If you had a deployment running, you can re-deploy the containers with `sourced restart`. Please note: `sourced-ui` contains changes to the color palettes for the default dashboard charts, and these changes will only be visible when you run `sourced init local/org` with a new path or organization. This is a cosmetic improvement that you can ignore safely. If you want to apply the new default dashboards over your existing deployment, you will need to run `sourced prune` (or `sourced prune --all`) and `sourced init local/org` again. Important: running `prune` will delete all your current data and customizations, including charts or dashboards. You can choose to not `prune` your existing deployments, keeping you previous default dashboards and charts. ```shell $ sourced version sourced version v0.16.0 $ sourced compose download Docker compose file successfully downloaded to your ~/.sourced/compose-files directory. It is now the active compose file. To update your current installation use `sourced restart` $ sourced status workdirs bblfsh * src-d $ sourced prune --all $ sourced init orgs src-d $ sourced init orgs bblfsh ``` ## [v0.15.1](https://github.com/src-d/sourced-ce/releases/tag/v0.15.1) - 2019-08-27 ### Fixed - Fix incompatibility of empty resource limits ([#227](https://github.com/src-d/sourced-ce/issues/227)). - Fix incorrect value for `GITCOLLECTOR_LIMIT_CPU` in some cases ([#225](https://github.com/src-d/sourced-ce/issues/225)). - Fix gitbase `LOG_LEVEL` environment variable in the compose file ([#228](https://github.com/src-d/sourced-ce/issues/228)). ### Removed - Remove the `completion` sub-command on Windows, as it only works for bash ([#169](https://github.com/src-d/sourced-ce/issues/169)). ### Upgrading Install the new `v0.15.1` binary, then run `sourced compose download`. For an upgrade from `v0.15.0`, you just need to run `sourced restart` to re-deploy the containers. For an upgrade from `v0.14.0`, please see the upgrade instructions in the release notes for `v0.15.0`. ## [v0.15.0](https://github.com/src-d/sourced-ce/releases/tag/v0.15.0) - 2019-08-21 ### Components - `srcd/sourced-ui` has been updated to [v0.5.0](https://github.com/src-d/sourced-ui/releases/tag/v0.5.0). - `srcd/ghsync` has been updated to [v0.2.0](https://github.com/src-d/ghsync/releases/tag/v0.2.0). ### Added - Add a monitoring of containers state while waiting for the web UI to open during initialization ([#147](https://github.com/src-d/sourced-ce/issues/147)). - Exclude forks by default in `sourced init orgs`, adding a new flag `--with-forks` to include them if needed ([#109](https://github.com/src-d/sourced-ce/issues/109)). ### Changed - Refactor of the `status` command ([#203](https://github.com/src-d/sourced-ce/issues/203)): - `sourced status components` shows the previous output of `sourced status` - `sourced status workdirs` replaces `sourced workdirs` - `sourced status config` shows the contents of the Docker Compose environment variables. This is useful, for example, to check if the active working directory was configured to include or skip forks when downloading the data from GitHub - `sourced status all` shows all of the above ### Upgrading Install the new `v0.15.0` binary, then run `sourced compose download`. If you had a deployment running, you can re-deploy the containers with `sourced restart`. Please note: `sourced-ui` contains fixes for the default dashboard charts that will only be visible when you run `sourced init local/org` with a new path or organization. If you want to apply the new default dashboards over your existing deployment, you will need to run `sourced prune` (or `sourced prune --all`) and `sourced init local/org` again. Important: running `prune` will delete all your current data and customizations, including charts or dashboards. You can choose to not `prune` your existing deployments, keeping you previous default dashboards and charts. ```shell $ sourced version sourced version v0.15.0 build 08-21-2019_08_30_24 $ sourced compose download Docker compose file successfully downloaded to your ~/.sourced/compose-files directory. It is now the active compose file. To update your current installation use `sourced restart` $ sourced status workdirs bblfsh * src-d $ sourced prune --all $ sourced init orgs src-d $ sourced init orgs bblfsh ``` ## [v0.14.0](https://github.com/src-d/sourced-ce/releases/tag/v0.14.0) - 2019-08-07 Initial release of **source{d} Community Edition (CE)**, the data platform for your software development life cycle. The `sourced` binary is a wrapper for Docker Compose that downloads the `docker-compose.yml` file from this repository, and includes the following sub commands: - `init`: Initialize source{d} to work on local or GitHub organization datasets - `local`: Initialize source{d} to analyze local repositories - `orgs`: Initialize source{d} to analyze GitHub organizations - `status`: Show the status of all components - `stop`: Stop any running components - `start`: Start any stopped components - `logs`: Show logs from components - `web`: Open the web interface in your browser - `sql`: Open a MySQL client connected to a SQL interface for Git - `prune`: Stop and remove components and resources - `workdirs` List all working directories - `compose`: Manage source{d} docker compose files - `download`: Download docker compose files - `list`: List the downloaded docker compose files - `set`: Set the active docker compose file - `restart`: Update current installation according to the active docker compose file ### Known Issues - On Windows, if you use `sourced init local` on a directory with a long path, you may encounter the following error: ``` Can't find a suitable configuration file in this directory or any parent. Are you in the right directory? ``` This is caused by the [`MAX_PATH` limitation on windows](https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation). The only workaround is to move the target directory to a shorter path, closer to the root of your drive ([#191](https://github.com/src-d/sourced-ce/issues/191)). - Linux only: Docker installed from snap packages is not supported, please install it following [the official documentation](https://docs.docker.com/install/) ([#78](https://github.com/src-d/sourced-ce/issues/78)). ### Upgrading For internal releases we don't support upgrading. If you have a previous `sourced-ce` pre-release version installed, clean up all your data **before** downloading this release. This will delete everything, including the UI data for dashboards, charts, users, etc: ```shell sourced prune --all rm -rf ~/.sourced ``` ================================================ FILE: DCO ================================================ Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 660 York Street, Suite 102, San Francisco, CA 94110 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ================================================ FILE: LICENSE.md ================================================ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) 2017 Sourced Technologies, S.L. This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . ================================================ FILE: MAINTAINERS ================================================ David Pordomingo (@dpordomingo) Lou Marvin Caraig (@se7entyse7en) ================================================ FILE: Makefile ================================================ # Package configuration PROJECT = sourced-ce COMMANDS = cmd/sourced PKG_OS ?= darwin linux windows # Including ci Makefile CI_REPOSITORY ?= https://github.com/src-d/ci.git CI_PATH ?= $(shell pwd)/.ci CI_VERSION ?= v1 MAKEFILE := $(CI_PATH)/Makefile.main $(MAKEFILE): git clone --quiet --branch $(CI_VERSION) --depth 1 $(CI_REPOSITORY) $(CI_PATH); -include $(MAKEFILE) GOTEST_BASE = go test -v -timeout 20m -parallel 1 -count 1 -ldflags "$(LD_FLAGS)" GOTEST_INTEGRATION = $(GOTEST_BASE) -tags="forceposix integration" OS := $(shell uname) # override clean target from CI to avoid executing `go clean` # see https://github.com/src-d/sourced-ce/pull/154 clean: rm -rf $(BUILD_PATH) $(BIN_PATH) $(VENDOR_PATH) ifeq ($(OS),Darwin) test-integration-clean: $(eval TMPDIR_INTEGRATION_TEST := $(PWD)/integration-test-tmp) $(eval GOTEST_INTEGRATION := TMPDIR=$(TMPDIR_INTEGRATION_TEST) $(GOTEST_INTEGRATION)) rm -rf $(TMPDIR_INTEGRATION_TEST) mkdir $(TMPDIR_INTEGRATION_TEST) else test-integration-clean: endif test-integration-no-build: test-integration-clean $(GOTEST_INTEGRATION) github.com/src-d/sourced-ce/test/ test-integration: clean build test-integration-no-build ================================================ FILE: NOTICE.md ================================================ sourced-ce is the data platform for your software development life cycle Copyright (C) 2019 source{d} This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ================================================ FILE: README.md ================================================ source{d} Community Edition (CE) **source{d} Community Edition (CE) is the data platform for your software development life cycle.** [![GitHub version](https://badge.fury.io/gh/src-d%2Fsourced-ce.svg)](https://github.com/src-d/sourced-ce/releases) [![Build Status](https://travis-ci.com/src-d/sourced-ce.svg?branch=master)](https://travis-ci.com/src-d/sourced-ce) ![Beta](https://svg-badge.appspot.com/badge/stability/beta?color=D6604A) [![Go Report Card](https://goreportcard.com/badge/github.com/src-d/sourced-ce)](https://goreportcard.com/report/github.com/src-d/sourced-ce) [![GoDoc](https://godoc.org/github.com/src-d/sourced-ce?status.svg)](https://godoc.org/github.com/src-d/sourced-ce) [Website](https://www.sourced.tech) • [Documentation](https://docs.sourced.tech/community-edition) • [Blog](https://blog.sourced.tech) • [Slack](http://bit.ly/src-d-community) • [Twitter](https://twitter.com/sourcedtech) ![source{d} CE dashboard](docs/assets/dashboard.png) ## Introduction **source{d} Community Edition (CE)** helps you to manage all your code and engineering data in one place: - **Code Retrieval**: Retrieve and store the git history of the code of your organization as a dataset. - **Analysis in/for any Language**: Automatically identify languages, parse source code, and extract the pieces that matter in a language-agnostic way. - **History Analysis**: Extract information from the evolution, commits, and metadata of your codebase and from GitHub, generating detailed reports and insights. - **Familiar APIs**: Analyze your code through powerful SQL queries. Use tools you're familiar with to create reports and dashboards. This repository contains the code of **source{d} Community Edition (CE)** and its project documentation, which you can also see properly rendered at [docs.sourced.tech/community-edition](https://docs.sourced.tech/community-edition). ### Contents - [Introduction](README.md#introduction) - [Quick Start](README.md#quick-start) - [Architecture](README.md#architecture) - [Contributing](README.md#contributing) - [Community](README.md#community) - [Code of Conduct](README.md#code-of-conduct) - [License](README.md#license) ## Quick Start **source{d} CE** supports Linux, macOS, and Windows. To run it you only need: 1. To have Docker installed in your PC 1. Download `sourced` binary (for your OS) from [our releases](https://github.com/src-d/sourced-ce/releases) 1. Run it: ```bash $ sourced init orgs --token= ``` And log in into http://127.0.0.1:8088 with login: `admin`, and password: `admin`. If you want more details of each step, you will find in the [**Quick Start Guide**](docs/quickstart/README.md) all the steps to get started with **source{d} CE**, from the installation of its dependencies to running SQL queries to inspect git repositories. If you want to know more about **source{d} CE**, in the [next steps](docs/usage/README.md) section you will find some useful resources for guiding your experience using this tool. If you have any problem running **source{d} CE** you can take a look at our [Frequently Asked Questions](docs/learn-more/troubleshooting.md) or [Troubleshooting](docs/learn-more/troubleshooting.md) sections. You can also ask for help when using **source{d} CE** in our [source{d} Forum](https://forum.sourced.tech). If you spotted a bug, or you have a feature request, please [open an issue](https://github.com/src-d/sourced-ce/issues) to let us know about it. ## Architecture _For more details on the architecture of this project, read [docs/learn-more/architecture.md](docs/learn-more/architecture.md)._ **source{d} CE** is deployed as Docker containers, using Docker Compose. This tool is a wrapper for Docker Compose to manage the compose files and its containers easily. Moreover, `sourced` does not require a local installation of Docker Compose, if it is not found it will be deployed inside a container. The main entry point of **source{d} CE** is [sourced-ui](https://github.com/src-d/sourced-ui), the web interface from where you can access your data, create dashboards, run queries... The data exposed by the web interface is prepared and processed by the following services: - [babelfish](https://doc.bblf.sh): universal code parser. - [gitcollector](https://github.com/src-d/gitcollector): fetches the git repositories owned by your organization. - [ghsync](https://github.com/src-d/ghsync): fetches metadata from GitHub (users, pull requests, issues...). - [gitbase](https://github.com/src-d/gitbase): SQL database interface to Git repositories. ## Contributing [Contributions](https://github.com/src-d/sourced-ce/issues) are **welcome and very much appreciated** 🙌 Please refer to [our Contribution Guide](docs/CONTRIBUTING.md) for more details. ## Community source{d} has an amazing community of developers and contributors who are interested in Code As Data and/or Machine Learning on Code. Please join us! 👋 - [Community](https://sourced.tech/community/) - [Slack](http://bit.ly/src-d-community) - [Twitter](https://twitter.com/sourcedtech) - [Email](mailto:hello@sourced.tech) ## Code of Conduct All activities under source{d} projects are governed by the [source{d} code of conduct](https://github.com/src-d/guide/blob/master/.github/CODE_OF_CONDUCT.md). ## License GPL v3.0, see [LICENSE](LICENSE.md). ================================================ FILE: cmd/sourced/cmd/compose.go ================================================ package cmd import ( "fmt" "strconv" composefile "github.com/src-d/sourced-ce/cmd/sourced/compose/file" "gopkg.in/src-d/go-cli.v0" ) type composeCmd struct { cli.PlainCommand `name:"compose" short-description:"Manage source{d} docker compose files" long-description:"Manage source{d} docker compose files"` } type composeDownloadCmd struct { Command `name:"download" short-description:"Download docker compose files" long-description:"Download docker compose files. By default the command downloads the file for this binary version.\n\nUse the 'version' argument to choose a specific revision from\nthe https://github.com/src-d/sourced-ce repository, or to set a\nURL to a docker-compose.yml file.\n\nExamples:\n\nsourced compose download\nsourced compose download v0.0.1\nsourced compose download master\nsourced compose download https://raw.githubusercontent.com/src-d/sourced-ce/master/docker-compose.yml"` Args struct { Version string `positional-arg-name:"version" description:"Either a revision (tag, full sha1) or a URL to a docker-compose.yml file"` } `positional-args:"yes"` } func (c *composeDownloadCmd) Execute(args []string) error { v := c.Args.Version if v == "" { v = version } err := composefile.ActivateFromRemote(v) if err != nil { return err } fmt.Println("Docker compose file successfully downloaded to your ~/.sourced/compose-files directory. It is now the active compose file.") fmt.Println("To update your current installation use `sourced restart`") return nil } type composeListCmd struct { Command `name:"list" short-description:"List the downloaded docker compose files" long-description:"List the downloaded docker compose files"` } func (c *composeListCmd) Execute(args []string) error { active, err := composefile.Active() if err != nil { return err } files, err := composefile.List() if err != nil { return err } for index, file := range files { fmt.Printf("[%d]", index) if file == active { fmt.Printf("* %s\n", file) } else { fmt.Printf(" %s\n", file) } } return nil } type composeSetDefaultCmd struct { Command `name:"set" short-description:"Set the active docker compose file" long-description:"Set the active docker compose file"` Args struct { File string `positional-arg-name:"index/name" description:"Provide name or index of compose file on 'sourced compose list'"` } `positional-args:"yes" required:"yes"` } func (c *composeSetDefaultCmd) Execute(args []string) error { files, err := composefile.List() if err != nil { return err } index, err := strconv.Atoi(c.Args.File) if err == nil { if index >= 0 && index < len(files) { active := files[index] err = composefile.SetActive(active) } else { return fmt.Errorf("Index is out of range, please check the output of 'sourced compose list'") } } else { err := composefile.SetActive(c.Args.File) if err != nil { return err } } fmt.Println("Active docker compose file was changed successfully.") fmt.Println("To update your current installation use `sourced restart`") return nil } func init() { c := rootCmd.AddCommand(&composeCmd{}) c.AddCommand(&composeDownloadCmd{}) c.AddCommand(&composeListCmd{}) c.AddCommand(&composeSetDefaultCmd{}) } ================================================ FILE: cmd/sourced/cmd/init.go ================================================ package cmd import ( "context" "fmt" "net/http" "os" "path/filepath" "strings" "time" "github.com/src-d/sourced-ce/cmd/sourced/compose" "github.com/src-d/sourced-ce/cmd/sourced/compose/workdir" "gopkg.in/src-d/go-cli.v0" "github.com/pkg/errors" ) type initCmd struct { cli.PlainCommand `name:"init" short-description:"Initialize source{d} to work on local or GitHub orgs datasets" long-description:"Initialize source{d} to work on local or Github orgs datasets"` } type initLocalCmd struct { Command `name:"local" short-description:"Initialize source{d} to analyze local repositories" long-description:"Install, initialize, and start all the required docker containers, networks, volumes, and images.\n\nThe repos directory argument must point to a directory containing git repositories.\nIf it's not provided, the current working directory will be used."` Args struct { Reposdir string `positional-arg-name:"workdir"` } `positional-args:"yes"` } func (c *initLocalCmd) Execute(args []string) error { wdHandler, err := workdir.NewHandler() if err != nil { return err } reposdir, err := c.reposdirArg() if err != nil { return err } wd, err := workdir.InitLocal(reposdir) if err != nil { return err } if err := activate(wdHandler, wd); err != nil { return err } return OpenUI(60 * time.Minute) } func (c *initLocalCmd) reposdirArg() (string, error) { reposdir := c.Args.Reposdir reposdir = strings.TrimSpace(reposdir) var err error if reposdir == "" { reposdir, err = os.Getwd() } else { reposdir, err = filepath.Abs(reposdir) } if err != nil { return "", errors.Wrap(err, "could not get directory") } info, err := os.Stat(reposdir) if err != nil || !info.IsDir() { return "", fmt.Errorf("path '%s' is not a valid directory", reposdir) } return reposdir, nil } type initOrgsCmd struct { Command `name:"orgs" short-description:"Initialize source{d} to analyze GitHub organizations" long-description:"Install, initialize, and start all the required docker containers, networks, volumes, and images.\n\nThe orgs argument must a comma-separated list of GitHub organization names to be analyzed."` Token string `short:"t" long:"token" env:"SOURCED_GITHUB_TOKEN" description:"GitHub token for the passed organizations. It should be granted with 'repo' and 'read:org' scopes." required:"true"` WithForks bool `long:"with-forks" description:"Download GitHub forked repositories"` Args struct { Orgs []string `required:"yes"` } `positional-args:"yes" required:"1"` } func (c *initOrgsCmd) Execute(args []string) error { wdHandler, err := workdir.NewHandler() if err != nil { return err } orgs := c.orgsList() if err := c.validate(orgs); err != nil { return err } wd, err := workdir.InitOrgs(orgs, c.Token, c.WithForks) if err != nil { return err } if err := activate(wdHandler, wd); err != nil { return err } return OpenUI(60 * time.Minute) } // allows to pass organizations separated not only by a space // but by comma as well func (c *initOrgsCmd) orgsList() []string { orgs := c.Args.Orgs if len(c.Args.Orgs) == 1 { orgs = strings.Split(c.Args.Orgs[0], ",") } for i, org := range orgs { orgs[i] = strings.Trim(org, " ,") } return orgs } func (c *initOrgsCmd) validate(orgs []string) error { client := &http.Client{Transport: &authTransport{token: c.Token}} r, err := client.Get("https://api.github.com/user") if err != nil { return errors.Wrapf(err, "could not validate user token") } if r.StatusCode == http.StatusUnauthorized { return fmt.Errorf("github token is not valid") } for _, org := range orgs { r, err := client.Get("https://api.github.com/orgs/" + org) if err != nil { return errors.Wrapf(err, "could not validate organization") } if r.StatusCode == http.StatusNotFound { return fmt.Errorf("organization '%s' is not found", org) } } return nil } func activate(wdHandler *workdir.Handler, workdir *workdir.Workdir) error { // Before setting a new workdir, stop the current containers compose.Run(context.Background(), "stop") err := wdHandler.SetActive(workdir) if err != nil { return err } fmt.Printf("docker-compose working directory set to %s\n", workdir.Path) return compose.Run(context.Background(), "up", "--detach") } type authTransport struct { token string } func (t *authTransport) RoundTrip(r *http.Request) (*http.Response, error) { r.Header.Set("Authorization", "token "+t.token) return http.DefaultTransport.RoundTrip(r) } func init() { c := rootCmd.AddCommand(&initCmd{}) c.AddCommand(&initOrgsCmd{}) c.AddCommand(&initLocalCmd{}) } ================================================ FILE: cmd/sourced/cmd/logs.go ================================================ package cmd import ( "context" "github.com/src-d/sourced-ce/cmd/sourced/compose" ) type logsCmd struct { Command `name:"logs" short-description:"Fetch the logs of source{d} components" long-description:"Fetch the logs of source{d} components"` Follow bool `short:"f" long:"follow" description:"Follow log output"` Args struct { Components []string `positional-arg-name:"component" description:"Component names from where to fetch logs"` } `positional-args:"yes"` } func (c *logsCmd) Execute(args []string) error { command := []string{"logs"} if c.Follow { command = append(command, "--follow") } if components := c.Args.Components; len(components) > 0 { command = append(command, components...) } return compose.Run(context.Background(), command...) } func init() { rootCmd.AddCommand(&logsCmd{}) } ================================================ FILE: cmd/sourced/cmd/prune.go ================================================ package cmd import ( "context" "github.com/src-d/sourced-ce/cmd/sourced/compose" "github.com/src-d/sourced-ce/cmd/sourced/compose/workdir" ) type pruneCmd struct { Command `name:"prune" short-description:"Stop and remove components and resources" long-description:"Stops containers and removes containers, networks, volumes and configuration created by 'init' for the current working directory.\nTo delete resources for all working directories pass --all flag.\nImages are not deleted unless you specify the --images flag."` All bool `short:"a" long:"all" description:"Remove containers and resources for all working directories"` Images bool `long:"images" description:"Remove docker images"` } func (c *pruneCmd) Execute(args []string) error { workdirHandler, err := workdir.NewHandler() if err != nil { return err } if !c.All { return c.pruneActive(workdirHandler) } wds, err := workdirHandler.List() if err != nil { return err } for _, wd := range wds { if err := workdirHandler.SetActive(wd); err != nil { return err } if err = c.pruneActive(workdirHandler); err != nil { return err } } return nil } func (c *pruneCmd) pruneActive(workdirHandler *workdir.Handler) error { a := []string{"down", "--volumes"} if c.Images { a = append(a, "--rmi", "all") } if err := compose.Run(context.Background(), a...); err != nil { return err } wd, err := workdirHandler.Active() if err != nil { return err } if err := workdirHandler.Remove(wd); err != nil { return err } return workdirHandler.UnsetActive() } func init() { rootCmd.AddCommand(&pruneCmd{}) } ================================================ FILE: cmd/sourced/cmd/restart.go ================================================ package cmd import ( "context" "github.com/src-d/sourced-ce/cmd/sourced/compose" ) type restartCmd struct { Command `name:"restart" short-description:"Update current installation according to the active docker compose file" long-description:"Update current installation according to the active docker compose file. It only recreates the component containers, keeping all your data, as charts, dashboards, repositories and GitHub metadata."` } func (c *restartCmd) Execute(args []string) error { return compose.Run(context.Background(), "up", "--force-recreate", "--detach") } func init() { rootCmd.AddCommand(&restartCmd{}) } ================================================ FILE: cmd/sourced/cmd/root.go ================================================ package cmd import ( "fmt" "os" "runtime" "github.com/src-d/sourced-ce/cmd/sourced/compose" "github.com/src-d/sourced-ce/cmd/sourced/compose/file" "github.com/src-d/sourced-ce/cmd/sourced/compose/workdir" "github.com/src-d/sourced-ce/cmd/sourced/dir" "github.com/src-d/sourced-ce/cmd/sourced/format" "gopkg.in/src-d/go-cli.v0" ) const name = "sourced" var version = "master" var rootCmd = cli.NewNoDefaults(name, "source{d} Community Edition & Enterprise Edition CLI client") // Init sets the version rewritten by the CI build and adds default sub commands func Init(v, build string) { version = v rootCmd.AddCommand(&cli.VersionCommand{ Name: name, Version: version, Build: build, }) if runtime.GOOS != "windows" { rootCmd.AddCommand(&cli.CompletionCommand{ Name: name, }, cli.InitCompletionCommand(name)) } } // Command implements the default group flags. It is meant to be embedded into // other application commands to provide default behavior for logging, config type Command struct { cli.PlainCommand cli.LogOptions `group:"Log Options"` } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { if err := dir.Prepare(); err != nil { fmt.Println(err) log(err) os.Exit(1) } if err := rootCmd.Run(os.Args); err != nil { log(err) os.Exit(1) } } func log(err error) { switch { case workdir.ErrMalformed.Is(err) || dir.ErrNotExist.Is(err): printRed("Cannot perform this action, source{d} needs to be initialized first with the 'init' sub command") case workdir.ErrInitFailed.Is(err): printRed("Cannot perform this action, full re-initialization is needed, run 'prune' command first") case dir.ErrNotValid.Is(err): printRed("Cannot perform this action, config directory is not valid") case compose.ErrComposeAlternative.Is(err): printRed("docker-compose is not installed, and there was an error while trying to use the container alternative") printRed(" see: https://docs.sourced.tech/community-edition/quickstart/1-install-requirements#docker-compose") case file.ErrConfigDownload.Is(err): printRed("The source{d} CE config file could not be set as active") case fmt.Sprintf("%T", err) == "*flags.Error": // syntax error is already logged by go-cli default: // unknown errors have no special message } switch { case dir.ErrNetwork.Is(err): // TODO(dpordomingo): if start using "https://golang.org/pkg/errors/", // we could do `var myErr ErrNetwork; errors.As(err, &myErr)` // to provide more info about the actual network error printRed("The resource could not be downloaded from the Internet") printRed(" see: https://docs.sourced.tech/community-edition/quickstart/1-install-requirements#internet-connection") case dir.ErrWrite.Is(err): // TODO(dpordomingo): see todo above printRed("could not write file") } } func printRed(message string) { fmt.Println(format.Colorize(format.Red, message)) } ================================================ FILE: cmd/sourced/cmd/sql.go ================================================ package cmd import ( "context" "os" "github.com/src-d/sourced-ce/cmd/sourced/compose" "golang.org/x/crypto/ssh/terminal" ) type sqlCmd struct { Command `name:"sql" short-description:"Open a MySQL client connected to a SQL interface for Git" long-description:"Open a MySQL client connected to a SQL interface for Git"` Args struct { Query string `positional-arg-name:"query" description:"SQL query to be run by the SQL interface for Git"` } `positional-args:"yes"` } func (c *sqlCmd) Execute(args []string) error { command := []string{"exec"} if !terminal.IsTerminal(int(os.Stdout.Fd())) || !terminal.IsTerminal(int(os.Stdin.Fd())) { command = append(command, "-T") } command = append(command, "gitbase", "mysql") if c.Args.Query != "" { command = append(command, "--execute", c.Args.Query) } return compose.Run(context.Background(), command...) } func init() { rootCmd.AddCommand(&sqlCmd{}) } ================================================ FILE: cmd/sourced/cmd/start.go ================================================ package cmd import ( "context" "time" "github.com/src-d/sourced-ce/cmd/sourced/compose" ) type startCmd struct { Command `name:"start" short-description:"Start any stopped components" long-description:"Start any stopped components.\nThe containers must be initialized before with 'init'."` } func (c *startCmd) Execute(args []string) error { if err := compose.Run(context.Background(), "start"); err != nil { return err } return OpenUI(30 * time.Minute) } func init() { rootCmd.AddCommand(&startCmd{}) } ================================================ FILE: cmd/sourced/cmd/status.go ================================================ package cmd import ( "context" "fmt" "io/ioutil" "os" "path/filepath" "github.com/src-d/sourced-ce/cmd/sourced/compose" "github.com/src-d/sourced-ce/cmd/sourced/compose/workdir" ) type statusCmd struct { Command `name:"status" short-description:"Show the list of working directories and the current deployment" long-description:"Show the list of working directories and the current deployment"` } type statusAllCmd struct { Command `name:"all" short-description:"Show all the available status information" long-description:"Show all the available status information"` } func (c *statusAllCmd) Execute(args []string) error { fmt.Print("List of all working directories:\n") err := printWorkdirsCmd() if err != nil { return err } active, err := activeWorkdir() if isNotExist(err) { // skip printing the config and components when there is no active dir return nil } if err != nil { return err } fmt.Print("\nConfiguration used for the active working directory:\n\n") err = printConfigCmd(active) if err != nil { return err } fmt.Print("\nStatus of all components:\n\n") err = printComponentsCmd() if err != nil { return err } return nil } type statusComponentsCmd struct { Command `name:"components" short-description:"Show the status of the components containers" long-description:"Show the status of the components containers"` } func (c *statusComponentsCmd) Execute(args []string) error { return printComponentsCmd() } func printComponentsCmd() error { return compose.Run(context.Background(), "ps") } type statusWorkdirsCmd struct { Command `name:"workdirs" short-description:"List all working directories" long-description:"List all the previously initialized working directories"` } func (c *statusWorkdirsCmd) Execute(args []string) error { return printWorkdirsCmd() } func printWorkdirsCmd() error { workdirHandler, err := workdir.NewHandler() if err != nil { return err } wds, err := workdirHandler.List() if err != nil { return err } activePath, err := activeWorkdir() // active directory does not necessarily exist if err != nil && !isNotExist(err) { return err } for _, wd := range wds { if wd.Path == activePath { fmt.Printf("* %s\n", wd.Name) } else { fmt.Printf(" %s\n", wd.Name) } } return nil } type statusConfigCmd struct { Command `name:"config" short-description:"Show the configuration for the active working directory" long-description:"Show the docker-compose environment variables configuration for the active working directory"` } func (c *statusConfigCmd) Execute(args []string) error { active, err := activeWorkdir() if err != nil { return err } return printConfigCmd(active) } func printConfigCmd(path string) error { content, err := ioutil.ReadFile(filepath.Join(path, ".env")) if err != nil { return err } fmt.Printf("%s\n", content) return nil } func isNotExist(err error) bool { if os.IsNotExist(err) { return true } if cause, ok := err.(causer); ok { return isNotExist(cause.Cause()) } return false } type causer interface { Cause() error } func activeWorkdir() (string, error) { workdirHandler, err := workdir.NewHandler() if err != nil { return "", err } active, err := workdirHandler.Active() if err != nil { return "", err } return active.Path, err } func init() { c := rootCmd.AddCommand(&statusCmd{}) c.AddCommand(&statusAllCmd{}) c.AddCommand(&statusComponentsCmd{}) c.AddCommand(&statusWorkdirsCmd{}) c.AddCommand(&statusConfigCmd{}) } ================================================ FILE: cmd/sourced/cmd/stop.go ================================================ package cmd import ( "context" "github.com/src-d/sourced-ce/cmd/sourced/compose" ) type stopCmd struct { Command `name:"stop" short-description:"Stop any running components" long-description:"Stop any running components without removing them.\nThey can be started again with 'start'."` } func (c *stopCmd) Execute(args []string) error { return compose.Run(context.Background(), "stop") } func init() { rootCmd.AddCommand(&stopCmd{}) } ================================================ FILE: cmd/sourced/cmd/web.go ================================================ package cmd import ( "bytes" "context" "fmt" "net/http" "os" "os/exec" "regexp" "runtime" "strings" "time" "github.com/pkg/browser" "github.com/pkg/errors" "github.com/src-d/sourced-ce/cmd/sourced/compose" ) // The service name used in docker-compose.yml for the srcd/sourced-ui image const containerName = "sourced-ui" type webCmd struct { Command `name:"web" short-description:"Open the web interface in your browser." long-description:"Open the web interface in your browser, by default at: http://127.0.0.1:8088 user:admin pass:admin"` } func (c *webCmd) Execute(args []string) error { return OpenUI(2 * time.Second) } func init() { rootCmd.AddCommand(&webCmd{}) } func openUI(address string) error { // docker-compose returns 0.0.0.0 which is correct for the bind address // but incorrect as connect address url := fmt.Sprintf("http://%s", strings.Replace(address, "0.0.0.0", "127.0.0.1", 1)) for { client := http.Client{Timeout: time.Second} if _, err := client.Get(url); err == nil { break } time.Sleep(1 * time.Second) } if err := browser.OpenURL(url); err != nil { return errors.Wrap(err, "could not open the browser") } return nil } var stateExtractor = regexp.MustCompile(`(?m)^srcd-\w+.*(Up|Exit (\d+))`) func checkServiceStatus(service string) error { var stdout bytes.Buffer if err := compose.RunWithIO(context.Background(), os.Stdin, &stdout, nil, "ps", service); err != nil { return errors.Wrapf(err, "cannot get status service %s", service) } matches := stateExtractor.FindAllStringSubmatch(strings.TrimSpace(stdout.String()), -1) for _, match := range matches { state := match[1] if strings.HasPrefix(state, "Exit") { if service != "ghsync" && service != "gitcollector" { return fmt.Errorf("service '%s' is in state '%s'", service, state) } returnCode := state[len("Exit "):len(state)] if returnCode != "0" { return fmt.Errorf("service '%s' exited with return code: %s", service, returnCode) } continue } if state != "Up" { return fmt.Errorf("service '%s' is in state '%s'", service, state) } } return nil } // runMonitor checks the status of the containers in order to early exit in case // an unrecoverable error occurs. // The monitoring is performed by running `docker-compose ps ` for each // service returned by `docker-compose config --services`, and by grepping the // state from the stdout using a regex. // Getting the state of all the containers in a single pass by running `docker-compose ps` // and by using a multi-line regex to extract both service name and state is not reliable. // The reason is that the prefix of a container can be very long, especially for local // initialization, due to the value that we set for `COMPOSE_PROJECT_NAME` env var, and // docker-compose may split the name into multiple lines. // E.g.: // // Name Command State Ports // ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ // srcd-l1vzzxjzl3nln2vudhlzztdlbi9qcm9qzwn0cy8uz28td29ya3nwywnll3nyyy9naxrodwiuy29tl3nln /bin/bblfsh-web -addr :808 ... Up 0.0.0.0:9999->8080/tcp // 2vudhlzztdlbg_bblfsh-web_1 func runMonitor(ch chan<- error) { var servicesBuf bytes.Buffer if err := compose.RunWithIO(context.Background(), os.Stdin, &servicesBuf, nil, "config", "--services"); err != nil { ch <- errors.Wrap(err, "cannot get list of services") return } services := strings.Split(strings.TrimSpace(servicesBuf.String()), "\n") for _, service := range services { if err := checkServiceStatus(service); err != nil { ch <- err return } time.Sleep(time.Second) } } func getContainerPublicAddress(containerName, privatePort string) (string, error) { var stdout bytes.Buffer for { err := compose.RunWithIO(context.Background(), nil, &stdout, nil, "port", containerName, privatePort) if err == nil { break } // skip any unsuccessful command exits if _, ok := err.(*exec.ExitError); !ok { return "", err } time.Sleep(1 * time.Second) } address := strings.TrimSpace(stdout.String()) if address == "" { return "", fmt.Errorf("could not find the public port of %s", containerName) } return address, nil } // OpenUI opens the browser with the UI. func OpenUI(timeout time.Duration) error { ch := make(chan error) go func() { address, err := getContainerPublicAddress(containerName, "8088") if err != nil { ch <- err return } ch <- openUI(address) }() go runMonitor(ch) fmt.Println(` Once source{d} is fully initialized, the UI will be available, by default at: http://127.0.0.1:8088 user:admin pass:admin `) if timeout > 5*time.Second { stopSpinner := startSpinner("Initializing source{d}...") defer stopSpinner() } select { case err := <-ch: return err case <-time.After(timeout): return fmt.Errorf("error opening the UI, the container is not running after %v", timeout) } } type spinner struct { msg string charset []int interval time.Duration stop chan bool } func startSpinner(msg string) func() { charset := []int{'⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'} if runtime.GOOS == "windows" { charset = []int{'|', '/', '-', '\\'} } s := &spinner{ msg: msg, charset: charset, interval: 200 * time.Millisecond, stop: make(chan bool), } s.Start() return s.Stop } func (s *spinner) Start() { go s.printLoop() } func (s *spinner) Stop() { s.stop <- true } func (s *spinner) printLoop() { i := 0 for { select { case <-s.stop: fmt.Println(s.msg) return default: char := string(s.charset[i%len(s.charset)]) if runtime.GOOS == "windows" { fmt.Printf("\r%s %s", s.msg, char) } else { fmt.Printf("%s %s\n\033[A", s.msg, char) } time.Sleep(s.interval) } i++ if len(s.charset) == i { i = 0 } } } ================================================ FILE: cmd/sourced/compose/compose.go ================================================ package compose import ( "context" "fmt" "io" "os" "os/exec" "path/filepath" "regexp" "runtime" "strconv" "strings" "github.com/blang/semver" "github.com/src-d/sourced-ce/cmd/sourced/compose/workdir" "github.com/src-d/sourced-ce/cmd/sourced/dir" "github.com/pkg/errors" goerrors "gopkg.in/src-d/go-errors.v1" ) // v1.20.0 is the first version that supports `--compatibility` flag we rely on // there is no mention of it in changelog // the version has been found by trying downgrading unless it started to error var minDockerComposeVersion = semver.Version{ Major: 1, Minor: 20, Patch: 0, } // this version is choosen to be always compatible with docker-compose version // docker-compose v1.20.0 introduced compose files version 3.6 // which requires Docker Engine 18.02.0 or above var minDockerVersion = semver.Version{ Major: 18, Minor: 2, Patch: 0, } // dockerComposeVersion is the version of docker-compose to download // if docker-compose isn't already present in the system const dockerComposeVersion = "1.24.0" var composeContainerURL = fmt.Sprintf("https://github.com/docker/compose/releases/download/%s/run.sh", dockerComposeVersion) // ErrComposeAlternative is returned when docker-compose alternative could not be installed var ErrComposeAlternative = goerrors.NewKind("error while trying docker-compose container alternative") type Compose struct { bin string workdirHandler *workdir.Handler } func (c *Compose) Run(ctx context.Context, arg ...string) error { return c.RunWithIO(ctx, os.Stdin, os.Stdout, os.Stderr, arg...) } func (c *Compose) RunWithIO(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, arg ...string) error { arg = append([]string{"--compatibility"}, arg...) cmd := exec.CommandContext(ctx, c.bin, arg...) wd, err := c.workdirHandler.Active() if err != nil { return err } if err := c.workdirHandler.Validate(wd); err != nil { return err } cmd.Dir = wd.Path cmd.Stdin = stdin cmd.Stdout = stdout cmd.Stderr = stderr return cmd.Run() } func newCompose() (*Compose, error) { // check docker first and exit fast dockerVersion, err := getDockerVersion() if err != nil { return nil, err } if !dockerVersion.GE(minDockerVersion) { return nil, fmt.Errorf("Minimal required docker version is %s but %s found", minDockerVersion, dockerVersion) } workdirHandler, err := workdir.NewHandler() if err != nil { return nil, err } bin, err := getOrInstallComposeBinary() if err != nil { return nil, err } dockerComposeVersion, err := getDockerComposeVersion(bin) if err != nil { return nil, err } if !dockerComposeVersion.GE(minDockerComposeVersion) { return nil, fmt.Errorf("Minimal required docker-compose version is %s but %s found", minDockerComposeVersion, dockerComposeVersion) } return &Compose{ bin: bin, workdirHandler: workdirHandler, }, nil } func getOrInstallComposeBinary() (string, error) { path, err := exec.LookPath("docker-compose") if err == nil { bin := strings.TrimSpace(path) if bin != "" { return bin, nil } } path, err = getOrInstallComposeContainer() if err != nil { return "", ErrComposeAlternative.Wrap(err) } return path, nil } func getOrInstallComposeContainer() (altPath string, err error) { datadir, err := dir.Path() if err != nil { return "", err } dirPath := filepath.Join(datadir, "bin") path := filepath.Join(dirPath, fmt.Sprintf("docker-compose-%s.sh", dockerComposeVersion)) readExecAccessMode := os.FileMode(0500) if info, err := os.Stat(path); err == nil { if info.Mode()&readExecAccessMode != readExecAccessMode { return "", fmt.Errorf("%s can not be run", path) } return path, nil } else if !os.IsNotExist(err) { return "", err } if err := downloadCompose(path); err != nil { return "", err } cmd := exec.CommandContext(context.Background(), "chmod", "+x", path) if err := cmd.Run(); err != nil { return "", errors.Wrapf(err, "cannot change permission to %s", path) } return path, nil } func downloadCompose(path string) error { if runtime.GOOS == "windows" { return fmt.Errorf("compose in container is not compatible with Windows") } return dir.DownloadURL(composeContainerURL, path) } func Run(ctx context.Context, arg ...string) error { comp, err := newCompose() if err != nil { return err } return comp.Run(ctx, arg...) } func RunWithIO(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, arg ...string) error { comp, err := newCompose() if err != nil { return err } return comp.RunWithIO(ctx, stdin, stdout, stderr, arg...) } var dockerVersionRe = regexp.MustCompile(`version (\d+).(\d+).(\d+)`) var dockerComposeVersionRe = regexp.MustCompile(`version (\d+.\d+.\d+)`) // docker doesn't use semver, so simple `semver.Parse` would fail // but semver.Version struct fits us to allow simple comparation func getDockerVersion() (*semver.Version, error) { if _, err := exec.LookPath("docker"); err != nil { return nil, err } out, err := exec.Command("docker", "--version").Output() if err != nil { return nil, err } submatches := dockerVersionRe.FindSubmatch(out) if len(submatches) != 4 { return nil, fmt.Errorf("can't parse docker version") } v := &semver.Version{} v.Major, err = strconv.ParseUint(string(submatches[1]), 10, 64) if err != nil { return nil, fmt.Errorf("can't parse docker version") } v.Minor, err = strconv.ParseUint(string(submatches[2]), 10, 64) if err != nil { return nil, fmt.Errorf("can't parse docker version") } v.Patch, err = strconv.ParseUint(string(submatches[3]), 10, 64) if err != nil { return nil, fmt.Errorf("can't parse docker version") } return v, nil } func getDockerComposeVersion(bin string) (*semver.Version, error) { out, err := exec.Command(bin, "--version").Output() if err != nil { return nil, err } submatches := dockerComposeVersionRe.FindSubmatch(out) if len(submatches) != 2 { return nil, fmt.Errorf("can't parse docker-compose version") } v, err := semver.ParseTolerant(string(submatches[1])) if err != nil { return nil, fmt.Errorf("can't parse docker-compose version: %s", err) } return &v, nil } ================================================ FILE: cmd/sourced/compose/file/file.go ================================================ // Package file provides functions to manage docker compose files inside the // $HOME/.sourced/compose-files directory package file import ( "encoding/base64" "fmt" "io/ioutil" "net/url" "os" "path/filepath" datadir "github.com/src-d/sourced-ce/cmd/sourced/dir" "github.com/pkg/errors" goerrors "gopkg.in/src-d/go-errors.v1" ) // ErrConfigDownload is returned when docker-compose.yml could not be downloaded var ErrConfigDownload = goerrors.NewKind("docker-compose.yml config file could not be downloaded") // ErrConfigActivation is returned when docker-compose.yml could not be set as active var ErrConfigActivation = goerrors.NewKind("docker-compose.yml could not be set as active") const ( orgName = "src-d" repoName = "sourced-ce" composeFileTmpl = "https://raw.githubusercontent.com/%s/%s/%s/docker-compose.yml" ) var version = "master" // activeDir is the name of the directory containing the symlink to the // active docker compose file const activeDir = "__active__" // RevOrURL is a revision (tag name, full sha1) or a valid URL to a // docker-compose.yml file type RevOrURL = string // composeFileURL returns the URL to download the raw github docker-compose.yml // file for the given revision (tag or full sha1) func composeFileURL(revision string) string { return fmt.Sprintf(composeFileTmpl, orgName, repoName, revision) } // SetVersion sets the version rewritten by the CI build func SetVersion(v string) { version = v } // InitDefault checks if there is an active docker compose file, and if there // isn't the file for this release is downloaded. // The current build version must be set with SetVersion. // It returns the absolute path to the active docker-compose.yml file func InitDefault() (string, error) { activeFilePath, err := path(activeDir) if err != nil { return "", err } _, err = os.Stat(activeFilePath) if err == nil { return activeFilePath, nil } if !os.IsNotExist(err) { return "", err } err = ActivateFromRemote(version) if err != nil { return "", err } return activeFilePath, nil } // ActivateFromRemote downloads the docker-compose.yml file from the given revision // or URL, and sets it as the active compose file. func ActivateFromRemote(revOrURL RevOrURL) (err error) { var url string if isURL(revOrURL) { url = revOrURL } else { url = composeFileURL(revOrURL) } outPath, err := path(revOrURL) if err != nil { return err } err = datadir.DownloadURL(url, outPath) if err != nil { return ErrConfigDownload.Wrap(err) } err = SetActive(revOrURL) if err != nil { return ErrConfigActivation.Wrap(err) } return nil } // SetActive makes a symlink from // $HOME/.sourced/compose-files/__active__/docker-compose.yml to the compose file // for the given revision or URL. func SetActive(revOrURL RevOrURL) error { filePath, err := path(revOrURL) if err != nil { return err } if _, err := os.Stat(filePath); err != nil { if os.IsNotExist(err) { return errors.Wrapf(err, "could not find a docker-compose.yml file in `%s`", filePath) } return err } activeFilePath, err := path(activeDir) if err != nil { return err } err = os.MkdirAll(filepath.Dir(activeFilePath), os.ModePerm) if err != nil { return errors.Wrapf(err, "error while creating directory for %s", activeFilePath) } if _, err := os.Lstat(activeFilePath); err == nil { if err := os.Remove(activeFilePath); err != nil { return errors.Wrap(err, "failed to unlink") } } return os.Symlink(filePath, activeFilePath) } // Active returns the revision (tag name, full sha1) or the URL of the active // docker compose file func Active() (RevOrURL, error) { activeFilePath, err := path(activeDir) if err != nil { return "", err } if _, err := os.Stat(activeFilePath); err != nil { if os.IsNotExist(err) { return "", nil } return "", err } dest, err := filepath.EvalSymlinks(activeFilePath) if err != nil { return "", err } _, name := filepath.Split(filepath.Dir(dest)) return composeName(name), nil } // List returns a list of installed docker compose files. Each name is the // revision (tag name, full sha1) or the URL func List() ([]RevOrURL, error) { list := []RevOrURL{} dir, err := dir() if err != nil { return list, err } if _, err := os.Stat(dir); err != nil { if os.IsNotExist(err) { return list, nil } return list, err } files, err := ioutil.ReadDir(dir) if err != nil { return list, err } for _, f := range files { entry := f.Name() if entry == activeDir { continue } list = append(list, composeName(entry)) } return list, nil } func composeName(rev string) string { if decoded, err := base64.URLEncoding.DecodeString(rev); err == nil { return string(decoded) } return rev } func isURL(revOrURL RevOrURL) bool { _, err := url.ParseRequestURI(revOrURL) return err == nil } // dir returns the absolute path for $HOME/.sourced/compose-files func dir() (string, error) { path, err := datadir.Path() if err != nil { return "", err } return filepath.Join(path, "compose-files"), nil } // path returns the absolute path to // $HOME/.sourced/compose-files/revOrURL/docker-compose.yml func path(revOrURL RevOrURL) (string, error) { composeDirPath, err := dir() if err != nil { return "", err } subPath := revOrURL if isURL(revOrURL) { subPath = base64.URLEncoding.EncodeToString([]byte(revOrURL)) } dirPath := filepath.Join(composeDirPath, subPath) return filepath.Join(dirPath, "docker-compose.yml"), nil } ================================================ FILE: cmd/sourced/compose/workdir/env_file_test.go ================================================ package workdir import ( "strings" "testing" "github.com/stretchr/testify/assert" ) // default limits depend on host system and can't be used in tests func setResourceLimits(f *envFile) { f.GitcollectorLimitCPU = 2.0 f.GitbaseLimitCPU = 1.5 f.GitbaseLimitMem = 100 } const localContent = `COMPOSE_PROJECT_NAME=srcd-dir-name GITBASE_VOLUME_TYPE=bind GITBASE_VOLUME_SOURCE=repo-dir GITBASE_SIVA= GITHUB_ORGANIZATIONS= GITHUB_TOKEN= NO_FORKS= GITBASE_LIMIT_CPU=1.5 GITCOLLECTOR_LIMIT_CPU=2 GITBASE_LIMIT_MEM=100 ` const orgsContent = `COMPOSE_PROJECT_NAME=srcd-dir-name GITBASE_VOLUME_TYPE=volume GITBASE_VOLUME_SOURCE=gitbase_repositories GITBASE_SIVA=true GITHUB_ORGANIZATIONS=org1,org2 GITHUB_TOKEN=token NO_FORKS=true GITBASE_LIMIT_CPU=1.5 GITCOLLECTOR_LIMIT_CPU=2 GITBASE_LIMIT_MEM=100 ` const emptyContent = `COMPOSE_PROJECT_NAME= GITBASE_VOLUME_TYPE= GITBASE_VOLUME_SOURCE= GITBASE_SIVA= GITHUB_ORGANIZATIONS= GITHUB_TOKEN= NO_FORKS= GITBASE_LIMIT_CPU=0 GITCOLLECTOR_LIMIT_CPU=0 GITBASE_LIMIT_MEM=0 ` func TestEnvMarshal(t *testing.T) { assert := assert.New(t) f := newLocalEnvFile("dir-name", "repo-dir") setResourceLimits(&f) b, err := f.MarshalEnv() assert.Nil(err) assert.Equal(localContent, strings.ReplaceAll(string(b), "\r\n", "\n")) f = newOrgEnvFile("dir-name", []string{"org1", "org2"}, "token", false) setResourceLimits(&f) b, err = f.MarshalEnv() assert.Nil(err) assert.Equal(orgsContent, strings.ReplaceAll(string(b), "\r\n", "\n")) f = envFile{} b, err = f.MarshalEnv() assert.Nil(err) assert.Equal(emptyContent, strings.ReplaceAll(string(b), "\r\n", "\n")) } func TestEnvUnmarshal(t *testing.T) { assert := assert.New(t) b := []byte(localContent) f := envFile{} assert.Nil(f.UnmarshalEnv(b)) assert.Equal(envFile{ ComposeProjectName: "srcd-dir-name", GitbaseVolumeType: "bind", GitbaseVolumeSource: "repo-dir", GitcollectorLimitCPU: 2.0, GitbaseLimitCPU: 1.5, GitbaseLimitMem: 100, }, f) b = []byte(orgsContent) f = envFile{} assert.Nil(f.UnmarshalEnv(b)) assert.Equal(envFile{ ComposeProjectName: "srcd-dir-name", GitbaseVolumeType: "volume", GitbaseVolumeSource: "gitbase_repositories", GitbaseSiva: true, GithubOrganizations: []string{"org1", "org2"}, GithubToken: "token", NoForks: true, GitcollectorLimitCPU: 2.0, GitbaseLimitCPU: 1.5, GitbaseLimitMem: 100, }, f) b = []byte("") f = envFile{} assert.Nil(f.UnmarshalEnv(b)) b = []byte(" COMPOSE_PROJECT_NAME=srcd-dir-name \n\n GITBASE_VOLUME_TYPE=volume ") f = envFile{} assert.Nil(f.UnmarshalEnv(b)) assert.Equal(envFile{ ComposeProjectName: "srcd-dir-name", GitbaseVolumeType: "volume", }, f) b = []byte("UNKNOWN=1") f = envFile{} assert.Nil(f.UnmarshalEnv(b)) } ================================================ FILE: cmd/sourced/compose/workdir/factory.go ================================================ package workdir import ( "bufio" "bytes" "encoding/base64" "fmt" "io/ioutil" "os" "os/exec" "path/filepath" "reflect" "runtime" "sort" "strconv" "strings" "github.com/pbnjay/memory" "github.com/pkg/errors" "github.com/serenize/snaker" composefile "github.com/src-d/sourced-ce/cmd/sourced/compose/file" ) // InitLocal initializes the workdir for local path and returns the Workdir instance func InitLocal(reposdir string) (*Workdir, error) { dirName := encodeDirName(reposdir) envf := newLocalEnvFile(dirName, reposdir) return initialize(dirName, "local", envf) } // InitOrgs initializes the workdir for organizations and returns the Workdir instance func InitOrgs(orgs []string, token string, withForks bool) (*Workdir, error) { // be indifferent to the order of passed organizations sort.Strings(orgs) dirName := encodeDirName(strings.Join(orgs, ",")) envf := envFile{} err := readEnvFile(dirName, "orgs", &envf) if err == nil && envf.NoForks == withForks { return nil, ErrInitFailed.Wrap( fmt.Errorf("workdir was previously initialized with a different value for forks support")) } if err != nil && !os.IsNotExist(err) { return nil, err } // re-create env file to make sure all fields are updated envf = newOrgEnvFile(dirName, orgs, token, withForks) return initialize(dirName, "orgs", envf) } func readEnvFile(dirName string, subPath string, envf *envFile) error { workdir, err := workdirPath(dirName, subPath) if err != nil { return err } envPath := filepath.Join(workdir, ".env") b, err := ioutil.ReadFile(envPath) if err != nil { return err } return envf.UnmarshalEnv(b) } func encodeDirName(dirName string) string { return base64.URLEncoding.EncodeToString([]byte(dirName)) } func workdirPath(dirName string, subPath string) (string, error) { path, err := workdirsPath() if err != nil { return "", err } workdir := filepath.Join(path, subPath, dirName) if err != nil { return "", err } return workdir, nil } func initialize(dirName string, subPath string, envf envFile) (*Workdir, error) { path, err := workdirsPath() if err != nil { return nil, err } workdir, err := workdirPath(dirName, subPath) if err != nil { return nil, err } err = os.MkdirAll(workdir, 0755) if err != nil { return nil, errors.Wrap(err, "could not create working directory") } defaultFilePath, err := composefile.InitDefault() if err != nil { return nil, err } composePath := filepath.Join(workdir, "docker-compose.yml") if err := link(defaultFilePath, composePath); err != nil { return nil, err } envPath := filepath.Join(workdir, ".env") contents, err := envf.MarshalEnv() if err != nil { return nil, err } err = ioutil.WriteFile(envPath, contents, 0644) if err != nil { return nil, errors.Wrap(err, "could not write .env file") } b := &builder{workdirsPath: path} return b.Build(workdir) } type envFile struct { ComposeProjectName string GitbaseVolumeType string GitbaseVolumeSource string GitbaseSiva bool GithubOrganizations []string GithubToken string NoForks bool GitbaseLimitCPU float32 GitcollectorLimitCPU float32 GitbaseLimitMem uint64 } func newLocalEnvFile(dirName, repoDir string) envFile { f := envFile{ ComposeProjectName: fmt.Sprintf("srcd-%s", dirName), GitbaseVolumeType: "bind", GitbaseVolumeSource: repoDir, } f.addResourceLimits() return f } func newOrgEnvFile(dirName string, orgs []string, token string, withForks bool) envFile { f := envFile{ ComposeProjectName: fmt.Sprintf("srcd-%s", dirName), GitbaseVolumeType: "volume", GitbaseVolumeSource: "gitbase_repositories", GitbaseSiva: true, GithubOrganizations: orgs, GithubToken: token, NoForks: !withForks, } f.addResourceLimits() return f } func (f *envFile) addResourceLimits() { // limit CPU for containers dockerCPUs, err := dockerNumCPU() if err != nil { // show warning fmt.Println(err) } // apply gitbase resource limits only when docker runs without any global limits // it's default behaviour on linux if runtime.NumCPU() == dockerCPUs { f.GitbaseLimitCPU = float32(dockerCPUs) - 0.1 } // always apply gitcollector limit if dockerCPUs > 0 { halfCPUs := float32(dockerCPUs) / 2.0 // let container consume more than a half if there is only one cpu available // otherwise it will be too slow if halfCPUs < 1 { halfCPUs = 1 } f.GitcollectorLimitCPU = halfCPUs - 0.1 } // limit memory for containers dockerMem, err := dockerTotalMem() if err != nil { // show warning fmt.Println(err) } // apply memory limits only when only when docker runs without any global limits // it's default behaviour on linux if dockerMem == memory.TotalMemory() { f.GitbaseLimitMem = uint64(float64(dockerMem) * 0.9) } } var newlineChar = "\n" func init() { if runtime.GOOS == "windows" { newlineChar = "\r\n" } } // implementation can be moved to separate package if we need to marshal any other structs // supports only simple types func (f envFile) MarshalEnv() ([]byte, error) { var b bytes.Buffer v := reflect.ValueOf(f) rType := v.Type() for i := 0; i < rType.NumField(); i++ { field := rType.Field(i) fieldEl := v.Field(i) if field.Anonymous { panic("struct composition isn't supported") } name := strings.ToUpper(snaker.CamelToSnake(field.Name)) switch field.Type.Kind() { case reflect.Slice: slice := make([]string, fieldEl.Len()) for i := 0; i < fieldEl.Len(); i++ { slice[i] = fmt.Sprintf("%v", fieldEl.Index(i).Interface()) } fmt.Fprintf(&b, "%s=%v%s", name, strings.Join(slice, ","), newlineChar) case reflect.Bool: // marshal false value as empty string instead of "false" string if fieldEl.Interface().(bool) { fmt.Fprintf(&b, "%s=true%s", name, newlineChar) } else { fmt.Fprintf(&b, "%s=%s", name, newlineChar) } default: fmt.Fprintf(&b, "%s=%v%s", name, fieldEl.Interface(), newlineChar) } } return b.Bytes(), nil } // implementation can be moved to separate package if we need to unmarshal any other structs // supports only simple types func (f *envFile) UnmarshalEnv(b []byte) error { v := reflect.ValueOf(f).Elem() r := bytes.NewReader(b) scanner := bufio.NewScanner(r) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || !strings.Contains(line, "=") { continue } parts := strings.SplitN(line, "=", 2) name := parts[0] value := parts[1] field := v.FieldByName(snaker.SnakeToCamel(strings.ToLower(name))) // skip unknown values if !field.IsValid() { continue } // skip empty values if value == "" { continue } switch field.Kind() { case reflect.String: field.SetString(value) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: i, err := strconv.ParseInt(value, 10, 64) if err != nil { return fmt.Errorf("can't parse variable %s with value %s: %v", name, value, err) } field.SetInt(i) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: i, err := strconv.ParseUint(value, 10, 64) if err != nil { return fmt.Errorf("can't parse variable %s with value %s: %v", name, value, err) } field.SetUint(i) case reflect.Float32, reflect.Float64: i, err := strconv.ParseFloat(value, 64) if err != nil { return fmt.Errorf("can't parse variable %s with value %s: %v", name, value, err) } field.SetFloat(i) case reflect.Bool: if value == "true" { field.SetBool(true) } else { field.SetBool(false) } case reflect.Slice: if field.Type().Elem().Kind() != reflect.String { panic("only slices of strings are supported") } vs := strings.Split(value, ",") slice := reflect.MakeSlice(field.Type(), len(vs), len(vs)) for i, v := range vs { slice.Index(i).SetString(v) } field.Set(slice) default: panic(fmt.Sprintf("unsupported type: %v", field.Kind())) } } return scanner.Err() } // returns number of CPUs available to docker func dockerNumCPU() (int, error) { // use cli instead of connection to docker server directly // in case server exposed by http or non-default socket path info, err := exec.Command("docker", "info", "--format", "{{.NCPU}}").Output() if err != nil { return 0, err } cpus, err := strconv.Atoi(strings.TrimSpace(string(info))) if err != nil || cpus == 0 { return 0, fmt.Errorf("Couldn't get number of available CPUs in docker") } return cpus, nil } // returns total memory in bytes available to docker func dockerTotalMem() (uint64, error) { info, err := exec.Command("docker", "info", "--format", "{{.MemTotal}}").Output() if err != nil { return 0, err } mem, err := strconv.ParseUint(strings.TrimSpace(string(info)), 10, 64) if err != nil || mem == 0 { return 0, fmt.Errorf("Couldn't get of available memory in docker") } return mem, nil } ================================================ FILE: cmd/sourced/compose/workdir/factory_test.go ================================================ package workdir import ( "os" "path" "path/filepath" "testing" "github.com/stretchr/testify/suite" ) type FactorySuite struct { suite.Suite originSrcdDir string } func TestFactorySuite(t *testing.T) { suite.Run(t, &FactorySuite{}) } func (s *FactorySuite) BeforeTest(suiteName, testName string) { s.originSrcdDir = os.Getenv("SOURCED_DIR") // on macOs os.TempDir returns symlink and tests fails tmpDir, _ := filepath.EvalSymlinks(os.TempDir()) srdPath := path.Join(tmpDir, testName) err := os.MkdirAll(srdPath, os.ModePerm) s.Nil(err) os.Setenv("SOURCED_DIR", srdPath) } func (s *FactorySuite) AfterTest(suiteName, testName string) { os.RemoveAll(os.Getenv("SOURCED_DIR")) os.Setenv("SOURCED_DIR", s.originSrcdDir) } func (s *FactorySuite) TestInitLocal() { reposdir := "some-dir" wd, err := InitLocal(reposdir) s.Nil(err) s.Equal(Local, wd.Type) s.Equal(reposdir, wd.Name) // check docker-compose.yml exists composeYmlPath := path.Join(wd.Path, "docker-compose.yml") _, err = os.Stat(composeYmlPath) s.Nil(err) // check .env file envPath := path.Join(wd.Path, ".env") _, err = os.Stat(envPath) s.Nil(err) envf := envFile{} s.Nil(readEnvFile(encodeDirName(reposdir), "local", &envf)) s.Equal(reposdir, envf.GitbaseVolumeSource) s.False(envf.NoForks) } func (s *FactorySuite) TestInitOrgs() { orgs := []string{"org2", "org1"} name := "org1,org2" token := "some-token" wd, err := InitOrgs(orgs, token, true) s.Nil(err) s.Equal(Orgs, wd.Type) s.Equal(name, wd.Name) // check docker-compose.yml exists composeYmlPath := path.Join(wd.Path, "docker-compose.yml") _, err = os.Stat(composeYmlPath) s.Nil(err) // check .env file envPath := path.Join(wd.Path, ".env") _, err = os.Stat(envPath) s.Nil(err) envf := envFile{} s.Nil(readEnvFile(encodeDirName(name), "orgs", &envf)) s.Equal("gitbase_repositories", envf.GitbaseVolumeSource) s.Equal(orgs, envf.GithubOrganizations) s.Equal(token, envf.GithubToken) s.False(envf.NoForks) } func (s *FactorySuite) TestReInitForksOrgs() { orgs := []string{"org2", "org1"} _, err := InitOrgs(orgs, "", false) s.Nil(err) _, err = InitOrgs(orgs, "", true) s.EqualError(err, "initialization failed: workdir was previously initialized with a different value for forks support") } ================================================ FILE: cmd/sourced/compose/workdir/handler.go ================================================ package workdir import ( "fmt" "io/ioutil" "os" "path/filepath" "github.com/pkg/errors" ) // Handler provides a way to interact with all the workdirs by exposing the following operations: // - read/set/unset active workdir, // - remove/validate a workdir, // - list workdirs. type Handler struct { workdirsPath string builder *builder } // NewHandler creates a handler that manages workdirs in the path returned by // the `workdirsPath` function func NewHandler() (*Handler, error) { path, err := workdirsPath() if err != nil { return nil, err } return &Handler{ workdirsPath: path, builder: &builder{workdirsPath: path}, }, nil } // SetActive creates a symlink from the fixed active workdir path to the prodived workdir func (h *Handler) SetActive(w *Workdir) error { path := h.activeAbsolutePath() if err := h.UnsetActive(); err != nil { return err } err := os.Symlink(w.Path, path) if os.IsExist(err) { return nil } return err } // UnsetActive removes symlink for active workdir func (h *Handler) UnsetActive() error { path := h.activeAbsolutePath() _, err := os.Lstat(path) if !os.IsNotExist(err) { err = os.Remove(path) if err != nil { return errors.Wrap(err, "could not delete the previous active workdir directory symlink") } } return nil } // Active returns active working directory func (h *Handler) Active() (*Workdir, error) { path := h.activeAbsolutePath() resolvedPath, err := filepath.EvalSymlinks(path) if os.IsNotExist(err) { return nil, ErrMalformed.Wrap(err, "active") } return h.builder.Build(resolvedPath) } // List returns array of working directories func (h *Handler) List() ([]*Workdir, error) { dirs := make([]string, 0) err := filepath.Walk(h.workdirsPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { return nil } for _, f := range RequiredFiles { if !hasContent(path, f) { return nil } } dirs = append(dirs, path) return nil }) if os.IsNotExist(err) { return nil, ErrMalformed.Wrap(err, h.workdirsPath) } if err != nil { return nil, err } wds := make([]*Workdir, 0, len(dirs)) for _, p := range dirs { wd, err := h.builder.Build(p) if err != nil { return nil, err } wds = append(wds, wd) } return wds, nil } // Validate validates that the passed working directoy is valid // It's path must be a directory (or a symlink) containing docker-compose.yml and .env files func (h *Handler) Validate(w *Workdir) error { pointedDir, err := filepath.EvalSymlinks(w.Path) if err != nil { return ErrMalformed.Wrap(fmt.Errorf("is not a directory"), w.Path) } if info, err := os.Lstat(pointedDir); err != nil || !info.IsDir() { return ErrMalformed.Wrap(fmt.Errorf("is not a directory"), pointedDir) } for _, f := range RequiredFiles { if !hasContent(pointedDir, f) { return ErrMalformed.Wrap(fmt.Errorf("%s not found", f), pointedDir) } } return nil } // Remove removes working directory by removing required and optional files, // and recursively removes directories up to the workdirs root as long as they are empty func (h *Handler) Remove(w *Workdir) error { path := w.Path var subPath string switch w.Type { case Local: subPath = "local" case Orgs: subPath = "orgs" } basePath := filepath.Join(h.workdirsPath, subPath) for _, f := range RequiredFiles { file := filepath.Join(path, f) if _, err := os.Stat(file); os.IsNotExist(err) { continue } if err := os.Remove(file); err != nil { return errors.Wrap(err, "could not remove from workdir directory") } } for { files, err := ioutil.ReadDir(path) if err != nil { return errors.Wrap(err, "could not read workdir directory") } if len(files) > 0 { return nil } if err := os.Remove(path); err != nil { return errors.Wrap(err, "could not delete workdir directory") } path = filepath.Dir(path) if path == basePath { return nil } } } func (h *Handler) activeAbsolutePath() string { return filepath.Join(h.workdirsPath, activeDir) } ================================================ FILE: cmd/sourced/compose/workdir/handler_test.go ================================================ package workdir import ( "os" "path" "path/filepath" "testing" "github.com/stretchr/testify/suite" ) type HandlerSuite struct { suite.Suite h *Handler originSrcdDir string } func TestHandlerSuite(t *testing.T) { suite.Run(t, &HandlerSuite{}) } func (s *HandlerSuite) BeforeTest(suiteName, testName string) { s.originSrcdDir = os.Getenv("SOURCED_DIR") // on macOs os.TempDir returns symlink and tests fails tmpDir, _ := filepath.EvalSymlinks(os.TempDir()) srdPath := path.Join(tmpDir, testName) err := os.MkdirAll(srdPath, os.ModePerm) s.Nil(err) os.Setenv("SOURCED_DIR", srdPath) s.h, err = NewHandler() s.Nil(err) } func (s *HandlerSuite) AfterTest(suiteName, testName string) { os.RemoveAll(filepath.Dir(s.h.workdirsPath)) os.Setenv("SOURCED_DIR", s.originSrcdDir) } // This tests only public interface without checking implementation (filesystem) details func (s *HandlerSuite) TestSuccessFlow() { wd := s.createWd("flow") s.Nil(s.h.Validate(wd)) s.Nil(s.h.SetActive(wd)) active, err := s.h.Active() s.Nil(err) s.Equal(wd, active) s.Nil(s.h.UnsetActive()) _, err = s.h.Active() s.True(ErrMalformed.Is(err)) wds, err := s.h.List() s.Nil(err) s.Len(wds, 1) s.Equal(wd, wds[0]) s.Nil(s.h.Remove(wd)) wds, err = s.h.List() s.Nil(err) s.Len(wds, 0) } // All tests below rely on implementation details to check error cases func (s *HandlerSuite) TestSetActiveOk() { wd := s.createWd("some") // non-active before s.Nil(s.h.SetActive(wd)) // re-activation should also work s.Nil(s.h.SetActive(wd)) // validate link points correctly target, err := filepath.EvalSymlinks(path.Join(s.h.workdirsPath, activeDir)) s.Nil(err) s.Equal(wd.Path, target) } func (s *HandlerSuite) TestSetActiveError() { wd := s.createWd("some") // break active path by making it dir with files activePath := path.Join(s.h.workdirsPath, activeDir) s.Nil(os.MkdirAll(activePath, os.ModePerm)) _, err := os.Create(path.Join(activePath, "some-file")) s.Nil(err) s.Error(s.h.SetActive(wd)) } func (s *HandlerSuite) TestUnsetActiveOk() { activePath := path.Join(s.h.workdirsPath, activeDir) s.Nil(os.MkdirAll(s.h.workdirsPath, os.ModePerm)) _, err := os.Create(activePath) s.Nil(err) s.Nil(s.h.UnsetActive()) // unset without active dir s.Nil(s.h.UnsetActive()) // validate we deleted the file _, err = os.Stat(activePath) s.True(os.IsNotExist(err)) } func (s *HandlerSuite) TestUnsetActiveError() { // break active path by making it dir with files activePath := path.Join(s.h.workdirsPath, activeDir) s.Nil(os.MkdirAll(activePath, os.ModePerm)) _, err := os.Create(path.Join(activePath, "some-file")) s.Nil(err) s.Error(s.h.UnsetActive()) } func (s *HandlerSuite) TestValidateError() { // dir doesn't exist wd, err := s.h.builder.Build(path.Join(s.h.workdirsPath, "local", "some")) s.Nil(err) err = s.h.Validate(wd) s.True(ErrMalformed.Is(err)) // dir is a file s.Nil(os.MkdirAll(path.Join(s.h.workdirsPath, "local"), os.ModePerm)) _, err = os.Create(wd.Path) s.Nil(err) err = s.h.Validate(wd) s.True(ErrMalformed.Is(err)) s.Nil(os.RemoveAll(wd.Path)) // dir is empty s.Nil(os.MkdirAll(wd.Path, os.ModePerm)) err = s.h.Validate(wd) s.True(ErrMalformed.Is(err)) } func (s *HandlerSuite) TestListOk() { s.Nil(os.MkdirAll(s.h.workdirsPath, os.ModePerm)) // empty results wds, err := s.h.List() s.Nil(err) s.Len(wds, 0) // multiple results s.createWd("one") s.createWd("two") wds, err = s.h.List() s.Nil(err) s.Len(wds, 2) s.Equal("one", wds[0].Name) s.Equal("two", wds[1].Name) // incorrect directory should be skipped wd, err := s.h.builder.Build(path.Join(s.h.workdirsPath, "local", "some")) s.Nil(err) s.Nil(os.MkdirAll(wd.Path, os.ModePerm)) wds, err = s.h.List() s.Nil(err) s.Len(wds, 2) } func (s *HandlerSuite) TestListError() { // workdirs dir doesn't exist _, err := s.h.List() s.True(ErrMalformed.Is(err)) } func (s *HandlerSuite) TestRemoveOk() { // local wd, err := InitLocal("local") s.Nil(err) s.Nil(s.h.Remove(wd)) _, err = os.Stat(wd.Path) s.True(os.IsNotExist(err)) // org wd, err = InitOrgs([]string{"some-org"}, "token", false) s.Nil(err) s.Nil(s.h.Remove(wd)) _, err = os.Stat(wd.Path) s.True(os.IsNotExist(err)) // skip deleting dir with extra files wd = s.createWd("some") _, err = os.Create(path.Join(wd.Path, "some-file")) s.Nil(err) s.Nil(s.h.Remove(wd)) _, err = os.Stat(wd.Path) s.Nil(err) } func (s *HandlerSuite) createWd(name string) *Workdir { wd, err := InitLocal(name) s.Nil(err) return wd } ================================================ FILE: cmd/sourced/compose/workdir/workdir.go ================================================ package workdir import ( "encoding/base64" "fmt" "io/ioutil" "os" "path/filepath" "strings" "github.com/pkg/errors" goerrors "gopkg.in/src-d/go-errors.v1" datadir "github.com/src-d/sourced-ce/cmd/sourced/dir" ) const activeDir = "__active__" var ( // RequiredFiles list of required files in a directory to treat it as a working directory RequiredFiles = []string{".env", "docker-compose.yml"} // ErrMalformed is the returned error when the workdir is wrong ErrMalformed = goerrors.NewKind("workdir %s is not valid") // ErrInitFailed is an error returned on workdir initialization for custom cases ErrInitFailed = goerrors.NewKind("initialization failed") ) // Type defines the type of the workdir type Type int const ( // None refers to a failure in identifying the type of the workdir None Type = iota // Local refers to a workdir that has been initialized for local repos Local // Orgs refers to a workdir that has been initialized for organizations Orgs ) // Workdir represents a workdir associated with a local or an orgs initialization type Workdir struct { // Type is the type of working directory Type Type // Name is a human-friendly string to identify the workdir Name string // Path is the absolute path corresponding to the workdir Path string } type builder struct { workdirsPath string } // build returns the Workdir instance corresponding to the provided absolute path // the path must be inside `workdirsPath` func (b *builder) Build(path string) (*Workdir, error) { wdType, err := b.typeFromPath(path) if err != nil { return nil, err } if wdType == None { return nil, fmt.Errorf("invalid workdir type for path %s", path) } wdName, err := b.workdirName(wdType, path) if err != nil { return nil, err } return &Workdir{ Type: wdType, Name: wdName, Path: path, }, nil } // workdirName returns the workdir name given its type and absolute path func (b *builder) workdirName(wdType Type, path string) (string, error) { var subPath string switch wdType { case Local: subPath = "local" case Orgs: subPath = "orgs" } encoded, err := filepath.Rel(filepath.Join(b.workdirsPath, subPath), path) if err != nil { return "", err } decoded, err := base64.URLEncoding.DecodeString(encoded) if err == nil { return string(decoded), nil } return "", err } // typeFromPath returns the workdir type corresponding to the provided absolute path func (b *builder) typeFromPath(path string) (Type, error) { suffix, err := filepath.Rel(b.workdirsPath, path) if err != nil { return None, err } switch filepath.Dir(suffix) { case "local": return Local, nil case "orgs": return Orgs, nil default: return None, nil } } func hasContent(path, file string) bool { empty, err := isEmptyFile(filepath.Join(path, file)) return !empty && err == nil } // isEmptyFile returns true if the file does not exist or if it exists but // contains empty text func isEmptyFile(path string) (bool, error) { _, err := os.Stat(path) if err != nil { if !os.IsNotExist(err) { return false, err } return true, nil } contents, err := ioutil.ReadFile(path) if err != nil { return false, err } strContents := string(contents) return strings.TrimSpace(strContents) == "", nil } func link(linkTargetPath, linkPath string) error { _, err := os.Stat(linkPath) if err == nil { return nil } if !os.IsNotExist(err) { return errors.Wrap(err, "could not read the existing FILE_NAME file") } err = os.Symlink(linkTargetPath, linkPath) return errors.Wrap(err, fmt.Sprintf("could not create symlink to %s", linkTargetPath)) } func workdirsPath() (string, error) { path, err := datadir.Path() if err != nil { return "", err } return filepath.Join(path, "workdirs"), nil } ================================================ FILE: cmd/sourced/compose/workdir/workdir_test.go ================================================ package workdir import ( "os" "path" "testing" "github.com/stretchr/testify/assert" ) func TestBuilder(t *testing.T) { assert := assert.New(t) workdirsPath := path.Join(os.TempDir(), "builder") defer func() { os.RemoveAll(workdirsPath) }() b := builder{workdirsPath: workdirsPath} // incorrect: not in workdirsPath _, err := b.Build("/not/in/workdirs") assert.EqualError(err, "invalid workdir type for path /not/in/workdirs") // incorrect: unknown type unknownDir := path.Join(workdirsPath, "unknown") _, err = b.Build(unknownDir) assert.EqualError(err, "invalid workdir type for path "+unknownDir) // local name := "some" localDir := path.Join(workdirsPath, "local", encodeDirName(name)) wd, err := b.Build(localDir) assert.Nil(err) assert.Equal(Local, wd.Type) assert.Equal(name, wd.Name) assert.Equal(localDir, wd.Path) // org orgDir := path.Join(workdirsPath, "orgs", encodeDirName(name)) wd, err = b.Build(orgDir) assert.Nil(err) assert.Equal(Orgs, wd.Type) assert.Equal(name, wd.Name) assert.Equal(orgDir, wd.Path) } func TestIsEmptyFile(t *testing.T) { assert := assert.New(t) // not exist ok, err := isEmptyFile("/does/not/exist") assert.Nil(err) assert.True(ok) // empty emptyPath := path.Join(os.TempDir(), "empty") defer func() { os.RemoveAll(emptyPath) }() f, err := os.Create(emptyPath) assert.Nil(err) assert.Nil(f.Close()) ok, err = isEmptyFile(emptyPath) assert.Nil(err) assert.True(ok) // not empty nonEmptyPath := path.Join(os.TempDir(), "non-empty") defer func() { os.RemoveAll(nonEmptyPath) }() f, err = os.Create(nonEmptyPath) assert.Nil(err) _, err = f.Write([]byte("some content")) assert.Nil(err) assert.Nil(f.Close()) ok, err = isEmptyFile(nonEmptyPath) assert.Nil(err) assert.False(ok) } ================================================ FILE: cmd/sourced/dir/dir.go ================================================ // Package dir provides functions to manage the config directories. package dir import ( "fmt" "io" "net/http" "os" "path/filepath" "github.com/pkg/errors" goerrors "gopkg.in/src-d/go-errors.v1" ) // ErrNotExist is returned when config dir does not exists var ErrNotExist = goerrors.NewKind("%s does not exist") // ErrNotValid is returned when config dir is not valid var ErrNotValid = goerrors.NewKind("%s is not a valid config directory: %s") // ErrNetwork is returned when could not download var ErrNetwork = goerrors.NewKind("network error downloading %s") // ErrWrite is returned when could not write var ErrWrite = goerrors.NewKind("write error at %s") // Path returns the absolute path for $SOURCED_DIR, or $HOME/.sourced if unset // and returns an error if it does not exist or it could not be read. func Path() (string, error) { srcdDir, err := srcdPath() if err != nil { return "", err } if err := validate(srcdDir); err != nil { return "", err } return srcdDir, nil } func srcdPath() (string, error) { if d := os.Getenv("SOURCED_DIR"); d != "" { abs, err := filepath.Abs(d) if err != nil { return "", errors.Wrap(err, fmt.Sprintf("could not resolve SOURCED_DIR='%s'", d)) } return abs, nil } homedir, err := os.UserHomeDir() if err != nil { return "", errors.Wrap(err, "could not detect home directory") } return filepath.Join(homedir, ".sourced"), nil } // Prepare tries to create the config directory, returning an error if it could not // be created, or nil if already exist or was successfully created. func Prepare() error { srcdDir, err := srcdPath() if err != nil { return err } err = validate(srcdDir) if ErrNotExist.Is(err) { if err := os.MkdirAll(srcdDir, os.ModePerm); err != nil { return ErrNotValid.New(srcdDir, err) } return nil } return err } // validate validates that the passed config dir path is valid func validate(path string) error { info, err := os.Stat(path) if os.IsNotExist(err) { return ErrNotExist.New(path) } if err != nil { return ErrNotValid.New(path, err) } if !info.IsDir() { return ErrNotValid.New(path, "it is not a directory") } readWriteAccessMode := os.FileMode(0700) if info.Mode()&readWriteAccessMode != readWriteAccessMode { return ErrNotValid.New(path, "it has no read-write access") } return nil } // DownloadURL downloads the given url to a file to the // dst path, creating the directory if it's needed func DownloadURL(url, dst string) (err error) { resp, err := http.Get(url) if err != nil { return ErrNetwork.Wrap(err, url) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return ErrNetwork.Wrap(fmt.Errorf("HTTP status %v", resp.Status), url) } if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { return ErrWrite.Wrap(err, filepath.Dir(dst)) } out, err := os.Create(dst) if err != nil { return ErrWrite.Wrap(err, dst) } defer out.Close() _, err = io.Copy(out, resp.Body) if err != nil { return ErrWrite.Wrap(err, dst) } return nil } // TmpPath returns the absolute path for /tmp/srcd func TmpPath() string { return filepath.Join(os.TempDir(), "srcd") } ================================================ FILE: cmd/sourced/dir/dir_test.go ================================================ package dir import ( "fmt" "io/ioutil" "net/http" "net/http/httptest" "os" "path" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) func TestValidate(t *testing.T) { assert := assert.New(t) err := validate("/does/not/exists") assert.True(ErrNotExist.Is(err)) assert.EqualError(err, "/does/not/exists does not exist") // with a file tmpFile := path.Join(os.TempDir(), "tmp-file") f, err := os.Create(tmpFile) assert.Nil(err) assert.Nil(f.Close()) defer func() { os.RemoveAll(tmpFile) }() err = validate(tmpFile) assert.True(ErrNotValid.Is(err)) assert.EqualError(err, tmpFile+" is not a valid config directory: it is not a directory") // with a dir tmpDir := path.Join(os.TempDir(), "tmp-dir") assert.Nil(os.Mkdir(tmpDir, os.ModePerm)) defer func() { os.RemoveAll(tmpDir) }() err = validate(tmpDir) assert.Nil(err) // read only assert.Nil(os.Chmod(tmpDir, 0444)) err = validate(tmpDir) assert.True(ErrNotValid.Is(err)) assert.EqualError(err, tmpDir+" is not a valid config directory: it has no read-write access") // write only assert.Nil(os.Chmod(tmpDir, 0222)) err = validate(tmpDir) assert.True(ErrNotValid.Is(err)) assert.EqualError(err, tmpDir+" is not a valid config directory: it has no read-write access") } func TestPrepare(t *testing.T) { assert := assert.New(t) tmpDir := path.Join(os.TempDir(), "tmp-dir") assert.Nil(os.Mkdir(tmpDir, os.ModePerm)) defer func() { os.RemoveAll(tmpDir) }() originSrcdDir := os.Getenv("SOURCED_DIR") defer func() { os.Setenv("SOURCED_DIR", originSrcdDir) }() os.Setenv("SOURCED_DIR", tmpDir) assert.Nil(Prepare()) toCreateDir := path.Join(os.TempDir(), "to-create-dir") defer func() { os.RemoveAll(toCreateDir) }() _, err := os.Stat(toCreateDir) assert.True(os.IsNotExist(err)) os.Setenv("SOURCED_DIR", toCreateDir) assert.Nil(Prepare()) _, err = os.Stat(toCreateDir) assert.Nil(err) } func TestDownloadURL(t *testing.T) { assert := assert.New(t) // success fileContext := []byte("hello") handler := func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(fileContext) } server := httptest.NewServer(http.HandlerFunc(handler)) dirPath := path.Join(os.TempDir(), "some-dir") filePath := path.Join(dirPath, "file-to-download") defer func() { os.RemoveAll(dirPath) }() assert.Nil(DownloadURL(server.URL, filePath)) _, err := os.Stat(filePath) assert.Nil(err) b, err := ioutil.ReadFile(filePath) assert.Nil(err) assert.Equal(fileContext, b) // error handler = func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) } server = httptest.NewServer(http.HandlerFunc(handler)) err = DownloadURL(server.URL, "/dev/null") errExpected := errors.Wrapf( fmt.Errorf("HTTP status %v", "404 Not Found"), "network error downloading %s", server.URL, ) assert.EqualError(err, errExpected.Error()) } ================================================ FILE: cmd/sourced/format/colors.go ================================================ package format import ( "fmt" "runtime" ) // Color represents a color code type Color string const ( // Red for errors Red Color = "31" // Yellow for warnings Yellow Color = "33" ) // Colorize returns the passed string with the passed color func Colorize(color Color, s string) string { if runtime.GOOS == "windows" { return s } return fmt.Sprintf("\x1b[%sm%s\x1b[0m", color, s) } ================================================ FILE: cmd/sourced/main.go ================================================ package main import ( "fmt" "github.com/src-d/sourced-ce/cmd/sourced/cmd" composefile "github.com/src-d/sourced-ce/cmd/sourced/compose/file" "github.com/src-d/sourced-ce/cmd/sourced/format" "github.com/src-d/sourced-ce/cmd/sourced/release" ) // this variable is rewritten during the CI build step var version = "master" var build = "dev" func main() { composefile.SetVersion(version) cmd.Init(version, build) checkUpdates() cmd.Execute() } func checkUpdates() { if version == "master" { return } update, latest, err := release.FindUpdates(version) if err != nil { return } if update { s := fmt.Sprintf( `There is a newer version. Current version: %s, latest version: %s Please go to https://github.com/src-d/sourced-ce/releases/latest to upgrade. `, version, latest) fmt.Println(format.Colorize(format.Yellow, s)) } } ================================================ FILE: cmd/sourced/release/release.go ================================================ // Package release deals with versioning and releases package release import ( "context" "net/http" "os" "path/filepath" "github.com/src-d/sourced-ce/cmd/sourced/dir" "github.com/blang/semver" "github.com/google/go-github/v25/github" "github.com/gregjones/httpcache" "github.com/gregjones/httpcache/diskcache" ) // FindUpdates calls the GitHub API to check the latest release tag. It returns // true if the latest stable release is newer than the current tag, and also // that latest tag name. func FindUpdates(current string) (update bool, latest string, err error) { currentV, err := semver.ParseTolerant(current) if err != nil { return false, "", err } diskcachePath := filepath.Join(dir.TmpPath(), "httpcache") err = os.MkdirAll(diskcachePath, os.ModePerm) if err != nil { return false, "", err } cache := diskcache.New(diskcachePath) client := github.NewClient(&http.Client{Transport: httpcache.NewTransport(cache)}) rel, _, err := client.Repositories.GetLatestRelease(context.Background(), "src-d", "sourced-ce") if err != nil { return false, "", err } latestV, err := semver.ParseTolerant(rel.GetTagName()) if err != nil { return false, "", err } update = latestV.GT(currentV) latest = latestV.String() return update, latest, nil } ================================================ FILE: cmd/sourced/release/release_test.go ================================================ package release import ( "bytes" "fmt" "io/ioutil" "net/http" "os" "path/filepath" "testing" "github.com/src-d/sourced-ce/cmd/sourced/dir" "github.com/stretchr/testify/assert" ) type testCase struct { responseTag string // tag returned by github current string update bool latest string } func TestFindUpdatesSuccess(t *testing.T) { os.RemoveAll(filepath.Join(dir.TmpPath(), "httpcache")) cases := []testCase{ { responseTag: "v0.14.0", current: "v0.14.0", update: false, latest: "0.14.0", }, { responseTag: "v0.11.0", current: "v0.14.0", update: false, latest: "0.11.0", }, { responseTag: "v0.14.0", current: "v0.13.0", update: true, latest: "0.14.0", }, { responseTag: "v0.14.0", current: "v0.13.1", update: true, latest: "0.14.0", }, } for _, c := range cases { name := fmt.Sprintf("%s_to_%s", c.current, c.responseTag) t.Run(name, func(t *testing.T) { restore := mockGithub(c.responseTag) defer restore() update, latest, err := FindUpdates(c.current) assert.Nil(t, err) assert.Equal(t, c.update, update) assert.Equal(t, c.latest, latest) }) } } func mockGithub(tag string) func() { originalTransport := http.DefaultTransport http.DefaultTransport = &ghTransport{tag: tag} return func() { http.DefaultTransport = originalTransport } } type ghTransport struct { tag string } func (t *ghTransport) RoundTrip(*http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusOK, Header: make(http.Header), Body: ioutil.NopCloser(bytes.NewBufferString(fmt.Sprintf(`{"tag_name": "%s"}`, t.tag))), }, nil } ================================================ FILE: docker-compose.yml ================================================ version: '3.4' x-superset-env: &superset-env SYNC_MODE: ${GITBASE_SIVA} ADMIN_LOGIN: admin ADMIN_FIRST_NAME: admin ADMIN_LAST_NAME: admin ADMIN_EMAIL: admin@example.com ADMIN_PASSWORD: admin POSTGRES_DB: superset POSTGRES_USER: superset POSTGRES_PASSWORD: superset POSTGRES_HOST: postgres POSTGRES_PORT: 5432 REDIS_HOST: redis REDIS_PORT: 6379 GITBASE_DB: gitbase GITBASE_USER: root GITBASE_PASSWORD: GITBASE_HOST: gitbase GITBASE_PORT: 3306 METADATA_DB: metadata METADATA_USER: metadata METADATA_PASSWORD: metadata METADATA_HOST: metadatadb METADATA_PORT: 5432 BBLFSH_WEB_HOST: bblfsh-web BBLFSH_WEB_PORT: 8080 services: bblfsh: image: bblfsh/bblfshd:v2.15.0-drivers restart: unless-stopped privileged: true ports: - 9432:9432 gitcollector: image: srcd/gitcollector:v0.0.4 # wait for db command: ['/bin/sh', '-c', 'sleep 10s && gitcollector download'] environment: GITHUB_ORGANIZATIONS: ${GITHUB_ORGANIZATIONS-} GITHUB_TOKEN: ${GITHUB_TOKEN-} # use main db GITCOLLECTOR_METRICS_DB_URI: postgresql://superset:superset@postgres:5432/superset?sslmode=disable GITCOLLECTOR_NO_UPDATES: 'true' GITCOLLECTOR_NO_FORKS: ${NO_FORKS-true} LOG_LEVEL: ${LOG_LEVEL-info} depends_on: - postgres volumes: - type: ${GITBASE_VOLUME_TYPE} source: ${GITBASE_VOLUME_SOURCE} target: /library consistency: delegated deploy: resources: limits: cpus: ${GITCOLLECTOR_LIMIT_CPU-0.0} ghsync: image: srcd/ghsync:v0.2.0 entrypoint: ['/bin/sh'] # wait for db to be created # we need to use something like https://github.com/vishnubob/wait-for-it # or implement wait in ghsync itself command: ['-c', 'sleep 10s && ghsync migrate && ghsync shallow'] depends_on: - metadatadb environment: GHSYNC_ORGS: ${GITHUB_ORGANIZATIONS-} GHSYNC_TOKEN: ${GITHUB_TOKEN-} GHSYNC_POSTGRES_DB: metadata GHSYNC_POSTGRES_USER: metadata GHSYNC_POSTGRES_PASSWORD: metadata GHSYNC_POSTGRES_HOST: metadatadb GHSYNC_POSTGRES_PORT: 5432 GHSYNC_NO_FORKS: ${NO_FORKS-true} LOG_LEVEL: ${LOG_LEVEL-info} gitbase: image: srcd/gitbase:v0.23.1 restart: unless-stopped ports: - 3306:3306 environment: BBLFSH_ENDPOINT: bblfsh:9432 SIVA: ${GITBASE_SIVA} GITBASE_LOG_LEVEL: ${LOG_LEVEL-info} depends_on: - bblfsh volumes: - type: ${GITBASE_VOLUME_TYPE} source: ${GITBASE_VOLUME_SOURCE} target: /opt/repos read_only: true consistency: delegated - gitbase_indexes:/var/lib/gitbase/index deploy: resources: limits: cpus: ${GITBASE_LIMIT_CPU-0.0} memory: ${GITBASE_LIMIT_MEM-0} bblfsh-web: image: bblfsh/web:v0.11.4 restart: unless-stopped command: -bblfsh-addr bblfsh:9432 ports: - 9999:8080 depends_on: - bblfsh environment: LOG_LEVEL: ${LOG_LEVEL-info} redis: image: redis:5-alpine restart: unless-stopped ports: - 6379:6379 volumes: - redis:/data postgres: image: postgres:10-alpine restart: unless-stopped environment: POSTGRES_DB: superset POSTGRES_PASSWORD: superset POSTGRES_USER: superset ports: - 5432:5432 volumes: - postgres:/var/lib/postgresql/data metadatadb: image: postgres:10-alpine restart: unless-stopped environment: POSTGRES_DB: metadata POSTGRES_PASSWORD: metadata POSTGRES_USER: metadata ports: - 5433:5432 volumes: - metadata:/var/lib/postgresql/data sourced-ui: image: srcd/sourced-ui:v0.8.1 restart: unless-stopped environment: <<: *superset-env SUPERSET_ENV: production ports: - 8088:8088 depends_on: - postgres - metadatadb - redis - gitbase - bblfsh-web sourced-ui-celery: image: srcd/sourced-ui:v0.8.1 restart: unless-stopped environment: <<: *superset-env SUPERSET_ENV: celery depends_on: - postgres - metadatadb - redis - gitbase - sourced-ui volumes: gitbase_repositories: external: false gitbase_indexes: external: false metadata: external: false postgres: external: false redis: external: false ================================================ FILE: docs/CONTRIBUTING.md ================================================ # Contribution Guidelines As all source{d} projects, this project follows the [source{d} Contributing Guidelines](https://github.com/src-d/guide/blob/master/engineering/documents/CONTRIBUTING.md). # Additional Contribution Guidelines In addition to the [source{d} Contributing Guidelines](https://github.com/src-d/guide/blob/master/engineering/documents/CONTRIBUTING.md), this project follows the following guidelines. ## Changelog This project lists the important changes between releases in the [`CHANGELOG.md`](../CHANGELOG.md) file. If you open a PR, you should also add a brief summary in the `CHANGELOG.md` mentioning the new feature, change or bugfix that you proposed. ## How To Restore Dashboards and Charts to Defaults The official way to restore **source{d} CE** to its initial state, is to remove the running components with `sourced prune --all`, and then init again with `sourced init`. In some circumstances you need to restore only the state modified from the UI (charts, dashboards, saved queries, users, roles, etcetera), using the default ones for the version of **source{d} CE** that you're currently using, and preserve the repositories and metadata fetched from GitHub organizations. To do so, you only need to delete the docker volume containing the PostgreSQL database, and restart **source{d} CE**. It can be done following these steps if you already have [Docker Compose](https://docs.docker.com/compose/) installed: ```shell $ cd ~/.sourced/workdirs/__active__ $ source .env $ docker-compose stop postgres $ docker-compose rm -f postgres $ ENV_PREFIX=`awk '{print tolower($0)}' <<< ${COMPOSE_PROJECT_NAME}` $ docker volume rm ${ENV_PREFIX}_postgres $ docker-compose up -d postgres $ docker-compose exec -u superset sourced-ui bash -c 'sleep 10s && python bootstrap.py' ``` ================================================ FILE: docs/README.md ================================================ # Table of contents * [Introduction](../README.md) * [Quickstart](./quickstart/README.md) * [Dependencies](./quickstart/1-install-requirements.md) * [Install **source{d} CE**](./quickstart/2-install-sourced.md) * [Run **source{d} CE**](./quickstart/3-init-sourced.md) * [Explore Your Data](./quickstart/4-explore-sourced.md) * [Next steps](./usage/README.md) ## Usage * [sourced Command Reference](./usage/commands.md) * [Multiple Datasets](./usage/multiple-datasets.md) * [SQL Examples](./usage/examples.md) * [Babelfish UAST](./usage/bblfsh.md) ## Learn More * [FAQ](./learn-more/faq.md) * [Troubleshooting](./learn-more/troubleshooting.md) * [Architecture](./learn-more/architecture.md) * [Contribute](./CONTRIBUTING.md) * [Changelog](../CHANGELOG.md) * [License](../LICENSE.md) ## Resources * [GitHub Repository](https://github.com/src-d/sourced-ce) * [Product Page](https://www.sourced.tech) * [Book a Demo](https://go.sourced.tech/community-demo) * [Get in Touch With Us](http://go.sourced.tech/contact) * [Join Us on Slack](https://sourced-community.slack.com/join/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM) * [source{d} Forum](https://forum.sourced.tech) ================================================ FILE: docs/learn-more/architecture.md ================================================ # source{d} Community Editon Architecture **source{d} Community Editon** provides a frictionless experience for trying source{d} for Code Analysis. ## Technical Architecture The `sourced` binary, a single CLI binary [written in Go](../../cmd/sourced/main.go), is the user's main interaction mechanism with **source{d} CE**. It is also the only piece (other than Docker) that the user will need to explicitly download on their machine to get started. The `sourced` binary manages the different installed environments and their configurations, acting as a wrapper of Docker Compose. The whole architecture is based on Docker containers, orchestrated by Docker Compose and managed by `sourced`. ## Components of source{d} **source{d} CE** relies on different components to handle different use cases and to cover different functionalities. Each component is implemented as a running Docker container. - `bblfsh`: parses source code into UASTs using [Babelfish](https://docs.sourced.tech/babelfish/); you can learn more about it in our [Babelfish UAST guide](usage/bblfsh.md) - `gitbase`: runs [gitbase](https://docs.sourced.tech/gitbase), a SQL database interface to Git repositories. - `gitcollector`: is responsible for fetching repositories from the organizations used to initialize **source{d} CE**. It uses [gitcollector](https://github.com/src-d/gitcollector). - `ghsync`: is responsible for fetching repository metadata from the organizations used to initialize **source{d} CE**. It uses [ghsync](https://github.com/src-d/ghsync) - `metadatadb`: runs the PostgreSQL database that stores the repositories metadata (users, pull requests, issues...) extracted by `ghsync`. - `postgres`: runs the PostgreSQL database that stores the state of the UI (charts, dashboards, users, saved queries and such). - `sourced-ui`: runs the **source{d} CE** Web Interface. This component queries data from `bblfsh`, `gitbase`, `metadatadb` and `postgres`. Some of these components can be accessed from the outside as described by [Docker Networking section](#docker-networking). ## Docker Set Up In order to make this work in the easiest way, some design decisions were made: ### Isolated Environments. _Read more in [Working With Multiple Data Sets](../usage/multiple-datasets.md)_ Each dataset runs in an isolated environment, and only one environment can run at the same time. Each environment is defined by one `docker-compose.yml` and one `.env`, stored in `~/.sourced`. ### Docker Naming All the Docker containers from the same environment share its prefix: `srcd-_` followed by the name of the service running inside, e.g `srcd-c3jjlwq_gitbase_1` and `srcd-c3jjlwq_bblfsh_1` will contain gitbase and babelfish for the same environment. ### Docker Networking In order to provide communication between the multiple containers started, all of them are attached to the same single bridge network. The network name also has the same prefix than the containers inside the same environment, e.g. `srcd-c3jjlwq_default`. Some environment services can be accessed from the outside, using their exposed port and connection values: - `bblfsh`: - port: `9432` - `gitbase`: - port: `3306` - database: `gitbase` - user: `root` - `metadatadb`: - port: `5433` - database: `metadata` - user: `metadata` - password: `metadata` - `sourced-ui`: - port: `8088` ### Persistence To prevent losing data when restarting services, or upgrading containers, the data is stored in volumes. These volumes also share the same prefix with the containers in the same environment, e.g. `srcd-c3jjlwq_gitbase_repositories`. These are the most relevant volumes: - `gitbase_repositories`, stores the repositories to be analyzed - `gitbase_indexes`, stores the gitbases indexes - `metadata`, stores the metadata from GitHub pull requests, issues, users... - `postgres`, stores the dashboards and charts used by the web interface ================================================ FILE: docs/learn-more/faq.md ================================================ # Frequently Asked Questions _For tips and advices to deal with unexpected errors, please refer to [Troubleshooting guide](./troubleshooting.md)_ ## Index - [Where Can I Find More Assistance to Run source{d} or Notify You About Any Issue or Suggestion?](#where-can-i-find-more-assistance-to-run-source-d-or-notify-you-about-any-issue-or-suggestion) - [How Can I Update My Version Of **source{d} CE**?](#how-can-i-update-my-version-of-source-d-ce) - [How to Update the Data from the Organizations That I'm Analyzing](#how-to-update-the-data-from-the-organizations-being-analyzed) - [Can I Query Gitbase or Babelfish with External Tools?](#can-i-query-gitbase-or-babelfish-with-external-tools) - [Where Can I Read More About the Web Interface?](#where-can-i-read-more-about-the-web-interface) - [I Get IOError Permission denied](#i-get-ioerror-permission-denied) - [Why Do I Need Internet Connection?](#why-do-i-need-internet-connection) ## Where Can I Find More Assistance to Run source{d} or Notify You About Any Issue or Suggestion? _If you're dealing with an error or something that you think that can be caused by an unexpected error, please refer to our [Troubleshooting guide](./troubleshooting.md). With the info that you can obtain following those steps, you could fix the problem or you will be able to explain it better in the following channels:_ * [open an issue](https://github.com/src-d/sourced-ce/issues), if you want to suggest a new feature, if you need assistance with a contribution, or if you found any bug. * [Visit the source{d} Forum](https://forum.sourced.tech) where users and community members discuss anything source{d} related. You will find there some common questions from other source{d} users, or ask yours. * [join our community on Slack](https://sourced-community.slack.com/join/shared_invite/enQtMjc4Njk5MzEyNzM2LTFjNzY4NjEwZGEwMzRiNTM4MzRlMzQ4MmIzZjkwZmZlM2NjODUxZmJjNDI1OTcxNDAyMmZlNmFjODZlNTg0YWM), and talk with some of our engineers. ## How Can I Update My Version Of source{d} CE? When there is a new release of **source{d} CE**, it is noticed every time a `sourced` command is called. When it happens you can download the new version from [src-d/sourced-ce/releases/latest](https://github.com/src-d/sourced-ce/releases/latest), and proceed as it follows: (You can also follow these steps if you want to update to any beta version, to downgrade, or to use your own built version of **source{d} CE**) 1. replace your current version of `sourced` from its current location with the one you're installing ([see Quickstart. Install](quickstart/2-install-sourced.md)), and confirm it was done by running `sourced version`. 1. run `sourced compose download` to download the new configuration. 1. run `sourced restart` to apply the new configuration. This process will reinstall **source{d} CE** with the new components, but it will keep your current data (repositories, metadata, charts, dashboards, etc) of your existent workdirs. If you want to replace all your current customizations —including charts and dashboards—, with the ones from the release that you just installed, the official way to proceed is to `prune` the running workdirs, and `init` them again. _**disclaimer:** pruning a workdir will delete all its data: its saved queries and charts, and if you were using repositories and metadata downloaded from a GitHub organization, they will be deleted, and downloaded again._ 1. `sourced status workdirs` to get the list of your current workdirs 1. Prune the workdirs you need, or prune all of them at once running `sourced prune --all` 1. `sourced init [local|orgs] ...` for each workdir again, to initialize them with the new configuration. ## How to Update the Data from the Organizations Being Analyzed There is no way to update imported data, and [when a scraper is restarted](./troubleshooting.md#how-can-i-restart-one-scraper), it procedes as it follows: ### gitcollector Organizations and repositories are downloaded independently, so if they fail, the process is not stopped until all the organizations and repositories have been iterated. If `gitcollector` is restarted, it will download more repositories, but it won’t update any of the already existent ones. You can see the progress of the new process in the welcome dashboard; since already existent repositories won't be updated, those will appear as `failed` in progress status. ### ghsync The way how metadata is imported by `ghsync` is a bit different, and it is done sequentially per each organization, so if any step fails, the whole importation will fail. Pull requests, issues, and users of the same organization, are imported in that order in separate transaction each one, and if one transaction fails, the process will be stopped so the next ones won't be processed. Once the three different entities have been imported, the organization will be considered as "done", and restarting `ghsync` won't cause to update its data. If `ghsync` is restarted, it will only import data from organizations that could not be finished considering the rules explained above. The process of `ghsync` will be updated in the welcome dashboard and if an organization was already imported, it will appear as "nothing imported" in the status chart. ## Can I Query Gitbase or Babelfish with External Tools? Yes, as explained in our docs about [**source{d} CE** Architecture](./architecture.md#docker-networking), these and other components are exposed to the host machine, to be used by third party tools like [Jupyter Notebook](https://jupyter.org/), [gitbase clients](https://docs.sourced.tech/gitbase/using-gitbase/supported-clients) and [Babelfish clients](https://docs.sourced.tech/babelfish/using-babelfish/clients). The connection values that you should use to connect to these components, are defined in the [`docker-compose.yml`](../docker-compose.yml), and sumarized in the [Architecture documentation](./architecture.md#docker-networking) ## Where Can I Read More About the Web Interface? The user interface is based in the open-sourced [Apache Superset](http://superset.apache.org), so you can also refer to [Superset tutorials](http://superset.apache.org/tutorial.html) for advanced usage of the web interface. ## I Get IOError Permission denied If you get this error message: ``` IOError: [Errno 13] Permission denied: u'./.env' ``` This may happen if you have installed Docker from a snap package. This installation mode is not supported, please install it following [the official documentation](./quickstart/1-install-requirements.md#install-docker) (See [#78](https://github.com/src-d/sourced-ce/issues/78)). ## Why Do I Need Internet Connection? source{d} CE automatically fetches some resources from the Internet when they are not found locally: - the source{d} CE configuration is fetched automatically when initializing it for the first time, using the proper version for the current version of `sourced`, e.g. if using `v0.16.0` it will automatically fetch `https://raw.githubusercontent.com/src-d/sourced-ce/v0.16.0/docker-compose.yml`. - to download the docker images of the source{d} CE components when initializing source{d} for the first time, or when initializing it after changing its configuration. - to download repositories and its metadata from GitHub when you initialize source{d} CE with `sourced init orgs`. - to download and install [Docker Compose alternative](#docker-compose) if there is no local installation of Docker Compose. If your connection to the network does not let source{d} CE to access to Internet, you should manually provide all these dependencies. ================================================ FILE: docs/learn-more/troubleshooting.md ================================================ # Troubleshooting: _For commonly asked questions and their answers, you can refer to the [FAQ](./faq.md)_ Currently, **source{d} CE** does not expose nor log all errors directly into the UI. In the current stage of **source{d} CE**, following these steps is the better way to know if something is failing, why, and to know how to recover the app from some problems. The first two steps use to be always mandatory: 1. **[To see if any component is broken](#how-can-i-see-the-status-of-source-d-ce-components)** 1. **[To see the logs of the running components](#how-can-i-see-logs-of-the-running-components)** 1. [To know if scrapers finished their job](#how-can-i-see-what-happened-with-the-scrapers) - [To restart one scraper](#how-can-i-restart-one-scraper) 1. [To restart o initialize **source{d} CE** again](#how-to-restart-source-d-ce) 1. [To ask for help if the issue could not be solved](./faq.md#where-can-i-find-more-assistance-to-run-source-d-or-notify-you-about-any-issue-or-suggestion) Other issues that we detected, and which are strictly related to the UI are: - [When I Try to Create a Chart from a Query, Nothing Happens.](#when-i-try-to-create-a-chart-from-a-query-nothing-happens) - [When I Try to Export a Dashboard, Nothing Happens.](#when-i-try-to-export-a-dashboard-nothing-happens) - [The Dashboard Takes a Long to Load, and the UI Freezes.](#the-dashboard-takes-a-long-to-load-and-the-ui-freezes) ## source{d} CE Fails During Its Initialization The initialization can fail fast if there is any port conflict, or missing config file, etcetera; those errors are clearly logged in the terminal when they appear. If when initializing **source{d} CE**, all the required components appear as created, but the loading spinner keeps spinning forever (more than 1 minute can be symptomatic), there can be an underlying problem causing the UI not to be opened. In this situation you should: 1. **[See if any component is broken](#how-can-i-see-the-status-of-source-d-ce-components)** 1. **[See app logs or certain component logs](#how-can-i-see-logs-of-the-running-components)** 1. [Restart o initialize **source{d} CE** again](#how-to-restart-source-d-ce) 1. [To ask for help if the issue could not be solved](./faq.md#where-can-i-find-more-assistance-to-run-source-d-or-notify-you-about-any-issue-or-suggestion) ## How Can I See the Status of source{d} CE Components? To see the status of **source{d} CE** components, just run: ``` $ sourced status Name Command State Ports ------------------------------------------------------------------------------ srcd-xxx_sourced-ui_1 /entrypoint.sh Up (healthy) :8088->8088 srcd-xxx_gitbase_1 ./init.sh Up :3306->3306 srcd-xxx_bblfsh_1 /tini -- bblfshd Up :9432->9432 srcd-xxx_bblfsh-web_1 /bin/bblfsh-web -addr ... Up :9999->8080 srcd-xxx_metadatadb_1 docker-entrypoint.sh ... Up :5433->5432 srcd-xxx_postgres_1 docker-entrypoint.sh ... Up :5432->5432 srcd-xxx_redis_1 docker-entrypoint.sh ... Up :6379->6379 srcd-xxx_ghsync_1 /bin/sh -c sleep 10s ... Exit 0 srcd-xxx_gitcollector_1 /bin/dumb-init -- /bi ... Exit 0 ``` It will report the status of all **source{d} CE** component. All components should be `Up`, but the scrapers: `ghsync` and `gitcollector`; these exceptions are explanined in [How Can I See What Happened with the Scrapers?](#how-can-i-see-what-happened-with-the-scrapers) If any component is not `Up` (but the scrapers), here are some key points to understand what might be happening: - All the components (but the scrapers) are restarted by Docker Compose automatically —process that can take some seconds—; if the component enters in a restart loop, something wrong is happening. - When any component is failing, or died, you should [see its logs to understand what is happening](#how-can-i-see-logs-of-the-running-components) When one of the required components fails, it uses to print an error in the UI, e.g. `lost connection to mysql server during query` while running a query might mean that `gitbase` went down. e.g. `unable to establish a connection with the bblfsh server: deadline exceeded` in SQL Lab might mean that `bblfsh` went down. If the failing component is not successfully restarted in a few seconds, or if it goes down when running certain queries, it could be a good idea to [open an issue](https://github.com/src-d/sourced-ce/issues) describing the problem. ## How Can I See Logs of The Running Components? ```shell $ sourced logs [-f] [components...] ``` Adding `-f` will keep the connection opened, and the logs will appear as they come instead of exiting after the last logged one. You can pass a space-separated list of component names to see only their logs (i.e. `sourced-ui`, `gitbase`, `bblfsh`, `gitcollector`, `ghsync`, `metadatadb`, `postgres`, `redis`). If you do not pass any component name, there will appear the logs of all of them. Currently, there is no way to filter by error level, so you could try with `grep`, e.g. ```shell sourced logs gitcollector | grep error ``` will output only log lines where `error` word appears. ## How Can I See What Happened with the Scrapers? _When **souece{d} CE** is initialized with `sourced init local`, the scrapers are not relevant because the repositories to analyze comes from your local data, so `ghsync` and `gitcollector` status is not relevant in this case._ When running **souece{d} CE** to analyze data from a list of GitHub organizations, `gitcollector` component is in charge of fetching GitHub repositories and `ghsync` component is in charge of fetching GitHub metadata (issues, pull requests...) Once the UI is opened, you can see the progress of the importation in the welcome dashboard, reporting the data imported, skipped, failed and completed. The process can take many minutes if the organization is big, so be patient. You can manually refresh both charts to confirm that the process is progressing, and it is not stuck. If you believe that there can be any problem during the process, the better way to find what is happening is: - **[check the components status](#how-can-i-see-the-status-of-source-d-ce-components) with `sourced status`**; `gitcollector` and `ghsync` should be `Up` (the process didn't finish yet), or `Exit 0` (the process finished succesfully). They are independent components, so they can finish on different order depending on how many repositories or metadata is needed to process. - **[check the logs](#how-can-i-see-logs-of-the-running-components) of the failing component with `sourced logs [-f] {gitcollector,ghsync}`** to get more info about the errors found. ## How Can I Restart One Scraper? _Restarting a scraper should be done to recover from temporal problems like connectivity loss, or lack of space in disc, not [to update the data you're analyzing](./faq.md#how-to-update-the-data-from-the-organizations-being-analyzed)_ **source{d} CE** does not provide way to start only one scraper. The recommended way to restart them would be [to restart the whole **source{d} CE**](#how-to-restart-source-d-ce), which is fast and safe for your data. In order to restart **source{d} CE**, run: ```shell $ sourced restart ``` _Read more about [which data will be imported after restarting a scraper](./faq.md#how-to-update-the-data-from-the-organizations-being-Analyzed)_ If you feel comfortable enough with Docker Compose, you could also try restarting each scraper separatelly, running: ```shell $ cd ~/.sourced/workdirs/__active__ $ docker-compose run gitcollector # to restart gitcollector $ docker-compose run ghsync # to restart ghsync ``` ## How to Restart source{d} CE Restarting **source{d} CE**, can fix some errors and is also the official way to restart the scrapers. It is also needed after downloading a new config (by running `sourced compose download`). **source{d} CE** is restarted with the command: ```shell $ sourced restart ``` It only recreates the component containers, keeping all your data, like charts, dashboards, repositories, and GitHub metadata. ## When I Try to Create a Chart from a Query, Nothing Happens. The charts can be created from the SQL Lab, using the `Explore` button once you run a query. If nothing happens, the browser may be blocking the new window that should be opened to edit the new chart. You should configure your browser to let source{d} UI to open pop-ups (e.g. in Chrome it is done allowing `127.0.0.1:8088` to handle `pop-ups and redirects` from the `Site Settings` menu). ## When I Try to Export a Dashboard, Nothing Happens. If nothing happens when pressing the `Export` button from the dashboard list, then you should configure your browser to let source{d} UI to open pop-ups (e.g. in Chrome it is done allowing `127.0.0.1:8088` to handle `pop-ups and redirects` from the `Site Settings` menu) ## The Dashboard Takes a Long to Load and the UI Freezes. _This is a known issue that we're trying to address, but here is more info about it._ In some circumstances, loading the data for the dashboards can take some time, and the UI can be frozen in the meanwhile. It can happen —on big datasets—, the first time you access the dashboards, or when they are refreshed. There are some limitations with how Apache Superset handles long-running SQL queries, which may affect the dashboard charts. Since most of the charts of the Overview dashboard loads its data from gitbase, its queries can take more time than the expected for the UI. When it happens, the UI can be frozen, or you can get this message in some charts: >_Query timeout - visualization queries are set to timeout at 300 seconds. Perhaps your data has grown, your database is under unusual load, or you are simply querying a data source that is too large to be processed within the timeout range. If that is the case, we recommend that you summarize your data further._ When it occurs, you should wait till the UI is responsive again, and separately refresh each failing chart with its `force refresh` option (on its top-right corner). With some big datasets, it took 3 refreshes and 15 minutes to get data for all charts. ================================================ FILE: docs/quickstart/1-install-requirements.md ================================================ # Install source{d} Community Edition Dependencies ## Install Docker _Please note that Docker Toolbox is not supported. In case that you're running Docker Toolbox, please consider updating to newer Docker Desktop for Mac or Docker Desktop for Windows._ _For Linux installations, using Docker installed from a snap package is not supported._ Follow the instructions based on your OS: - [Docker for Ubuntu Linux](https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-docker-ce-1) - [Docker for Arch Linux](https://wiki.archlinux.org/index.php/Docker#Installation) - [Docker for macOS](https://store.docker.com/editions/community/docker-ce-desktop-mac) - [Docker Desktop for Windows](https://hub.docker.com/editions/community/docker-ce-desktop-windows) (Make sure to read the [system requirements for Docker on Windows](https://docs.docker.com/docker-for-windows/install/).) Minimal supported version 18.02.0. ## Docker Compose **source{d} CE** is deployed as Docker containers, using [Docker Compose](https://docs.docker.com/compose), but it is not required to have a local installation of Docker Compose; if it is not found it will be downloaded from docker sources, and deployed inside a container. If you prefer a local installation of Docker Compose, or you have no access to internet to download it, you can follow the [Docker Compose install guide](https://docs.docker.com/compose/install) Minimal supported version 1.20.0. ## Internet Connection source{d} CE automatically fetches some resources from the Internet when they are not found locally, so in order to use all source{d} CE capacities, Internet connection is needed. For more details, you can refer to [Why Do I Need Internet Connection?](../learn-more/faq.md#why-do-i-need-internet-connection) ================================================ FILE: docs/quickstart/2-install-sourced.md ================================================ # Install source{d} Community Edition Download the **[latest release](https://github.com/src-d/sourced-ce/releases/latest)** for your Linux, macOS (Darwin) or Windows. ## On Linux or macOS Extract `sourced` binary from the release you downloaded, and move it into your bin folder to make it executable from any directory: ```bash $ tar -xvf path/to/sourced-ce_REPLACE-VERSION_REPLACE-OS_amd64.tar.gz $ sudo mv path/to/sourced-ce_REPLACE-OS_amd64/sourced /usr/local/bin/ ``` ## On Windows *Please note that from now on we assume that the commands are executed in `powershell` and not in `cmd`.* Create a directory for `sourced.exe` and add it to your `$PATH`, running these commands in a powershell as administrator: ```powershell mkdir 'C:\Program Files\sourced' # Add the directory to the `%path%` to make it available from anywhere setx /M PATH "$($env:path);C:\Program Files\sourced" # Now open a new powershell to apply the changes ``` Extract the `sourced.exe` executable from the release you downloaded, and copy it into the directory you created in the previous step: ```powershell mv \path\to\sourced-ce_windows_amd64\sourced.exe 'C:\Program Files\sourced' ``` ================================================ FILE: docs/quickstart/3-init-sourced.md ================================================ # Initialize source{d} Community Edition _For the full list of the sub-commands offered by `sourced`, please take a look at [the `sourced` sub-commands inventory](../usage/commands.md)._ **source{d} CE** can be initialized from 2 different data sources: GitHub organizations, or local Git repositories. Please note that you have to choose one data source to initialize **source{d} CE**, but you can have more than one isolated environment, and they can have different sources. See the guide about [Working With Multiple Data Sets](../usage/multiple-datasets.md) for more details. **source{d} CE** will download and install Docker images on demand. Therefore, the first time you run some of these commands, they might take a bit of time to start up. Subsequent runs will be faster. ## From GitHub Organizations When using GitHub organizations to populate the **source{d} CE** database you only need to provide a list of organization names and a [GitHub personal access token](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/). If no scope is granted to the user token, only public data will be fetched. To let **source{d} CE** access also private repositories and hidden users, the token should have the following scopes enabled: - `repo` Full control of private repositories - `read:org` Read org and team membership, read org projects Use this command to initialize, e.g. ```shell $ sourced init orgs --token src-d,bblfsh ``` It will also download, in the background, the repositories of the passed GitHub organizations, and their metadata: pull requests, issues, users... ## From Local Repositories ```shell $ sourced init local ``` It will initialize **source{d} CE** to analyze the git repositories under the passed path, or under the current directory if no one is passed. The repositories will be found recursively. **Note for macOS:** Docker for Mac [requires enabling file sharing](https://docs.docker.com/docker-for-mac/troubleshoot/#volume-mounting-requires-file-sharing-for-any-project-directories-outside-of-users) for any path outside of `/Users`. **Note for Windows:** Docker for Windows [requires shared drives](https://docs.docker.com/docker-for-windows/#shared-drives). Other than that, it's important to use a working directory that doesn't include any sub-directory whose access is not readable by the user running `sourced`. For example, using `C:\Users` as workdir, will most probably not work. For more details see [this issue](https://github.com/src-d/engine/issues/250). ## What's Next? Once **source{d} CE** has been initialized, it will automatically open the web UI. If the UI is not opened automatically, you can use `sourced web` command, or visit http://127.0.0.1:8088. Use login: `admin` and password: `admin`, to access the web interface. ================================================ FILE: docs/quickstart/4-explore-sourced.md ================================================ # Explore source{d} CE Web Interface _If you have any problem running **source{d} CE** you can take a look to our [Troubleshooting](../learn-more/troubleshooting.md) section, and to our [source{d} Forum](https://forum.sourced.tech), where you can also ask for help when using **source{d} CE**. If you spotted a bug, or have a feature request, please [open an issue](https://github.com/src-d/sourced-ce/issues) to let us know abut it._ _In some circumstances, loading the data for the dashboards can take some time, and the UI can be frozen in the meanwhile. It can happen —on big datasets—, the first time you access the dashboards, or when they are refreshed. Please, take a look to our [Troubleshooting](../learn-more/troubleshooting.md#the-dashboard-takes-a-long-to-load-and-the-ui-freezes) to get more info about this exact issue._ Once **source{d} CE** has been [initialized with `sourced init`](./3-init-sourced.md), it will automatically open the web UI. If the UI is not automatically opened, you can use `sourced web` command, or visit http://127.0.0.1:8088. Use login: `admin` and password: `admin`, to access the web interface. If you [initialized **source{d} CE** from GitHub Organizations](./3-init-sourced.md#from-github-oganizations), its repositories and metadata will be downloaded on background, and it will be available graduatelly. You will find more info in the welcome dashboard once you log in. ## Sections The most relevant features that **source{d} CE** Web Interface offers are: - **[SQL Lab](#sql-lab-querying-code-and-metadata)**, to query your repositories and its GitHub metadata. - **[Babelfish web](#uast-parsing-code)**, web interface to parse files into UAST. - **[Dashboards](#dashboards)**, to aggregate charts for exploring and visualizing your data. - **Charts**, to see your data with a rich set of data visualizations. - A flexible UI to manage users, data sources, export data... The user interface is based in the open-sourced [Apache Superset](http://superset.incubator.apache.org), so you can also refer to their documentation for advanced usage of the web interface. ## SQL Lab. Querying Code and Metadata _If you prefer to work within the terminal via command line, you can open a SQL REPL running `sourced sql`_ Using the `SQL Lab` tab, from the web interface, you can analyze your dataset using SQL queries, and create charts from those queries with the `Explore` button. You can find some sample queries in the [examples](../usage/examples.md). If you want to know what the database schema looks like you can use either regular `SHOW` or `DESCRIBE` queries, or you can refer to the [diagram about gitbase entities and relations](https://docs.sourced.tech/gitbase/using-gitbase/schema#database-diagram). ```bash $ sourced sql "SHOW tables;" +--------------+ | TABLE | +--------------+ | blobs | | commit_blobs | | commit_files | | commit_trees | | commits | | files | | ref_commits | | refs | | remotes | | repositories | | tree_entries | +--------------+ ``` ```bash $ sourced sql "DESCRIBE TABLE commits;" +---------------------+-----------+ | NAME | TYPE | +---------------------+-----------+ | repository_id | TEXT | | commit_hash | TEXT | | commit_author_name | TEXT | | commit_author_email | TEXT | | commit_author_when | TIMESTAMP | | committer_name | TEXT | | committer_email | TEXT | | committer_when | TIMESTAMP | | commit_message | TEXT | | tree_hash | TEXT | | commit_parents | JSON | +---------------------+-----------+ ``` ## UAST. Parsing code _Please, refer to the [quick explanation about what Babelfish is](../usage/bblfsh.md) to know more about it._ You can get UASTs from the `UAST` tab (parsing files by direct input), or using the `UAST` gitbase function over blob contents on `SQL Lab` tab. ## Dashboards _Please, refer to [Superset Tutorial, creating your first dashboard](http://superset.incubator.apache.org/tutorial.html) for more details._ The dashboards let you aggregate custom charts to show in the same place different metrics for your repositories. You can create them: - From the `Dashboard` tab, adding a new one, and then selecting new charts. - From any chart view, the `Save` button will let you to add it into a new or existent one. ================================================ FILE: docs/quickstart/README.md ================================================ # Quickstart This guide covers the full setup journey, from zero to populated dashboard with **source{d} Community Edition**. This process is divided into the following steps: 1. [Install **source{d} CE** Dependencies](./1-install-requirements.md) 1. [Install **source{d} CE**](./2-install-sourced.md) 1. [Initialize the Dataset](./3-init-sourced.md): - using local git data - using git repositories from your GitHub org 1. [Explore Your Dataset](./4-explore-sourced.md) ================================================ FILE: docs/usage/README.md ================================================ # Usage of source{d} Community Edition Once you know how to install and run **source{d} Community Edition**, you will find in this section some useful resources for guiding your first steps using this tool. - [`sourced` Command Reference](./commands.md) - [Using Multiple Datasets](./multiple-datasets.md) will show you how to analyze different datasets, no matter if they are stored locally or in GitHub. - [Some SQL Examples to Explore Your Dataset](./examples.md) - [Babelfish UAST](./bblfsh.md), about how to extract code features and understand code structure in a language-agnostic way". If you are interested in the different components of **source{d} Community Edition**, you can read more about it in the [docs about architecture](../learn-more/architecture.md) Some common questions are answered in the [FAQ](../learn-more/faq.md), and common problems and how to solve them in the [Troubleshooting](../learn-more/troubleshooting.md) guide. If you have any question about source{d} you can ask in our [source{d} Forum](https://forum.sourced.tech). If you spotted a bug, or have a feature request, please [open an issue](https://github.com/src-d/sourced-ce/issues) to let us know about it. ================================================ FILE: docs/usage/bblfsh.md ================================================ # Babelfish UAST _In the [Babelfish documentation](https://docs.sourced.tech/babelfish/), you will find detailed information about Babelfish specifications, usage, examples, etc._ One of the most important components of **source{d} CE** is the UAST, which stands for: [Universal Abstract Syntax Tree](https://docs.sourced.tech/babelfish/uast/uast-specification-v2). UASTs are a normalized form of a programming language's AST, annotated with language-agnostic roles and transformed with language-agnostic concepts (e.g. Functions, Imports, etc.). These enable an advanced static analysis of code and easy feature extraction for statistics or [Machine Learning on Code](https://github.com/src-d/awesome-machine-learning-on-source-code). ## UAST Usage From the web interface, you can use the `UAST` tab, to parse files by direct input, or you can also get UASTs from the `SQL Lab` tab, using the `UAST(content)` [gitbase function](https://docs.sourced.tech/gitbase/using-gitbase/functions). For the whole syntax about how to query the UASTs, you can refer to [How To Query UASTs With Babelfish](https://docs.sourced.tech/babelfish/using-babelfish/uast-querying) ## Supported Languages To see which languages are available, check the table of [Babelfish supported languages](https://docs.sourced.tech/babelfish/languages). ## Clients and Connectors The language parsing server (Babelfish) is available from the web interface, but you can also connect to the parsing server, deployed by **source{d} CE**, with several language clients, currently supported and maintained: - [Babelfish Go Client](https://github.com/bblfsh/go-client) - [Babelfish Python Client](https://github.com/bblfsh/client-python) - [Babelfish Scala Client](https://github.com/bblfsh/client-scala) ================================================ FILE: docs/usage/commands.md ================================================ # List of `sourced` Sub-Commands `sourced` binary offers you different kinds of sub-commands: - [to manage their containers](#manage-containers) - [to manage **source{d} CE** configuration](#manage-configuration) - [to open interfaces to access its data](#open-interfaces) - [show info about the command](#others) Here is the list of all these commands and its description; you can get more info about each one adding `--help` when you run it. ## Manage Containers ### sourced init _There is a dedicated section to document this command in the quickstart about [how to initialize **source{d} CE**](../quickstart/3-init-sourced.md)_ This command installs and initializes **source{d} CE** docker containers, networks, and volumes, downloading its docker images if needed. It can work over a local repository or a list of GitHub organizations. **source{d} CE** will download and install Docker images on demand. Therefore, the first time you run some of these commands, they might take a bit of time to start up. Subsequent runs will be faster. Once **source{d} CE** has been initialized, it will automatically open the web UI. If the UI is not opened automatically, you can use [`sourced web`](#sourced-web) command, or visit http://127.0.0.1:8088. Use login: `admin` and password: `admin`, to access the web interface. #### sourced init orgs ```shell $ sourced init orgs --token=_USER_TOKEN_ [--with-forks] org1,org2... ``` Installs and initializes **source{d} CE** for a list of GitHub organizations, downloading their repositories and metadata: Users, PullRequests, Issues... The `orgs` argument must be a comma-separated list of GitHub organizations. The `--token` must contain a valid GitHub user token for the given organizations. It should be granted with 'repo' and'read:org' scopes. If `--with-forks` is passed, it will also fetch repositories who are marked as forks. #### sourced init local ```shell $ sourced init local [/path/to/repos] ``` Installs and initializes **source{d} CE** using a local directory containing the git repositories to be processed by **source{d} CE**. If the local path to the `workdir` is not provided, the current working directory will be used. ### sourced start Starts all the components that were initialized with `init` and then stopped with `stop`. ### sourced stop Stops all running containers without removing them. They can be started again with `start`. ### sourced prune Stops containers and removes containers, networks, volumes, and configurations created by `init` for the current working directory. To delete resources for all the installed working directories, add the `--all` flag. Container images are not deleted unless you specify the `--images` flag. If you want to completely uninstall `sourced` you must also delete the `~/.sourced` directory. ### sourced logs Show logs from source{d} components. If `--follow` is used the logs are shown as they are logged until you exit with `Ctrl+C`. You can optionally pass component names to see only their logs. ```shell $ sourced logs $ sourced logs --follow $ sourced logs --follow gitbase bblfsh ``` ## Manage Configuration ### sourced status Shows the status of **source{d} CE** components, the installed working directories and the current deployment. #### sourced status all Show all the available status information, from the `components`, `config` and `workdirs`, sub-commands below. #### sourced status components Shows the status of the components containers of the running working directory #### sourced status config Shows the docker-compose environment variables configuration for the active working directory #### sourced status workdirs Lists all the previously initialized working directories ### sourced compose Manages Docker Compose files in the `~/.sourced` directory with the following subcommands: ### sourced compose download Download the `docker-compose.yml` file to define **source{d} CE** services. By default, the command downloads the file for this binary version, but you can also download other version or any other custom one using its URL. Examples: ```shell $ sourced compose download $ sourced compose download v0.0.1 $ sourced compose download master $ sourced compose download https://raw.githubusercontent.com/src-d/sourced-ce/master/docker-compose.yml ``` ### sourced compose list Lists the available `docker-compose.yml` files, and shows which one is active. You can activate any other with `compose set`. ### sourced compose set Sets the active `docker-compose.yml` file. Accepts either the name or index of the compose file as returned by 'compose list'. #### sourced restart Updates current installation according to the active docker compose file. It only recreates the component containers, keeping all your data, as charts, dashboards, repositories and GitHub metadata. ## Open Interfaces ### sourced sql Opens a MySQL client connected to gitbase. You can also pass a SQL query to be run by gitbase instead of opening the REPL, e.g. ```shell $ sourced sql "show databases" +----------+ | Database | +----------+ | gitbase | +----------+ ``` **source{d} CE** SQL supports a [UAST](#babelfish-uast) function that returns a Universal AST for the selected source text. UAST values are returned as binary blobs and are best visualized in the [SQL Lab, from the web interface](../quickstart/4-explore-sourced.md#sql-lab-querying-code) rather than the CLI where are seen as binary data. ### sourced web Opens the web interface in your browser. Use login: `admin` and password: `admin`, to access the web interface. ## Others ### sourced version Shows the version of the `sourced` command being used. ### sourced completion Prints a bash completion script for sourced; you can place its output in `/etc/bash_completion.d/sourced`, or add it to your `.bashrc` running: ```shell $ echo "source <(sourced completion)" >> ~/.bashrc ``` ================================================ FILE: docs/usage/examples.md ================================================ # SQL Examples to Analyze Your Data _If you want to know what the database schema looks like, you can refer to the [diagram about gitbase entities and relations](https://docs.sourced.tech/gitbase/using-gitbase/schema#database-diagram), or just use regular `SHOW` or `DESCRIBE` queries._ _In gitbase repository, you will find more [SQL examples of queries](https://docs.sourced.tech/gitbase/using-gitbase/examples)._ ## Index * [Queries For Repositories](#queries-for-repositories) * [Queries With Files](#queries-with-files) * [Queries With UASTs](#queries-with-uasts) * [Queries About Comitters](#queries-about-comitters) ## Queries For Repositories **Show me the repositories I am analyzing:** ```sql SELECT * FROM repositories; ``` **Last commit messages for HEAD for every repository** ```sql SELECT commit_message FROM refs NATURAL JOIN commits WHERE ref_name = 'HEAD'; ``` **Top 10 repositories by commit count from [HEAD](https://git-scm.com/book/en/v2/Git-Internals-Git-References#ref_the_ref):** ```sql SELECT repository_id,commit_count FROM ( SELECT repository_id, COUNT(*) AS commit_count FROM ref_commits WHERE ref_name = 'HEAD' GROUP BY repository_id ) AS q ORDER BY commit_count DESC LIMIT 10; ``` **10 top repos by file count at HEAD** ```sql SELECT repository_id, num_files FROM ( SELECT COUNT(*) num_files, repository_id FROM refs NATURAL JOIN commit_files WHERE ref_name = 'HEAD' GROUP BY repository_id ) AS t ORDER BY num_files DESC LIMIT 10; ``` ## Queries With Files **Query for all LICENSE & README files across history:** ```sql SELECT file_path, repository_id, blob_size FROM files WHERE file_path = 'LICENSE' OR file_path = 'README.md'; ``` **Query all files at [HEAD](https://git-scm.com/book/en/v2/Git-Internals-Git-References#ref_the_ref):** ```sql SELECT cf.file_path, f.blob_size FROM ref_commits rc NATURAL JOIN commit_files cf NATURAL JOIN files f WHERE rc.ref_name = 'HEAD' AND rc.history_index = 0; ``` ## Queries With UASTs _**Note**: UAST values are returned as binary blobs; they're best visualized in the web UI interface rather than the CLI where are seen as binary data._ **Retrieve the UAST for all files at [HEAD](https://git-scm.com/book/en/v2/Git-Internals-Git-References#ref_the_ref):** ```sql SELECT * FROM ( SELECT cf.file_path, UAST(f.blob_content, LANGUAGE(f.file_path, f.blob_content)) as uast FROM ref_commits r NATURAL JOIN commit_files cf NATURAL JOIN files f WHERE r.ref_name = 'HEAD' AND r.history_index = 0 ) t WHERE uast != ''; ``` ## Queries About Comitters **Top committers per repository** ```sql SELECT * FROM ( SELECT commit_author_email as author, repository_id as id, count(*) as num_commits FROM commits GROUP BY commit_author_email, repository_id ) AS t ORDER BY num_commits DESC; ``` ================================================ FILE: docs/usage/multiple-datasets.md ================================================ # Working With Multiple Data Sets You can deploy more than one **source{d} CE** instance with different sets of organizations, or repositories, to analyze. For example, you may have initially started **source{d} CE** with the repositories in the `src-d` organization, with the command: ```shell $ sourced init orgs --token src-d ``` After a while, you may want to analyze the data on another set of repositories. You can run `sourced init` again with a different organization: ```shell $ sourced init orgs --token bblfsh ``` This command will then stop all the running containers used for the previous dataset, create an isolated environment for the new data, and create a new, clean deployment. Please note that each path will have an isolated deployment. This means that for example any chart or dashboard created for the deployment for `src-d` will not be available to the new deployment for `bblfsh`. Each isolated environment is persistent (unless you run `sourced prune`). Which means that if you decide to re-deploy **source{d} CE** using the original organization: ```shell $ sourced init orgs --token src-d ``` You will get back to the previous state, and things like charts and dashboards will be restored. These isolated environments also allow you to deploy **source{d} CE** using a local set of Git repositories. For example, if we want a third deployment to analyze repositories already existing in the `~/repos` directory, we just need to run `init` again: ```shell $ sourced init local ~/repos ``` You can list all the installed instances, and know which one is active at any moment by running `sourced status workdirs`. If you are familiar with Docker Compose and you want more control over the underlying resources, you can explore the contents of your `~/.sourced` directory. There you will find a `docker-compose.yml` and `.env` files for each set of repositories used by `sourced init`. _You can read more about how the environments are isolated in the **source{d} CE** [architecture docs](../learn-more/architecture.md)_ ================================================ FILE: go.mod ================================================ module github.com/src-d/sourced-ce go 1.13 // See https://github.com/gotestyourself/gotest.tools/issues/156 // replace gotest.tools => gotest.tools v2.3.0 replace gotest.tools => gotest.tools v0.0.0-20181223230014-1083505acf35 require ( github.com/PuerkitoBio/goquery v1.5.0 github.com/blang/semver v3.5.1+incompatible github.com/golang/protobuf v1.3.1 // indirect github.com/google/btree v1.0.0 // indirect github.com/google/go-github/v25 v25.1.1 github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc github.com/jessevdk/go-flags v1.4.0 // indirect github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d // indirect github.com/lib/pq v1.1.1 github.com/mattn/go-colorable v0.1.1 // indirect github.com/mattn/go-isatty v0.0.7 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/onsi/ginkgo v1.8.0 // indirect github.com/onsi/gomega v1.5.0 // indirect github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 github.com/pkg/errors v0.8.1 github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516 github.com/sirupsen/logrus v1.4.1 // indirect github.com/src-d/envconfig v1.0.0 // indirect github.com/stretchr/testify v1.3.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c gopkg.in/src-d/go-cli.v0 v0.0.0-20190422143124-3a646154da79 gopkg.in/src-d/go-errors.v1 v1.0.0 gopkg.in/src-d/go-log.v1 v1.0.2 // indirect gopkg.in/yaml.v2 v2.2.2 // indirect gotest.tools v0.0.0-00010101000000-000000000000 ) ================================================ FILE: go.sum ================================================ github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-github/v25 v25.1.1 h1:6eW++i/CXcR5GKfYaaJT7oJJtHNU+/iiw55noEPNVao= github.com/google/go-github/v25 v25.1.1/go.mod h1:6z5pC69qHtrPJ0sXPsj4BLnd82b+r6sLB7qcBoRZqpw= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d h1:cVtBfNW5XTHiKQe7jDaDBSh/EVM4XLPutLAGboIXuM0= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4 h1:MfIUBZ1bz7TgvQLVa/yPJZOGeKEgs6eTKUjz3zB4B+U= github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4/go.mod h1:RMU2gJXhratVxBDTFeOdNhd540tG57lt9FIUV0YLvIQ= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516 h1:ofR1ZdrNSkiWcMsRrubK9tb2/SlZVWttAfqUjJi6QYc= github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/src-d/envconfig v1.0.0 h1:/AJi6DtjFhZKNx3OB2qMsq7y4yT5//AeSZIe7rk+PX8= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/src-d/go-cli.v0 v0.0.0-20190422143124-3a646154da79 h1:MBr8uUjT5gZe4udsLGNZ3nWy/+ck0LSB91mQjgpwLUI= gopkg.in/src-d/go-cli.v0 v0.0.0-20190422143124-3a646154da79/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-errors.v1 v1.0.0 h1:cooGdZnCjYbeS1zb1s6pVAAimTdKceRrpn7aKOnNIfc= gopkg.in/src-d/go-errors.v1 v1.0.0/go.mod h1:q1cBlomlw2FnDBDNGlnh6X0jPihy+QxZfMMNxPCbdYg= gopkg.in/src-d/go-log.v1 v1.0.2 h1:dED4100pntH4l3qOTgD1xebQR6pVU8tuPbUCmqiMsb0= gopkg.in/src-d/go-log.v1 v1.0.2/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v0.0.0-20181223230014-1083505acf35 h1:zpdCK+REwbk+rqjJmHhiCN6iBIigrZ39glqSF0P3KF0= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= ================================================ FILE: run-integration-tests.bat ================================================ :: we can't use makefile for windows because it depends on CI makefile which depends on shell :: compile sourced-ce go build -tags=forceposix -o build/sourced-ce_windows_amd64/sourced.exe ./cmd/sourced :: run tests set TMPDIR_INTEGRATION_TEST=C:\tmp :: see https://stackoverflow.com/questions/22948189/batch-getting-the-directory-is-not-empty-on-rmdir-command del /f /s /q %TMPDIR_INTEGRATION_TEST% 1>nul rmdir /s /q %TMPDIR_INTEGRATION_TEST% mkdir %TMPDIR_INTEGRATION_TEST% set TMP=%TMPDIR_INTEGRATION_TEST% go test -timeout 20m -parallel 1 -count 1 -tags="forceposix integration" github.com/src-d/sourced-ce/test/ -v ================================================ FILE: test/commander.go ================================================ // +build integration package test import ( "fmt" "os" "time" "gotest.tools/icmd" ) type Commander struct { bin string sourcedDir string } // RunCmd runs a command with the appropriate SOURCED_DIR and returns a Result func (s *Commander) RunCmd(args []string, cmdOperators ...icmd.CmdOp) *icmd.Result { c := icmd.Command(s.bin, args...) newEnv := append(os.Environ(), fmt.Sprintf("SOURCED_DIR=%s", s.sourcedDir)) op := append(cmdOperators, icmd.WithEnv(newEnv...)) return icmd.RunCmd(c, op...) } // RunCommand is a convenience wrapper for RunCmd that accepts variadic arguments. // It runs a command with the appropriate SOURCED_DIR and returns a Result func (s *Commander) RunCommand(args ...string) *icmd.Result { return s.RunCmd(args) } // RunCommandWithTimeout runs a command with the given timeout func (s *Commander) RunCommandWithTimeout(timeout time.Duration, args ...string) *icmd.Result { return s.RunCmd(args, icmd.WithTimeout(timeout)) } ================================================ FILE: test/common.go ================================================ // +build integration package test import ( "fmt" "io/ioutil" "log" "os" "path/filepath" "runtime" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) // TODO (carlosms) this could be build/bin, workaround for https://github.com/src-d/ci/issues/97 var srcdBin = fmt.Sprintf("../build/sourced-ce_%s_%s/sourced", runtime.GOOS, runtime.GOARCH) func init() { if os.Getenv("SOURCED_BIN") != "" { srcdBin = os.Getenv("SOURCED_BIN") } } type IntegrationSuite struct { suite.Suite *Commander TestDir string } func (s *IntegrationSuite) SetupTest() { testDir, err := ioutil.TempDir("", strings.Replace(s.T().Name(), "/", "_", -1)) if err != nil { log.Fatal(err) } if runtime.GOOS == "windows" { testDir, err = filepath.EvalSymlinks(testDir) if err != nil { log.Fatal(err) } } s.TestDir = testDir s.Commander = &Commander{bin: srcdBin, sourcedDir: filepath.Join(s.TestDir, "sourced")} // Instead of downloading the compose file, copy the local file err = os.MkdirAll(filepath.Join(s.sourcedDir, "compose-files", "local"), os.ModePerm) s.Require().NoError(err) p, _ := filepath.Abs(filepath.FromSlash("../docker-compose.yml")) bytes, err := ioutil.ReadFile(p) s.Require().NoError(err) err = ioutil.WriteFile(filepath.Join(s.sourcedDir, "compose-files", "local", "docker-compose.yml"), bytes, 0644) s.Require().NoError(err) r := s.RunCommand("compose", "set", "local") s.Require().NoError(r.Error, r.Combined()) } func (s *IntegrationSuite) TearDownTest() { // don't run prune on failed test to help debug. But stop the containers // to avoid port conflicts in the next test if s.T().Failed() { s.RunCommand("stop") s.T().Logf("Test failed. sourced data dir left in %s", s.TestDir) s.T().Logf("Probably there are also docker volumes left untouched") return } s.RunCommand("prune", "--all") os.RemoveAll(s.TestDir) } func (s *IntegrationSuite) testSQL() { testCases := []string{ "show tables", "show tables;", " show tables ; ", "/* comment */ show tables;", `/* multi line comment */ show tables;`, } showTablesOutput := `Table blobs commit_blobs commit_files commit_trees commits files ref_commits refs remotes repositories tree_entries ` for _, query := range testCases { s.T().Run(query, func(t *testing.T) { assert := assert.New(t) r := s.RunCommand("sql", query) assert.NoError(r.Error, r.Combined()) assert.Contains(r.Stdout(), showTablesOutput) }) } } ================================================ FILE: test/compose_test.go ================================================ // +build integration package test import ( "testing" "github.com/stretchr/testify/suite" ) type ComposeTestSuite struct { IntegrationSuite } func TestComposeTestSuite(t *testing.T) { itt := ComposeTestSuite{} suite.Run(t, &itt) } func (s *ComposeTestSuite) TestListComposeFiles() { r := s.RunCommand("compose", "list") s.Contains(r.Stdout(), "[0]* local") } func (s *ComposeTestSuite) TestSetComposeFile() { r := s.RunCommand("compose", "set", "0") s.Contains(r.Stdout(), "Active docker compose file was changed successfully") r = s.RunCommand("compose", "list") s.Contains(r.Stdout(), "[0]* local") } func (s *ComposeTestSuite) TestSetComposeFilIndexOutOfRange() { r := s.RunCommand("compose", "set", "5") s.Contains(r.Stderr(), "Index is out of range, please check the output of 'sourced compose list'") } func (s *ComposeTestSuite) TestSetComposeNotFound() { r := s.RunCommand("compose", "set", "NotFound") s.Error(r.Error) } func (s *ComposeTestSuite) TestSetComposeFilesWithStringIndex() { r := s.RunCommand("compose", "set", "local") s.Contains(r.Stdout(), "Active docker compose file was changed successfully") r = s.RunCommand("compose", "list") s.Contains(r.Stdout(), "[0]* local") } ================================================ FILE: test/init_local_test.go ================================================ // +build integration package test import ( "fmt" "os" "os/exec" "path/filepath" "testing" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) type InitLocalTestSuite struct { IntegrationSuite } func TestInitLocalTestSuite(t *testing.T) { itt := InitLocalTestSuite{} suite.Run(t, &itt) } func (s *InitLocalTestSuite) TestWithInvalidWorkdir() { require := s.Require() invalidWorkDir := filepath.Join(s.TestDir, "invalid-workdir") _, err := os.Create(invalidWorkDir) if err != nil { s.T().Fatal(err) } r := s.RunCommand("init", "local", invalidWorkDir) require.Error(r.Error) require.Equal( fmt.Sprintf("path '%s' is not a valid directory\n", invalidWorkDir), r.Stderr(), ) } func (s *InitLocalTestSuite) TestChangeWorkdir() { req := s.Require() r := s.RunCommand("status", "workdirs") req.Error(r.Error) // Create 2 workdirs, each with a repo workdirA := filepath.Join(s.TestDir, "workdir_a") workdirB := filepath.Join(s.TestDir, "workdir_b") pathA := filepath.Join(workdirA, "repo_a") pathB := filepath.Join(workdirB, "repo_b") s.initGitRepo(pathA) s.initGitRepo(pathB) // init with workdir A r = s.RunCommand("init", "local", workdirA) req.NoError(r.Error, r.Combined()) r = s.RunCommand("status", "workdirs") req.NoError(r.Error, r.Combined()) req.Equal(fmt.Sprintf("* %v\n", workdirA), r.Stdout()) r = s.RunCommand("sql", "select * from repositories") req.NoError(r.Error, r.Combined()) req.Contains(r.Stdout(), `repository_id repo_a `) // init with workdir B r = s.RunCommand("init", "local", workdirB) req.NoError(r.Error, r.Combined()) r = s.RunCommand("status", "workdirs") req.NoError(r.Error, r.Combined()) req.Equal(fmt.Sprintf(" %v\n* %v\n", workdirA, workdirB), r.Stdout()) r = s.RunCommand("sql", "select * from repositories") req.NoError(r.Error, r.Combined()) req.Contains(r.Stdout(), `repository_id repo_b `) // Test SQL queries. This should be a different test, but since starting // the environment takes a long time, it is bundled together here to speed up // the tests s.testSQL() client, err := newSupersetClient() req.NoError(err) // Test the list of dashboards created in superset s.T().Run("dashboard-list", func(t *testing.T) { req := require.New(t) links, err := client.dashboards() req.NoError(err) s.Equal([]string{ `Overview`, `Welcome`, }, links) }) // Test gitbase queries through superset s.T().Run("superset-gitbase", func(t *testing.T) { req := require.New(t) rows, err := client.gitbase("select * from repositories") req.NoError(err) s.Equal([]map[string]interface{}{ {"repository_id": "repo_b"}, }, rows) }) // Test bblfsh queries through superset s.T().Run("superset-bblfsh", func(t *testing.T) { req := require.New(t) lang, err := client.bblfsh("hello.js", `console.log("hello");`) req.NoError(err) req.Equal("javascript", lang) }) // Test gitbase can connect to bblfsh with a SQL query that uses UAST s.T().Run("gitbase-bblfsh", func(t *testing.T) { req := require.New(t) rows, err := client.gitbase( `SELECT UAST('console.log("hello");', 'javascript') AS uast`) req.NoError(err) req.Len(rows, 1) req.NotEmpty(rows[0]["uast"]) }) } func (s *InitLocalTestSuite) initGitRepo(path string) { s.T().Helper() err := os.MkdirAll(path, os.ModePerm) s.Require().NoError(err) cmd := exec.Command("git", "init", path) err = cmd.Run() s.Require().NoError(err) } ================================================ FILE: test/init_orgs_test.go ================================================ // +build integration package test import ( "database/sql" "os" "testing" "time" _ "github.com/lib/pq" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) type InitOrgsTestSuite struct { IntegrationSuite } func TestInitOrgsTestSuite(t *testing.T) { itt := InitOrgsTestSuite{} if os.Getenv("SOURCED_GITHUB_TOKEN") == "" { t.Skip("SOURCED_GITHUB_TOKEN is not set") return } suite.Run(t, &itt) } func checkGhsync(require *require.Assertions, repos int) { connStr := "user=metadata password=metadata dbname=metadata port=5433 sslmode=disable" db, err := sql.Open("postgres", connStr) require.NoError(err) defer db.Close() var id int var org, entity string var done, failed, total int // try for 2 minutes for i := 0; i < 24; i++ { time.Sleep(5 * time.Second) row := db.QueryRow("SELECT * FROM status") err = row.Scan(&id, &org, &entity, &done, &failed, &total) if err == sql.ErrNoRows { continue } require.NoError(err) if done == repos { break } } require.Equal(repos, done, "id = %v, org = %v, entity = %v, done = %v, failed = %v, total = %v", id, org, entity, done, failed, total) } func checkGitcollector(require *require.Assertions, repos int) { connStr := "user=superset password=superset dbname=superset port=5432 sslmode=disable" db, err := sql.Open("postgres", connStr) require.NoError(err) defer db.Close() var org string var discovered, downloaded, updated, failed int // try for 2 minutes for i := 0; i < 24; i++ { time.Sleep(5 * time.Second) row := db.QueryRow("SELECT * FROM gitcollector_metrics") err = row.Scan(&org, &discovered, &downloaded, &updated, &failed) if err == sql.ErrNoRows { continue } require.NoError(err) if downloaded == repos { break } } require.Equal(repos, downloaded, "org = %v, discovered = %v, downloaded = %v, updated = %v, failed = %v", org, discovered, downloaded, updated, failed) } func (s *InitOrgsTestSuite) TestOneOrg() { req := s.Require() // TODO will need to change with https://github.com/src-d/sourced-ce/issues/144 r := s.RunCommand("status", "workdirs") req.Error(r.Error) r = s.RunCommand("init", "orgs", "golang-migrate") req.NoError(r.Error, r.Combined()) r = s.RunCommand("status", "workdirs") req.NoError(r.Error, r.Combined()) req.Equal("* golang-migrate\n", r.Stdout()) checkGhsync(req, 1) checkGitcollector(req, 1) // Check gitbase can also see the repositories r = s.RunCommand("sql", "select * from repositories where repository_id='github.com/golang-migrate/migrate'") req.NoError(r.Error, r.Combined()) req.Contains(r.Stdout(), `repository_id github.com/golang-migrate/migrate `) // Test SQL queries. This should be a different test, but since starting // the environment takes a long time, it is bundled together here to speed up // the tests s.testSQL() client, err := newSupersetClient() req.NoError(err) // Test the list of dashboards created in superset s.T().Run("dashboard-list", func(t *testing.T) { req := require.New(t) links, err := client.dashboards() req.NoError(err) s.Equal([]string{ `Overview`, `Welcome`, `Collaboration`, }, links) }) // Test gitbase queries through superset s.T().Run("superset-gitbase", func(t *testing.T) { req := require.New(t) rows, err := client.gitbase("select * from repositories") req.NoError(err) s.Equal([]map[string]interface{}{ {"repository_id": "github.com/golang-migrate/migrate"}, }, rows) }) // Test metadata queries through superset s.T().Run("superset-metadata", func(t *testing.T) { req := require.New(t) rows, err := client.metadata("select * from organizations") req.NoError(err) req.Len(rows, 1) s.Equal("golang-migrate", rows[0]["login"]) }) // Test bblfsh queries through superset s.T().Run("superset-bblfsh", func(t *testing.T) { req := require.New(t) lang, err := client.bblfsh("hello.js", `console.log("hello");`) req.NoError(err) req.Equal("javascript", lang) }) // Test gitbase can connect to bblfsh with a SQL query that uses UAST s.T().Run("gitbase-bblfsh", func(t *testing.T) { req := require.New(t) rows, err := client.gitbase( `SELECT UAST('console.log("hello");', 'javascript') AS uast`) req.NoError(err) req.Len(rows, 1) req.NotEmpty(rows[0]["uast"]) }) } ================================================ FILE: test/superset.go ================================================ // +build integration package test import ( "bytes" "encoding/json" "fmt" "io/ioutil" "log" "net/http" "net/http/cookiejar" "net/url" "github.com/PuerkitoBio/goquery" "golang.org/x/net/publicsuffix" ) type supersetClient struct { *http.Client csrf string } func newSupersetClient() (*supersetClient, error) { // Superset client needs a session cookie jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) if err != nil { return nil, err } client := &http.Client{ Jar: jar, } // To POST in /login we need to first GET /login, read the hidden CSRF value // from the HTML, and send it back in the POST res, err := client.Get("http://127.0.0.1:8088/login/") if err != nil { return nil, err } doc, err := goquery.NewDocumentFromReader(res.Body) if err != nil { log.Fatal(err) } csrf, ok := doc.Find("input#csrf_token").First().Attr("value") if !ok { return nil, fmt.Errorf("CSRF token was not found in the /login page") } res, err = client.PostForm("http://127.0.0.1:8088/login/", url.Values{ "csrf_token": {csrf}, "username": {"admin"}, "password": {"admin"}, }) if err != nil { return nil, err } // After a successful POST the client is authenticated and can be used to call // the API return &supersetClient{client, csrf}, nil } func (c *supersetClient) dashboards() ([]string, error) { res, err := c.Get("http://127.0.0.1:8088/dashboard/api/read") if err != nil { return nil, err } var decoded struct { Result []struct { Link string `json:"dashboard_link"` } } err = json.NewDecoder(res.Body).Decode(&decoded) if err != nil { return nil, err } links := []string{} for _, result := range decoded.Result { links = append(links, result.Link) } return links, nil } func (c *supersetClient) sql(query, dbId, schema string) ([]map[string]interface{}, error) { res, err := c.PostForm("http://127.0.0.1:8088/superset/sql_json/", url.Values{ //"client_id": {""}, "database_id": {dbId}, "json": {"true"}, "runAsync": {"false"}, "schema": {schema}, "sql": {query}, //"sql_editor_id": {"jzl0KCm5Z"}, //"tab": {"Untitled Query 2"}, //"tmp_table_name": {""}, //"select_as_cta": {"false"}, //"templateParams": {"{}"}, //"queryLimit": {"1000"}, "csrf_token": {c.csrf}, }) if err != nil { return nil, err } var decoded struct { Status string Error string Data []map[string]interface{} } err = json.NewDecoder(res.Body).Decode(&decoded) if err != nil { return nil, err } if decoded.Status != "success" { return nil, fmt.Errorf("/sql_json endpoint returned an error: " + decoded.Error) } return decoded.Data, nil } func (c *supersetClient) gitbase(query string) ([]map[string]interface{}, error) { return c.sql(query, "1", "gitbase") } func (c *supersetClient) metadata(query string) ([]map[string]interface{}, error) { return c.sql(query, "2", "public") } func (c *supersetClient) bblfsh(filename, content string) (string, error) { var jsonStr = []byte(fmt.Sprintf( `{"mode":"semantic", "filename":%q, "content":%q, "query":""}`, filename, content)) req, err := http.NewRequest("POST", "http://127.0.0.1:8088/bblfsh/api/parse", bytes.NewBuffer(jsonStr)) if err != nil { return "", err } req.Header.Set("Content-Type", "application/json") res, err := c.Do(req) if err != nil { return "", err } var decoded struct { Status int Language string // Uast interface{} Errors []struct{ Message string } } err = json.NewDecoder(res.Body).Decode(&decoded) if err != nil { body := "" bytes, readErr := ioutil.ReadAll(res.Body) if readErr != nil { body = string(bytes) } return "", fmt.Errorf("could not decode the response body: %s, err: %s", body, err) } if decoded.Status != 0 { return "", fmt.Errorf("/bblfsh/api/parse endpoint returned an error: %v", decoded.Errors) } return decoded.Language, nil }