Repository: hyperform-dev/hyperform Branch: master Commit: 24f4e1b742ca Files: 48 Total size: 144.0 KB Directory structure: gitextract_hn1b61f9/ ├── .eslintrc.json ├── .github/ │ └── workflows/ │ └── node.js.yml ├── .gitignore ├── LICENSE ├── README.md ├── authorizer-gen/ │ ├── index.js │ ├── index.oldtest.js │ ├── utils.js │ └── utils.oldtest.js ├── bundler/ │ ├── amazon/ │ │ └── index.js │ ├── google/ │ │ └── index.js │ ├── utils.js │ └── utils.test.js ├── cli.js ├── copier/ │ └── index.js ├── deployer/ │ ├── amazon/ │ │ ├── index.js │ │ ├── index.test.js │ │ └── utils.js │ └── google/ │ ├── index.js │ └── index.test.js ├── discoverer/ │ └── index.js ├── index.js ├── index.test.js ├── initer/ │ ├── index.js │ └── index.test.js ├── kindler/ │ ├── index.js │ └── index.tes.js ├── meta/ │ └── index.js ├── package.json ├── parser/ │ └── index.js ├── printers/ │ └── index.js ├── publisher/ │ └── amazon/ │ ├── index.js │ ├── utils.js │ └── utils.test.js ├── response-collector/ │ ├── .gitignore │ └── index.js ├── schemas/ │ ├── index.js │ └── index.test.js ├── surveyor/ │ └── index.js ├── template/ │ ├── .eslintrc.json │ └── index.js ├── transpiler/ │ └── index.js ├── uploader/ │ ├── amazon/ │ │ ├── index.js │ │ └── index.test.js │ └── google/ │ └── index.js └── zipper/ ├── google/ │ └── index.js ├── index.js └── index.test.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .eslintrc.json ================================================ { "env": { "commonjs": true, "es2021": true, "node": true, "jest": true }, "extends": [ "airbnb-base" ], "parserOptions": { "ecmaVersion": 12 }, "rules": { "no-console": "off", "no-trailing-spaces": "off", "semi": "off", "object-shorthand": "off", "no-unused-vars": "warn", "no-useless-catch": "warn", "no-underscore-dangle": "off", "no-await-in-loop": "warn", "no-else-return": "off", "camelcase": "off", "no-restricted-syntax": "warn", "prefer-destructuring": "warn", "no-continue": "warn" } } ================================================ FILE: .github/workflows/node.js.yml ================================================ # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions name: Node.js CI on: push: branches: [ master ] # pull_request: # branches: [ master ] jobs: build: strategy: matrix: platform: [ubuntu-latest] runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v2 - name: Use Node.js 10.x uses: actions/setup-node@v1 with: node-version: 10.x - name: Install run: npm ci # beware of dashes (-) before run - name: Test run: npm run test env: # env is only kept in this step AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} ================================================ FILE: .gitignore ================================================ node_modules/ coverage/ dde/lamb/ dde/lambs/ node_modules/ ultra/zoo/ dist/ hyperform.json .env ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2021 Hyperform Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ ![Hyperform Banner](https://github.com/qngapparat/hyperform/blob/master/hyperform-banner.png) > ⚡ Lightweight serverless framework for NodeJS * **Unopinionated** (Any JS code works) * **Lightweight** (no wrapping) * **1-click deploy** (1 command) * **Multi-Cloud** (for AWS & Google Cloud) * **Maintains** (provider's conventions) ## Install ```sh $ npm install -g hyperform-cli ``` ## Usage * Everything works like a normal NodeJS app. You can use NPM packages, external files, assets, since the entire folder containing `hyperform.json` is included with each function. ### AWS Lambda ```js // somefile.js // AWS Lambda uses 'event', 'context', and 'callback' convention // Learn more: https://docs.aws.amazon.com/lambda/latest/dg/nodejs-handler.html exports.foo = (event, context, callback) => { context.succeed({ message: "I'm Foo on AWS Lambda!" }) } exports.bar = (event, context, callback) => { context.succeed({ message: "I'm Bar on AWS Lambda!" }) } // ... ``` Create a `hyperform.json` in the current folder, with your AWS credentials: ```json { "amazon": { "aws_access_key_id": "...", "aws_secret_access_key": "...", "aws_region": "..." } } ``` In the terminal, type: ``` $ hyperform deploy somefile.js --amazon --url > 🟢 foo https://w3g434h.execute-api.us-east-2.amazonaws.com/foo > 🟢 bar https://w3g434h.execute-api.us-east-2.amazonaws.com/bar ``` ... and your functions are deployed & invocable via `GET` and `POST`. ### Google Cloud Functions ```js // somefile.js // Google Cloud uses Express's 'Request' and 'Response' convention // Learn more: https://expressjs.com/en/api.html#req // https://expressjs.com/en/api.html#res exports.foo = (req, res) => { let message = req.query.message || req.body.message || "I'm a Google Cloud Function, Foo"; res.status(200).send(message); }; exports.bar = (req, res) => { let message = req.query.message || req.body.message || "I'm a Google Cloud Function, Bar"; res.status(200).send(message); }; ``` Create a `hyperform.json` in the current folder with your Google Cloud credentials: ```json { "google": { "gc_project": "...", "gc_region": "...", } } ``` In the terminal, type: ``` $ hyperform deploy somefile.js --google --url > 🟢 foo https://us-central1-someproject-153dg2.cloudfunctions.net/foo > 🟢 bar https://us-central1-someproject-153dg2.cloudfunctions.net/bar ``` ... and your functions are deployed & invocable via `GET` and `POST`. ## Hints & Caveats * New functions are deployed with 256MB RAM, 60s timeouts * The flag `--url` creates **unprotected** URLs to the functions. Anyone with these URLs can invoke your functions * The entire folder containing `hyperform.json` will be deployed with each function, so you can use NPM packages, external files (...) just like normal. ### FAQ **Where are functions deployed to?** * On AWS: To AWS Lambda * On Google Cloud: To Google Cloud Functions **Where does deployment happen?** It's a client-side tool, so on your computer. It uses the credentials it finds in `hyperform.json` **Can I use NPM packages, external files, (...) ?** Yes. The entire folder where `hyperform.json` is is uploaded, excluding `.git`, `.gitignore`, `hyperform.json`, and for Google Cloud `node_modules` (Google Cloud installs NPM dependencies freshly from `package.json`). So everything works like a normal NodeJS app. **How does `--url` create URLs?** On AWS, it creates an API Gateway API (called `hf`), and a `GET` and `POST` route to your function. On Google Cloud, it removes IAM checking from the function by adding `allUsers` to the group "Cloud Functions Invoker" of that function. Note that in both cases, **anyone with the URL can invoke your function. Make sure to add Authentication logic inside your function**, if needed. ## Opening Issues Feel free to open issues if you find bugs. ## Contributing Always welcome ❤️ Please see CONTRIBUTING.md ## License Apache 2.0 ================================================ FILE: authorizer-gen/index.js ================================================ // const AWS = require('aws-sdk') // const { deployAmazon } = require('../deployer/amazon/index') // const { allowApiGatewayToInvokeLambda } = require('../publisher/amazon/utils') // const { zip } = require('../zipper/index') // const { ensureBearerTokenSecure } = require('./utils') // const { logdev } = require('../printers/index') // AWS.config.update({ // accessKeyId: process.env.AWS_ACCESS_KEY_ID, // secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, // region: process.env.AWS_REGION, // }) // /** // * @description Creates or updates Authorizer lambda with name "authorizerName" // * that if used as Authorizer in API Gateway, will // * greenlight requests with given expectedBearer token // * @param {string} authorizerName For example 'myfn-authorizer' // * @param {string} expectedBearer The 'Authorization': 'Bearer ...' token // * the Authorizer will greenlight // * @param {{region: string}} options // * @returns {Promise} ARN of the deployed authorizer lambda // */ // async function deployAuthorizerLambda(authorizerName, expectedBearer, options) { // if (options == null || options.region == null) { // throw new Error('optionsregion is required') // TODO HF programmer mistake // } // // will mess up weird user-given Tokens but that's on the user // // will lead to false negatives (still better than false positives or injections) // // This should not be needed, as expectedBearer is generated by our code, but to be sure // const sanitizedExpectedBearer = encodeURI(expectedBearer) // ensureBearerTokenSecure(sanitizedExpectedBearer) // const authorizerCode = ` // exports.handler = async(event) => { // const expected = \`Bearer ${sanitizedExpectedBearer}\` // const isAuthorized = (event.headers.authorization === expected) // return { // isAuthorized // } // }; // ` // // TODO do this private somehow, in RAM, so that no program can tamper with authorizer zip // const zipPath = await zip(authorizerCode) // const deployOptions = { // name: authorizerName, // timeout: 1, // 1 second is ample time // handler: 'index.handler', // region: options.region, // } // // create or update Authorizer Lambda // const authorizerArn = await deployAmazon(zipPath, deployOptions) // await allowApiGatewayToInvokeLambda(authorizerName, options.region) // return authorizerArn // } // /** // * @description Gets the RouteId of a route belonging to an API on API Gateway // * @param {string} apiId Id of the API in API Gateway // * @param {string} routeKey For example '$default' // * @param {string} region Region of the API in API Gateway // * @returns {Promise} RouteId of the route // * @throws If query items did not include a Route named "routeKey" // */ // async function getRouteId(apiId, routeKey, region) { // const apigatewayv2 = new AWS.ApiGatewayV2({ // apiVersion: '2018-11-29', // region: region, // }) // // TODO Amazon might return a paginated response here (?) // // In that case with many routes, the route we look for may not be on first page // const params = { // ApiId: apiId, // MaxResults: '9999', // string according to docs and it works... uuh? // } // const res = await apigatewayv2.getRoutes(params).promise() // const matchingRoutes = res.Items.filter((item) => item.RouteKey === routeKey) // if (matchingRoutes.length === 0) { // throw new Error(`Could not get RouteId of apiId, routeKey ${apiId}, ${routeKey}`) // } // // just take first one // // Hyperform convention is there's only one with any given name // const routeId = matchingRoutes[0].RouteId // return routeId // } // /** // * @description Sets the $default path of "apiId" to be guarded by "authorizerArn" lambda. // * @param {string} apiId Id of API in API Gateway to be guarded // * @param {string} authorizerArn ARN of Lambda that should act as the authorizer // * @returns {void} // * @throws Throws if authorizerArn is not formed like a Lambda ARN. // * Fails silently if authorizerArn Lambda does not exist. // */ // async function setDefaultRouteAuthorizer(apiId, authorizerArn, apiRegion) { // // TODO what happens when api (set to REGIONAL) and authorizer lambda are in different regions // // region is the fourth field // const authorizerRegion = authorizerArn.split(':')[3] // // name is the last field // const authorizerName = authorizerArn.split(':').slice(-1)[0] // const authorizerType = 'REQUEST' // const identitySource = '$request.header.Authorization' // const authorizerUri = `arn:aws:apigateway:${authorizerRegion}:lambda:path/2015-03-31/functions/${authorizerArn}/invocations` // // Try to create authorizer for that API // // succeeds => Authorizer with that name did not exist yet. Use that authorizerId going forward // // Fails => Authorizer already existed with that name. // // Get that one's authorizerId (Follow Hyperform conv: same name - assume identical) // const apigatewayv2 = new AWS.ApiGatewayV2({ // apiVersion: '2018-11-29', // region: apiRegion, // }) // const createAuthorizerParams = { // ApiId: apiId, // Name: authorizerName, // AuthorizerType: authorizerType, // IdentitySource: [identitySource], // AuthorizerUri: authorizerUri, // AuthorizerPayloadFormatVersion: '2.0', // EnableSimpleResponses: true, // } // let authorizerId // try { // const createRes = await apigatewayv2.createAuthorizer(createAuthorizerParams).promise() // // authorizer does not exist // authorizerId = createRes.AuthorizerId // } catch (e) { // if (e.code === 'BadRequestException') { // // authorizer already exists // // TODO update authorizer to make sure it points // // ...to authorizerArn lambda (to behave exactly as stated in @description) // // TODO pull-up this and/or add update authorizer // // obtain its id // const getAuthorizersParams = { // ApiId: apiId, // MaxResults: '9999', // } // const getRes = await apigatewayv2.getAuthorizers(getAuthorizersParams).promise() // const matchingRoutes = getRes.Items.filter((item) => item.Name === authorizerName) // if (matchingRoutes.length === 0) { // throw new Error(`Could not get AuthorizerId of apiId ${apiId}`) // } // // just take first one // // Hyperform convention is there's only one with any given name // authorizerId = matchingRoutes[0].AuthorizerId // } else { // // some other error // throw e // } // } // // attach authorizer to $default // const routeKey = '$default' // const routeId = await getRouteId(apiId, routeKey, apiRegion) // const updateRouteParams = { // ApiId: apiId, // RouteId: routeId, // AuthorizerId: authorizerId, // AuthorizationType: 'CUSTOM', // } // await apigatewayv2.updateRoute(updateRouteParams).promise() // logdev('set authorizer') // // done // } // /** // * @description Detaches the current authorizer, if any, from the $default route of API // * with ID "apiId". The route is then in any case unauthorized and the underlying Lambda becomes // * invokable by anyone with the URL. // * This does not delete the authorizer or the authorizer Lambda. // * @param {string} apiId // * @param {string} apiRegion // */ // async function detachDefaultRouteAuthorizer(apiId, apiRegion) { // const apigatewayv2 = new AWS.ApiGatewayV2({ // apiVersion: '2018-11-29', // region: apiRegion, // }) // const routeKey = '$default' // const routeId = await getRouteId(apiId, routeKey, apiRegion) // const updateRouteParams = { // ApiId: apiId, // RouteId: routeId, // AuthorizationType: 'NONE', // } // await apigatewayv2.updateRoute(updateRouteParams).promise() // logdev('detached authorizer ') // } // // TODO set authorizer cache ?? // module.exports = { // deployAuthorizerLambda, // setDefaultRouteAuthorizer, // detachDefaultRouteAuthorizer, // _only_for_testing_getRouteId: getRouteId, // } ================================================ FILE: authorizer-gen/index.oldtest.js ================================================ // /* eslint-disable global-require */ // const LAMBDANAME = 'jest-reserved-authorizer' // const LAMBDAREGION = 'us-east-2' // const APIREGION = 'us-east-2' // const LAMBDAARN = 'arn:aws:lambda:us-east-2:735406098573:function:jest-reserved-authorizer' // const BEARERTOKEN = 'somelengthyrandombearertoken1234567890' // // NOTE Currently convention is one API per endpoint // // Don't extend tests until we are sure of this convention / committed to // describe('authorizer-gen', () => { // describe('index', () => { // describe('getRouteId', () => { // test('returns non-empty string for existing route', async () => { // const getRouteId = require('./index')._only_for_testing_getRouteId // const apiId = 'vca3i8138h' // first-http-api in my API Gateway // const routeKey = '$default' // let err // let res // try { // res = await getRouteId(apiId, routeKey, LAMBDAREGION) // } catch (e) { // console.log(e) // err = e // } // // It did not throw // expect(err).not.toBeDefined() // // Returned routeid: a non-empty string // expect(res).toBeDefined() // expect(typeof res).toEqual('string') // expect(res.length > 0).toEqual(true) // }) // test('throws for non-existing API', async () => { // const getRouteId = require('./index')._only_for_testing_getRouteId // const apiId = 'invalid-api-id' // const routeKey = '$default' // let err // let res // try { // res = await getRouteId(apiId, routeKey, LAMBDAREGION) // } catch (e) { // err = e // } // // It threw // expect(err).toBeDefined() // expect(err.toString()).toMatch(/Invalid API identifier/) // }) // test('throws for non-existing route', async () => { // const getRouteId = require('./index')._only_for_testing_getRouteId // const apiId = 'vca3i8138h' // first-http-api in my API Gateway // const routeKey = 'invalid-route-name' // let err // let res // try { // res = await getRouteId(apiId, routeKey, LAMBDAREGION) // } catch (e) { // err = e // } // // It threw // expect(err).toBeDefined() // }) // }) // describe('deployAuthorizerLambda', () => { // test('throws on expectedBearer shorter than 10 digits (not secure)', async () => { // const { deployAuthorizerLambda } = require('./index') // const expectedBearer = '123456789 ' // const options = { // region: LAMBDAREGION, // } // let err // try { // await deployAuthorizerLambda(LAMBDANAME, expectedBearer, options) // } catch (e) { // err = e // } // expect(err).toBeDefined() // }) // test('throws on expectedBearer is null', async () => { // const { deployAuthorizerLambda } = require('./index') // const expectedBearer = null // const options = { // region: LAMBDAREGION, // } // let err // try { // await deployAuthorizerLambda(LAMBDANAME, expectedBearer, options) // } catch (e) { // err = e // } // expect(err).toBeDefined() // }) // test('throws on expectedBearer is empty string', async () => { // const { deployAuthorizerLambda } = require('./index') // const expectedBearer = ' ' // const options = { // region: LAMBDAREGION, // } // let err // try { // await deployAuthorizerLambda(LAMBDANAME, expectedBearer, options) // } catch (e) { // err = e // } // expect(err).toBeDefined() // }) // // allow 15 seconds // test('completes if authorizer lambda does not exist yet, returns ARN', async () => { // const { deployAuthorizerLambda } = require('./index') // const { deleteAmazon } = require('../deployer/amazon/index') // const expectedBearer = BEARERTOKEN // const options = { // region: LAMBDAREGION, // } // /// ////////////////////////////////////////////// // // Setup: delete authorizer if it exists already // try { // await deleteAmazon(LAMBDANAME, LAMBDAREGION) // // deleted lambda // } catch (e) { // if (e.code === 'ResourceNotFoundException') { // // does not exist in the first place, nice // } else { // throw e // } // } // /// /////////////////////////////////////// // let err // let res // try { // res = await deployAuthorizerLambda(LAMBDANAME, expectedBearer, options) // } catch (e) { // console.log(e) // err = e // } // expect(err).not.toBeDefined() // expect(typeof res).toBe('string') // }, 30 * 1000) // test('completes if authorizer lambda exists already, returns ARN', async () => { // const { deployAuthorizerLambda } = require('./index') // const expectedBearer = BEARERTOKEN // const options = { // region: LAMBDAREGION, // } // /// /////////////////////////////////////// // // Setup: ensure authorizer lambda exists // await deployAuthorizerLambda(LAMBDANAME, expectedBearer, options) // /// /////////////////////////////////////// // let err // let res // try { // res = await deployAuthorizerLambda(LAMBDANAME, expectedBearer, options) // } catch (e) { // console.log(e) // err = e // } // expect(err).not.toBeDefined() // expect(typeof res).toBe('string') // }, 30 * 1000) // }) // // allow 15 seconds // describe('setDefaultRouteAuthorizer', () => { // test('completes when authorizer exists already', async () => { // const { setDefaultRouteAuthorizer } = require('./index') // const apiId = 'vca3i8138h' // first-http-api in my API Gateway // let err // try { // await setDefaultRouteAuthorizer(apiId, LAMBDAARN, APIREGION) // } catch (e) { // console.log(e) // err = e // } // expect(err).not.toBeDefined() // }, 15 * 1000) // test('completes when API has no authorizer yet', async () => { // const { setDefaultRouteAuthorizer, detachDefaultRouteAuthorizer } = require('./index') // const apiId = 'vca3i8138h' // first-http-api in my API Gateway // /// //////////////////////////////////////////////// // // Setup: detach current authorizer (if any) // await detachDefaultRouteAuthorizer(apiId, APIREGION) // /// ///////////////////////////////////////////////// // let err // try { // await setDefaultRouteAuthorizer(apiId, LAMBDAARN, APIREGION) // } catch (e) { // console.log(e) // err = e // } // expect(err).not.toBeDefined() // // // }, 15 * 1000) // // TODO test for when authorizer does not exist yet // /// //////////////////// // }) // test('throws on authorizerArn not being valid ARN format', async () => { // const { setDefaultRouteAuthorizer } = require('./index') // const invalidArn = 'INVALID_AMAZON_ARN_FORMAT' // const apiId = 'vca3i8138h' // first-http-api in my API Gateway // let err // try { // await setDefaultRouteAuthorizer(apiId, invalidArn, APIREGION) // } catch (e) { // err = e // } // expect(err).toBeDefined() // }, 15 * 1000) // }) // }) ================================================ FILE: authorizer-gen/utils.js ================================================ // const uuidv4 = require('uuid').v4 // /** // * @description Generates a '''random''' bearer token TODO // * @returns {string} '''Random''' bearer token // */ // function generateRandomBearerToken() { // const token = uuidv4() // .replace(/-/g, '') // return token // } // /** // * @returns {void} // * @throws if "bearerToken" does not fix requirements // */ // function ensureBearerTokenSecure(bearerToken) { // // messages mostly for us // if (typeof bearerToken !== 'string') throw new Error(`Bearer token must be a string but is ${typeof bearerToken}`) // if (bearerToken.trim().length < 10) throw new Error('Bearer token, trimmed, must be equal longer than 10') // if (/^[a-zA-Z0-9]+$/.test(bearerToken) === false) throw new Error('Bearer token must fit regex /^[a-zA-Z0-9]+$/ (alphanumeric)') // } // module.exports = { // generateRandomBearerToken, // ensureBearerTokenSecure, // } ================================================ FILE: authorizer-gen/utils.oldtest.js ================================================ // /* eslint-disable global-require */ // describe('authorizer-gen', () => { // describe('utils', () => { // describe('generateRandomBearerToken', () => { // test('is between 0 and 50 characters', () => { // const { generateRandomBearerToken } = require('./utils') // const output = generateRandomBearerToken() // expect(output.length).toBeDefined() // expect(output.length <= 50).toEqual(true) // expect(output.length > 0).toEqual(true) // }) // test('is alphanumeric', () => { // const { generateRandomBearerToken } = require('./utils') // const output = generateRandomBearerToken() // const regex = /^[a-zA-Z0-9]+$/ // expect(regex.test(output)).toEqual(true) // }) // }) // }) // }) ================================================ FILE: bundler/amazon/index.js ================================================ const { _bundle } = require('../utils') /** * @description Bundles a given .js files for Amazon with its dependencies using webpack. * @param {string} inpath Path to entry .js file * @returns {Promise} The bundled code */ async function bundleAmazon(inpath) { const externals = { 'aws-sdk': 'aws-sdk', } const res = await _bundle(inpath, externals) return res } module.exports = { bundleAmazon, } ================================================ FILE: bundler/google/index.js ================================================ const fs = require('fs') const { _bundle } = require('../utils') /** * @description Bundles a given .js files for Google using Webpack. IMPORTANT: excludes any npm packages. * @param {string} inpath Path to entry .js file * @returns {Promise} The bundled code */ async function bundleGoogle(inpath) { // Exclude any npm packages (if present) // From https://github.com/webpack/webpack/issues/603#issuecomment-180509359 const externals = fs.existsSync('node_modules') && fs.readdirSync('node_modules') // TODO check if @google is included on Google? const res = await _bundle(inpath, externals) return res } module.exports = { bundleGoogle, } ================================================ FILE: bundler/utils.js ================================================ const webpack = require('webpack') const path = require('path') const fsp = require('fs').promises const os = require('os') const { log } = require('../printers/index') /** * @description Bundles a given .js files with its dependencies using webpack. * Does not include dependencies that are given in "externals". * @param {string} inpath Path to entry .js file * @param {*} externals Webpack 'externals' field of package names we don't need to bundle. * For example { 'aws-sdk': 'aws-sdk' } to skip 'aws-sdk' * @returns {Promise} The bundled code */ async function _bundle(inpath, externals) { // create out dir (silly webpack) const outdir = await fsp.mkdtemp(path.join(os.tmpdir(), 'bundle-')) const outpath = path.join(outdir, 'bundle.js') // console.log(`bundling to ${outpath}`) return new Promise((resolve, reject) => { webpack( { mode: 'development', entry: inpath, target: 'node', output: { path: outdir, filename: 'bundle.js', // so amazon sees it libraryTarget: 'commonjs', // Make webpack perform identical as node // See https://github.com/node-fetch/node-fetch/issues/450#issuecomment-494475397 // extensions: ['.js'], // mainFields: ['main'], // Fixes "global" // See https://stackoverflow.com/a/64639975 // globalObject: 'this', }, // aws-sdk is already provided in lambda externals: externals, }, (err, stats) => { if (err || stats.hasErrors() || (stats.compilation.errors.length > 0)) { // always show bundling error it's useful log(`Bundling ${inpath} did not work: `) log(stats.compilation.errors) reject(err, stats) } else { // return the bundle code fsp.readFile(outpath, { encoding: 'utf8' }) .then((code) => resolve(code)) // TODO clean up file // TODO do in-memory } }, ) }) } module.exports = { _bundle, } ================================================ FILE: bundler/utils.test.js ================================================ const fsp = require('fs').promises const os = require('os') const path = require('path') const uuidv4 = require('uuid').v4 const { _bundle } = require('./utils') describe('bundler', () => { test('does not throw on empty js file & returns string', async () => { const tmpd = await fsp.mkdtemp(path.join(os.tmpdir(), 'bundle-')) const inpath = path.join(tmpd, 'index.js') const filecontents = ' ' await fsp.writeFile(inpath, filecontents) let err; let res try { res = await _bundle(inpath) } catch (e) { console.log(e) err = e } expect(err).not.toBeDefined() expect(typeof res === 'string').toBe(true) }) test('does not throw on js file & returns string', async () => { const tmpd = await fsp.mkdtemp(path.join(os.tmpdir(), 'bundle-')) const code = 'module.exports.inc = (x) => x + 1' const inpath = path.join(tmpd, 'index.js') await fsp.writeFile(inpath, code) let err; let res try { res = await _bundle(inpath) } catch (e) { console.log(e) err = e } expect(err).not.toBeDefined() expect(typeof res === 'string').toBe(true) }) test('throws on invalid input path', async () => { const code = 'module.exports.inc = (x) => x + 1' const invalidinpath = path.join(os.tmpdir(), `surely-this-path-does-not-exist-${uuidv4()}`) let err; let res try { res = await _bundle(invalidinpath) console.log(res) } catch (e) { err = e } expect(err).toBeDefined() }) }) ================================================ FILE: cli.js ================================================ #!/usr/bin/env node const path = require('path') const fs = require('fs') const semver = require('semver') const { init, initDumb } = require('./initer/index') const { getParsedHyperformJson } = require('./parser/index') const { log } = require('./printers/index') const { maybeShowSurvey, answerSurvey } = require('./surveyor/index') const packagejson = require('./package.json') // Ingest CLI arguments // DEV NOTE: Keep it brief and synchronious const args = process.argv.slice(2) // Check node version const version = packagejson.engines.node if (semver.satisfies(process.version, version) !== true) { console.log(`Hyperform needs node ${version} or newer, but version is ${process.version}.`); process.exit(1); } if ( (/deploy/.test(args[0]) === false) || ((args.length === 1)) || (args.length === 2) || (args.length === 3 && args[2] !== '--amazon' && args[2] !== '--google') || (args.length === 4 && ([args[2], args[3]].includes('--url') === false)) || args.length >= 5) { log(`Usage: $ hf deploy ./some/file.js --amazon # Deploy exports to AWS Lambda $ hf deploy ./some/file.js --google # Deploy exports to Google Cloud Functions `) process.exit(1) } // $ hf MODE FPATH [--url] // const mode = args[0] const fpath = args[1] const isPublic = (args.length === 4 ? ([args[2], args[3]].includes('--url')) : false) const currdir = process.cwd() let platform if (args[2] === '--amazon') platform = 'amazon' if (args[2] === '--google') platform = 'google' // // Mode is init // if (mode === 'init') { // initDumb(currdir) // process.exit() // } // Mode is answer survey // if (mode === 'answer') { // const answer = args.slice(1) // words after $ hf answer // // Send anonymous answer (words and date recorded) // answerSurvey(answer) // .then(process.exit()) // } // Mode is deploy // try to read hyperform.json const hyperformJsonExists = fs.existsSync(path.join(currdir, 'hyperform.json')) if (hyperformJsonExists === false) { if (platform === 'amazon') { log(`No hyperform.json found in current directory. Create it with these fields: { "amazon": { "aws_access_key_id": "...", "aws_secret_access_key": "...", "aws_region": "..." } } `) } if (platform === 'google') { log(`No hyperform.json found in current directory. Create it with these fields: { "google": { "gc_project": "...", "gc_region": "...", } } `) } process.exit(1) } // parse and validate hyperform.json const parsedHyperformJson = getParsedHyperformJson(currdir, platform) // Dev Note: Do this as early as possible // Load AWS Credentials from hyperform.json into process.env // These are identical with variables that Amazon CLI uses, so they may be set // However, that is fine, hyperform.json should still take precedence if (parsedHyperformJson.amazon != null) { process.env.AWS_ACCESS_KEY_ID = parsedHyperformJson.amazon.aws_access_key_id, process.env.AWS_SECRET_ACCESS_KEY = parsedHyperformJson.amazon.aws_secret_access_key, process.env.AWS_REGION = parsedHyperformJson.amazon.aws_region // may, may not be defined. process.env.AWS_SESSION_TOKEN = parsedHyperformJson.amazon.aws_session_token } // Load GC Credentials from hyperform.json into process.env // These are different from what Google usually occupies (GCLOUD_...) if (parsedHyperformJson.google != null) { process.env.GC_PROJECT = parsedHyperformJson.google.gc_project process.env.GC_REGION = parsedHyperformJson.google.gc_region } // Top-level error boundary try { // Main // Do not import earlier, it needs to absorb process.env set above // TODO: make less sloppy const { main } = require('./index') main(currdir, fpath, platform, parsedHyperformJson, isPublic) // show anonymous survey question with 1/30 probability // .then(() => maybeShowSurvey()) } catch (e) { log(e) process.exit(1) } ================================================ FILE: copier/index.js ================================================ const { ncp } = require('ncp') const path = require('path') const os = require('os') const fsp = require('fs').promises /** * Creates a copy of a directory in /tmp. * @param {string} dir * * @param {string[]} except names of directories or files that will not be included * (usually ["node_modules", ".git", ".github"]) Uses substring check. * @returns {string} outpath */ async function createCopy(dir, except) { const outdir = await fsp.mkdtemp(path.join(os.tmpdir(), 'copy-')) // See https://www.npmjs.com/package/ncp // concurrency limit ncp.limit = 16 // Function that is called on every file / dir to determine if it'll be included in the zip const filterFunc = (p) => { for (let i = 0; i < except.length; i += 1) { if (p.includes(except[i])) { console.log(`Excluding ${p}`) return false } } return true } const res = await new Promise((resolve, rej) => { ncp( dir, outdir, { filter: filterFunc, }, (err) => { if (err) { rej(err) } else { resolve(outdir) } }, ); }) return res } module.exports = { createCopy, } ================================================ FILE: deployer/amazon/index.js ================================================ const { createLambda, deleteLambda, updateLambdaCode, createLambdaRole, isExistsAmazon, } = require('./utils') const { logdev } = require('../../printers/index') /** * @description If Lambda "options.name" does not exist yet in "options.region", * it deploys a new Lambda with given code ("pathToZip") and "options". * If Lambda exists, it just updates its code with "pathToZip", * and ignores all options except "options.name" and "options.region" * @param {*} pathToZip Path to the zipped Lambda code * @param {{ * name: string, * region: string, * ram?: number, * timeout?: number, * handler?: string * }} options * @returns {Promise} The Lambda ARN */ async function deployAmazon(pathToZip, options) { if (!options.name || !options.region) { throw new Error(`name and region must be specified, but are ${options.name}, ${options.region}`) // HF programmer mistake } const existsOptions = { name: options.name, region: options.region, } // check if lambda exists const exists = await isExistsAmazon(existsOptions) logdev(`amazon isexists ${options.name} : ${exists}`) // if not, create new role const roleName = `hf-${options.name}` const roleArn = await createLambdaRole(roleName) /* eslint-disable key-spacing */ const fulloptions = { name: options.name, region: options.region, role: roleArn, runtime: 'nodejs12.x', timeout: options.timeout || 60, // also prevents 0 ram: options.ram || 128, handler: options.handler || `index.${options.name}`, } /* eslint-enable key-spacing */ // anonymous function that when run, creates or updates Lambda let upload if (exists === true) { upload = async () => updateLambdaCode(pathToZip, fulloptions) } else { upload = async () => createLambda(pathToZip, fulloptions) } // Helper const sleep = async (millis) => new Promise((resolve) => { setTimeout(() => { resolve() }, millis); }) // Retry loop (4 times). Usually it fails once or twice // if role is newly created because it's too fresh // See: https://stackoverflow.com/a/37503076 let arn for (let i = 0; i < 4; i += 1) { try { logdev('trying to upload to amazon') arn = await upload() logdev('success uploading to amazon') break // we're done } catch (e) { // TODO write test that enters here, reliably if (e.code === 'InvalidParameterValueException') { logdev('amazon deploy threw InvalidParameterValueException (role not ready yet). Retrying in 3 seconds...') await sleep(3000) // wait 3 seconds continue } else { logdev(`Amazon upload errorred: ${e}`) logdev(JSON.stringify(e, null, 2)) throw e; } } } console.timeEnd(`Amazon-deploy-${options.name}`) return arn } /** * @description Deletes a Lambda function in a given region. * @param {string} name Name, ARN or partial ARN of the function * @param {string} region Region of the function * @throws ResourceNotFoundException, among others */ async function deleteAmazon(name, region) { await deleteLambda(name, region) } module.exports = { deployAmazon, deleteAmazon, } ================================================ FILE: deployer/amazon/index.test.js ================================================ /* eslint-disable global-require */ const LAMBDANAME = 'jest-reserved-returna1' const LAMBDAREGION = 'us-east-2' // After all tests, delete the Lambda afterAll(async () => { const { deleteAmazon } = require('./index') try { await deleteAmazon(LAMBDANAME, LAMBDAREGION) } catch (e) { /* tests themselves already deleted the Lambda */ } }) // Helpers const arnRegex = /arn:(aws[a-zA-Z-]*)?:lambda:[a-z]{2}((-gov)|(-iso(b?)))?-[a-z]+-\d{1}:\d{12}:function:[a-zA-Z0-9-_]+(:(\$LATEST|[a-zA-Z0-9-_]+))?/ const isArn = (str) => (typeof str === 'string') && (arnRegex.test(str) === true) describe('deployer', () => { describe('amazon', () => { describe('deployAmazon', () => { test('completes if Lambda does not exist, and returns ARN', async () => { const { deleteAmazon, deployAmazon } = require('./index.js') const { zip } = require('../../zipper/index') /// ////////////////////////////////////////////// // Setup: delete function if it exists already try { await deleteAmazon(LAMBDANAME, LAMBDAREGION) // deleted function } catch (e) { if (e.code === 'ResourceNotFoundException') { // does not exist in the first place, nice } else { throw e } } /// ////////////////////////////////////////////// // Setup: create code zip const code = `module.exports = { ${LAMBDANAME}: () => ({a: 1}) }` const zipPath = await zip({ 'index.js': code, }) /// //////////////////////////////////////////////////// const options = { name: LAMBDANAME, region: LAMBDAREGION, } let err let lambdaArn try { lambdaArn = await deployAmazon(zipPath, options) } catch (e) { console.log(e) err = e } // it completed expect(err).not.toBeDefined() // it's an ARN expect(isArn(lambdaArn)).toBe(true) // TODO }, 15 * 1000) test('completes if Lambda already exists, and returns ARN', async () => { const { deployAmazon } = require('./index') const { zip } = require('../../zipper/index') /// ////////////////////////////////////////////// // Setup: create code zip const code = `module.exports = { ${LAMBDANAME}: () => ({a: 1}) }` const zipPath = await zip({ 'index.js': code, }) /// ////////////////////////////////////////////// // Setup: ensure function exists const options = { name: LAMBDANAME, region: LAMBDAREGION, } try { await deployAmazon(zipPath, options) } catch (e) { if (e.code === 'ResourceConflictException') { /* exists already anyway, cool */ } else { throw e } } /// //////////////////////////////////////////////////// // Actual test let err let lambdaArn try { lambdaArn = await deployAmazon(zipPath, options) } catch (e) { console.log(e) err = e } // it completed expect(err).not.toBeDefined() // it's an ARN expect(isArn(lambdaArn)).toBe(true) }, 30 * 1000) }) // NOTE if we're short on API calls we can sacrifice this: // describe('isExistsAmazon', () => { // test('returns true on existing Lambda in name, region', async () => { // const { isExistsAmazon } = require('./utils') // const { deployAmazon } = require('./index') // const { zip } = require('../../zipper/index') // /// ////////////////////////////////////////////// // // Setup: create code zip // const code = `module.exports = { ${LAMBDANAME}: () => ({a: 1}) }` // const zipPath = await zip(code) // /// ////////////////////////////////////////////// // // Setup: ensure function exists // const deployOptions = { // name: LAMBDANAME, // region: LAMBDAREGION, // } // try { // await deployAmazon(zipPath, deployOptions) // } catch (e) { // if (e.code === 'ResourceConflictException') { // /* exists already anyway, cool */ // } else { // throw e // } // } // let err // let res // try { // res = await isExistsAmazon({ name: LAMBDANAME, region: LAMBDAREGION }) // } catch (e) { // err = e // } // expect(err).not.toBeDefined() // expect(typeof res).toBe('boolean') // expect(res).toEqual(true) // }) // test('returns false on non-existing Lambda in name, region', async () => { // const { isExistsAmazon } = require('./utils') // const uuidv4 = require('uuid').v4 // const options = { // name: `some-invalid-lambda-name-${uuidv4()}`, // region: LAMBDAREGION, // or whatever // } // let err // let res // try { // res = await isExistsAmazon(options) // } catch (e) { // err = e // } // expect(err).not.toBeDefined() // expect(typeof res).toBe('boolean') // expect(res).toEqual(false) // }) // }) // TODO more tests for the other methods }) }) ================================================ FILE: deployer/amazon/utils.js ================================================ const util = require('util'); const exec = util.promisify(require('child_process').exec); const AWS = require('aws-sdk') const fsp = require('fs').promises const { logdev } = require('../../printers/index') const conf = { accessKeyId: process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, region: process.env.AWS_REGION, // may, may not be defined // sessionToken: process.env.AWS_SESSION_TOKEN || undefined, } if (process.env.AWS_SESSION_TOKEN != null && process.env.AWS_SESSION_TOKEN !== 'undefined') { conf.sessionToken = process.env.AWS_SESSION_TOKEN } AWS.config.update(conf) /** * @description Creates a new Lambda with given function code and options * @param {string} pathToZip Path to the zipped Lambda code * @param {{ * name: string, * region: string, * role: string, * runtime: string, * timeout: number, * ram: number, * handler: string * }} options * @returns {Promise} The ARN of the Lambda * @throws If Lambda exists or creation did not succeed */ async function createLambda(pathToZip, options) { const lambda = new AWS.Lambda({ region: options.region, apiVersion: '2015-03-31', }) const zipContents = await fsp.readFile(pathToZip) const params = { Code: { ZipFile: zipContents, }, FunctionName: options.name, Timeout: options.timeout, Role: options.role, MemorySize: options.ram, Handler: options.handler, Runtime: options.runtime, } const res = await lambda.createFunction(params).promise() const arn = res.FunctionArn return arn } /** * @description Deletes a Lambda function in a given region. * @param {string} name Name, ARN or partial ARN of the function * @param {string} region Region of the function * @throws ResourceNotFoundException, among others */ async function deleteLambda(name, region) { const lambda = new AWS.Lambda({ region: region, apiVersion: '2015-03-31', }) const params = { FunctionName: name, } await lambda.deleteFunction(params).promise() } /** * @description Updates a Lambda's function code with a given .zip file * @param {string} pathToZip Path to the zipped Lambda code * @param {{ * name: string, * region: string * }} options * @throws If Lambda does not exist or update did not succeed * @returns {Promise} The ARN of the Lambda */ async function updateLambdaCode(pathToZip, options) { const lambda = new AWS.Lambda({ region: options.region, apiVersion: '2015-03-31', }) const zipContents = await fsp.readFile(pathToZip) const params = { FunctionName: options.name, ZipFile: zipContents, } const res = await lambda.updateFunctionCode(params).promise() const arn = res.FunctionArn return arn } /** * @description Creates a new role, and attaches a basic Lambda policy * (AWSLambdaBasicExecutionRole) to it. If role with that name * exists already, it just attaches the policy to it * @param {string} roleName Unique name to be given to the role * @returns {Promise} ARN of the created or updated role * @see https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/using-lambda-iam-role-setup.html */ async function createLambdaRole(roleName) { const iam = new AWS.IAM() const lambdaPolicy = { Version: '2012-10-17', Statement: [ { Effect: 'Allow', Principal: { Service: 'lambda.amazonaws.com', }, Action: 'sts:AssumeRole', }, ], } const createParams = { AssumeRolePolicyDocument: JSON.stringify(lambdaPolicy), RoleName: roleName, } let roleArn try { const createRes = await iam.createRole(createParams).promise() // Role did not exist yet roleArn = createRes.Role.Arn } catch (e) { if (e.code === 'EntityAlreadyExists') { // Role with that name already exists // Use that role, proceed normally const getParams = { RoleName: roleName, } logdev(`role with name ${roleName} already exists. getting its arn`) const getRes = await iam.getRole(getParams).promise() roleArn = getRes.Role.Arn } else { // some other error throw e } } // Attach a basic Lambda policy to the role (allows writing to cloudwatch logs etc) // Equivalent to in Lambda console, selecting 'Create new role with basic permissions' const policyParams = { PolicyArn: 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole', RoleName: roleName, } await iam.attachRolePolicy(policyParams).promise() logdev(`successfully attached AWSLambdaBasicExecutionRole to ${roleName}`) return roleArn } /** * @description Checks whether a Lambda exists in a given region * @param {{ * name: string, * region: string * }} options * @returns {Promise} */ async function isExistsAmazon(options) { const lambda = new AWS.Lambda({ region: options.region, apiVersion: '2015-03-31', }) const params = { FunctionName: options.name, } try { await lambda.getFunction(params).promise() return true } catch (e) { if (e.code === 'ResourceNotFoundException') { return false } else { // some other error throw e } } } // TODO // /** // * @throws If Lambda does not exist // */ // async function updateLambdaConfiguration() { // } module.exports = { createLambda, deleteLambda, updateLambdaCode, createLambdaRole, isExistsAmazon, } ================================================ FILE: deployer/google/index.js ================================================ const fsp = require('fs').promises const { CloudFunctionsServiceClient } = require('@google-cloud/functions'); const fetch = require('node-fetch') const { logdev } = require('../../printers/index') let gcOptions if (process.env.GC_PROJECT) { gcOptions = { projectId: process.env.GC_PROJECT, } } // Don't consult hyperform.json yet for Google credentials // if (process.env.GC_CLIENT_EMAIL && process.env.GC_PRIVATE_KEY && process.env.GC_PROJECT) { // gcOptions = { // credentials: { // client_email: process.env.GC_CLIENT_EMAIL, // private_key: process.env.GC_PRIVATE_KEY, // }, // projectId: process.env.GC_PROJECT, // } // } const client = new CloudFunctionsServiceClient(gcOptions) /** * @description Checks whether a GCF * exists in a given project & region * @param {{ * name: string, * project:string * region: string, * }} options * @returns {Promise} */ async function isExistsGoogle(options) { const getParams = { name: `projects/${options.project}/locations/${options.region}/functions/${options.name}`, } try { const res = await client.getFunction(getParams) if (res.length > 0 && res.filter((el) => el).length > 0) { return true } else { return false } } catch (e) { return false } } /** * @description Uploads a given file (usually code .zips) to a temporary * Google storage and returns * its so-called signed URL. * This URL can then be used, for example * as sourceUploadUrl for creating and updating Cloud Functions. * @param {string} pathToFile * @param {{ * project: string, * region: string * }} options * @returns {Promise} The signed upload URL * @see https://cloud.google.com/storage/docs/access-control/signed-urls Google Documentation */ async function uploadGoogle(pathToFile, options) { const generateUploadUrlOptions = { parent: `projects/${options.project}/locations/${options.region}`, } const signedUploadUrl = (await client.generateUploadUrl(generateUploadUrlOptions))[0].uploadUrl // Upload zip // TODO use createReadStream instead const zipBuf = await fsp.readFile(pathToFile) await fetch(signedUploadUrl, { method: 'PUT', headers: { 'content-type': 'application/zip', 'x-goog-content-length-range': '0,104857600', }, body: zipBuf, }) logdev('uploaded zip to google signed url') return signedUploadUrl } /** * @description Updates an existing GCF "options.name" in "options.project", "options.region" * with given uploaded code .zip. * And, in theory, arbitrary options too (timeout, availableMemoryMb), * but currently not needed but could easily be added. * Returns immediately, but Google updates for 1-2 minutes more * @param {string} signedUploadUrl Signed upload URL where .zip has been uploaded to already. * Output of "uploadGoogle". * @param {{ * name: string, * project: string, * region: string, * runtime: string * }} options */ async function _updateGoogle(signedUploadUrl, options) { const updateOptions = { function: { name: `projects/${options.project}/locations/${options.region}/functions/${options.name}`, httpsTrigger: { url: `https://${options.region}-${options.project}.cloudfunctions.net/${options.name}`, }, runtime: options.runtime, timeout: { seconds: 120, }, sourceUploadUrl: signedUploadUrl, }, // TODO Empty array to not overwrite 'timeout' or 'runtime' updateMask: null, // this does not work :( // updateMask: { // paths: ['sourceUploadUrl'] // } } const res = await client.updateFunction(updateOptions) logdev(`google: updated function ${options.name}`) } /** * @description Creates a new GCF "options.name" in "options.project", "options.region" * with given uploaded code .zip and options. * Returns immediately, but Google creates for 1-2 minutes more * @param {string} signedUploadUrl * @param {{ * name: string, * project: string, * region: string, * runtime: string, * entrypoint?: string, * }} options */ async function _createGoogle(signedUploadUrl, options) { const createOptions = { location: `projects/${options.project}/locations/${options.region}`, function: { name: `projects/${options.project}/locations/${options.region}/functions/${options.name}`, httpsTrigger: { url: `https://${options.region}-${options.project}.cloudfunctions.net/${options.name}`, }, entryPoint: options.entrypoint || options.name, runtime: options.runtime, sourceUploadUrl: signedUploadUrl, timeout: { seconds: 60, // }, // those are the defaults anyway availableMemoryMb: 256, }, } const res = await client.createFunction(createOptions) // TODO wait for operaton to complete (ie setInterval done && !error, promise resolve then) // TODO in _updateGoogle too logdev(`google: created function ${options.name}`) } /** * * @param {{ * name: string, * project: string, * region: string * }} options */ async function _allowPublicInvokeGoogle(options) { // TODO GetIam and get etag of current role first // And then specify that in setIam, to avoid race conditions // @see "etag" on https://cloud.google.com/functions/docs/reference/rest/v1/Policy const setIamPolicyOptions = { resource: `projects/${options.project}/locations/${options.region}/functions/${options.name}`, policy: { // @see https://cloud.google.com/functions/docs/reference/rest/v1/Policy#Binding bindings: [ { role: 'roles/cloudfunctions.invoker', members: ['allUsers'], version: 3, }, ], }, } logdev('setting IAM policy') const res = await client.setIamPolicy(setIamPolicyOptions) } /** * @description If Google Cloud Function "options.name" * does not exist yet in "options.project", "options.region", * it creates a new GCF with given code ("pathToZip") and "options". * If GCF exists already, it updates its code with "pathToZip". * If other options are specified, it can update those too (currently only "runtime"). * Returns IAM-protected URL immediately, but Cloud Function takes another 1-2 minutes to be invokable. * @param {string} pathToZip * @param {{ * name: string, * project: string, * region: string, * runtime: string, * entrypoint?: string * }} options * @returns {Promise} The endpoint URL * @see https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions#CloudFunction For the underlying Google SDK documentation */ async function deployGoogle(pathToZip, options) { if (!options.name || !options.project || !options.region || !options.runtime) { throw new Error(`name, project and region and runtime must be defined but are ${options.name}, ${options.project}, ${options.region}, ${options.runtime}`) // HF programmer mistake } const existsOptions = { name: options.name, project: options.project, region: options.region, } // Check if GCF exists const exists = await isExistsGoogle(existsOptions) logdev(`google isexists ${options.name}: ${exists}`) // Either way, upload the .zip // @see https://cloud.google.com/storage/docs/access-control/signed-urls const signedUploadUrl = await uploadGoogle(pathToZip, { project: options.project, region: options.region, }) // if GCF does not exist yet, create it if (exists !== true) { const createParams = { ...options, } await _createGoogle(signedUploadUrl, createParams) } else { // GCF exists, update code and options (currently none) const updateParams = { name: options.name, project: options.project, region: options.region, runtime: options.runtime, } await _updateGoogle(signedUploadUrl, updateParams) } // Construct endpoint URL (it's deterministic) const endpointUrl = `https://${options.region}-${options.project}.cloudfunctions.net/${options.name}` // Note: GCF likely not ready by the time we return its URL here return endpointUrl } /** * @description Allows anyone to call function via its HTTP endpoint. * Does so by turning IAM checking of Google off. * Unlike publishAmazon, publishgoogle it does not return an URL, deployGoogle does that already. * @param {*} name * @param {*} project * @param {*} region */ async function publishGoogle(name, project, region) { const allowPublicInvokeOptions = { name: name, project: project, region: region, } await _allowPublicInvokeGoogle(allowPublicInvokeOptions) } module.exports = { deployGoogle, publishGoogle, _only_for_testing_isExistsGoogle: isExistsGoogle, } ================================================ FILE: deployer/google/index.test.js ================================================ const GCFREGION = 'us-central1' const GCFPROJECT = 'firstnodefunc' const GCFRUNTIME = 'nodejs12' describe('deployer', () => { describe('google', () => { describe('deployGoogle', () => { // Google does not reliably complete deploy/delete at returning // Therefore you can't really setup it properly // because setup might overlap with the test itself test('completes if GCF exists already, and returns an URL', async () => { const { deployGoogle } = require('./index') const { zip } = require('../../zipper/index') /// ////////////////////////////////////////////// // Setup: create folder with code /// ////////////////////////////////////////////// // Setup: create code zip const name = 'jest_reserved_deployGoogle_A' const code = `module.exports = { ${name}: () => ({a: 1}) }` const zipPath = await zip({ 'index.js': code, }) /// //////////////////////////////////////////////////// const options = { name: name, project: GCFPROJECT, region: GCFREGION, runtime: GCFRUNTIME, } let err let res try { res = await deployGoogle(zipPath, options) } catch (e) { console.log(e) err = e } // it completed expect(err).not.toBeDefined() // it's an URL const tryUrl = () => new URL(res) expect(tryUrl).not.toThrow() // Note: the URL would then take another 1-2 minutes to point to a proper GCF }, 2 * 60 * 1000) }) describe('isExistsGoogle', () => { test('true on existing function in project, region', async () => { const isExistsGoogle = require('./index')._only_for_testing_isExistsGoogle const input = { name: 'endpoint_oho', // TODO make reserved funcs for tests project: GCFPROJECT, region: GCFREGION, } let err let res try { res = await isExistsGoogle(input) } catch (e) { err = e } expect(err).not.toBeDefined() expect(res).toEqual(true) }) test('false on non-existing function in project, region', async () => { const isExistsGoogle = require('./index')._only_for_testing_isExistsGoogle const input = { name: 'SOME_NONEXISTING_FUNCTION_0987654321', // TODO make reserved funcs for tests project: GCFPROJECT, region: GCFREGION, } let err let res try { res = await isExistsGoogle(input) } catch (e) { err = e } // should still not throw expect(err).not.toBeDefined() expect(res).toEqual(false) }) }) }) }) ================================================ FILE: discoverer/index.js ================================================ const findit = require('findit') const path = require('path') const { EOL } = require('os') const fs = require('fs') const { log } = require('../printers/index') const BLACKLIST = [ '.git', 'node_modules', ] /** * @description Searches "absdir" and its subdirectories for .js files * @param {string} absdir An absolute path to a directory * @returns {Promise} Array of absolute file paths to .js files */ // TODO do not follow symlinks (or do?) function getJsFilepaths(absdir) { return new Promise((resolve, reject) => { const fnames = [] const finder = findit(absdir) finder.on('directory', (_dir, stat, stop) => { const base = path.basename(_dir); if (BLACKLIST.includes(base)) { stop() } }); finder.on('file', (file, stat) => { // only return .js files if (/.js$/.test(file) === true) { fnames.push(file) } }); finder.on('end', () => { resolve(fnames) }) finder.on('error', (err) => { reject(err) }) }) } /** * @description Runs a .js file to get its named export keys * @param {string} filepath Path to .js file * @returns {string[]} Array of named export keys */ function getNamedExportKeys(filepath) { // hides that code is run but actually runs it lol // TODOfind a way to get exports without running code try { const imp = (() => // console.log = () => {} // console.error = () => {} // console.warn = () => {} require(filepath) )() const namedexpkeys = Object.keys(imp) return namedexpkeys } catch (e) { // if js file isn't parseable, top level code throws, etc // ignore it log(`Could not determine named exports of ${filepath}. Try to fix the error and try again: ${EOL} Error: ${e}`) return [] } } /** * Checks if file contents match a regex * @param {*} fpath * @param {*} regex * @returns {boolean} */ function isFileContains(fpath, regex) { const contents = fs.readFileSync(fpath, { encoding: 'utf-8' }) const res = regex.test(contents) return res } /** * @description Scouts "dir" and its subdirectories for .js files named * exports that match "fnregex" * @param {string} dir * @param {Regex} fnregex * @returns {[ { p: string, exps: string[] } ]} Array of "p" (path to js file) * and its named exports that match fnregex ("exps") */ async function getInfos(dir, fnregex) { let jsFilePaths = await getJsFilepaths(dir) // First check - filter out files that don't even contain a string matching fnregex (let alone export it) jsFilePaths = jsFilePaths .filter((p) => isFileContains(p, fnregex)) // Second check - determine exports by running file, and keep those that export sth matching fnregex const infos = jsFilePaths .map((p) => ({ p: p, exps: getNamedExportKeys(p), })) // skip files that don't have named exports .filter(({ exps }) => exps != null && exps.length > 0) // skip files that don't have named exports that fit fnregex .filter(({ exps }) => exps.some((exp) => fnregex.test(exp) === true)) // filter out exports that don't fit fnregex .map((el) => ({ ...el, exps: el.exps.filter((exp) => fnregex.test(exp) === true) })) return infos } module.exports = { getInfos, getNamedExportKeys, } ================================================ FILE: index.js ================================================ /* eslint-disable max-len */ const chalk = require('chalk') const semver = require('semver') const fs = require('fs') const fsp = require('fs').promises const path = require('path') const { EOL } = require('os') const { bundleAmazon } = require('./bundler/amazon/index') const { bundleGoogle } = require('./bundler/google/index') const { getNamedExportKeys } = require('./discoverer/index') const { deployAmazon } = require('./deployer/amazon/index') const { publishAmazon } = require('./publisher/amazon/index') const { spinnies, log, logdev } = require('./printers/index') const { zip } = require('./zipper/index') const { deployGoogle, publishGoogle } = require('./deployer/google/index') const { transpile } = require('./transpiler/index') const packagejson = require('./package.json') const { createCopy } = require('./copier/index') const { zipDir } = require('./zipper/google/index') const { kindle } = require('./kindler/index') const { amazonSchema, googleSchema } = require('./schemas/index') /** * * @param {string} fpath Path to .js file */ async function bundleTranspileZipAmazon(fpath) { // Bundle let amazonBundledCode try { amazonBundledCode = await bundleAmazon(fpath) } catch (e) { log(`Errored bundling ${fpath} for Amazon: ${e}`) return // just skip that file } // Transpile const amazonTranspiledCode = transpile(amazonBundledCode) // Zip try { const amazonZipPath = await zip({ 'index.js': amazonTranspiledCode, }) return amazonZipPath } catch (e) { // probably underlying issue with the zipping library or OS // skip that file log(`Errored zipping ${fpath} for Amazon: ${e}`) } } // TODO those are basically the same now // but for later it may be good to have them separate // in case they start to diverge // /** // * // * @param {string} fpath Path to .js file // * @param {string} dir // */ // async function bundleTranspileZipGoogle(fpath, dir) { // // Bundle (omits any npm packages) // let googleBundledCode // try { // googleBundledCode = await bundleGoogle(fpath) // } catch (e) { // log(`Errored bundling ${fpath} for Google: ${e}`) // return // just skip that file // } // // Transpile // const googleTranspiledCode = transpile(googleBundledCode) // // Try to locate a package.json // // Needed so google installs the npm packages // const packageJsonPath = path.join(dir, 'package.json') // let packageJsonContent // if (fs.existsSync(packageJsonPath)) { // packageJsonContent = fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }) // } else { // // warn // log(`No package.json found in this directory. // On Google, therefore no dependencies will be included`) // } // // Zip code and package.json // try { // const googleZipPath = await zip({ // 'index.js': googleTranspiledCode, // 'package.json': packageJsonContent || undefined, // }) // return googleZipPath // } catch (e) { // // probably underlying issue with the zipping library or OS // throw new Error(`Errored zipping ${fpath} for Google: ${e}`) // } // } async function bundleTranspileZipGoogle(fpath, dir, exps) { // warn if package.json does not exist // (Google won't install npm dependencies then) if (fs.existsSync(path.join(dir, 'package.json')) === false) { log(`No package.json found in this directory. On Google, therefore no dependencies will be included`) } // copy whole dir to /tmp so we can tinker with it const googlecopyDir = await createCopy( dir, ['node_modules', '.git', '.github', 'hyperform.json'], ) const indexJsPath = path.join(googlecopyDir, 'index.js') let indexJsAppendix = '' // add import-export appendix indexJsAppendix = kindle(indexJsAppendix, dir, [ { p: fpath, exps: exps, }, ]) // add platform appendix indexJsAppendix = transpile(indexJsAppendix) // write or append to index.js in our tinker folder if (fs.existsSync(indexJsPath) === false) { await fsp.writeFile(indexJsPath, indexJsAppendix, { encoding: 'utf-8' }) } else { await fsp.appendFile(indexJsPath, indexJsAppendix, { encoding: 'utf-8' }) } // zip tinker folder const googleZipPath = await zipDir( googlecopyDir, ['node_modules', '.git', '.github', 'hyperform.json'], // superfluous we didnt copy them in the first place ) return googleZipPath } /** * @description Deploys a given code .zip to AWS Lambda, and gives it a HTTP endpoint via API Gateway * @param {string} name * @param {string} region * @param {string} zipPath * @param {boolean} isPublic whether to publish * @returns {string?} If isPublic was true, URL of the endpoint of the Lambda */ async function deployPublishAmazon(name, region, zipPath, isPublic) { const amazonSpinnieName = `amazon-main-${name}` try { spinnies.add(amazonSpinnieName, { text: `Deploying ${name} to AWS Lambda` }) // Deploy it const amazonDeployOptions = { name: name, region: region, } const amazonArn = await deployAmazon(zipPath, amazonDeployOptions) let amazonUrl // Publish it if isPpublic if (isPublic === true) { amazonUrl = await publishAmazon(amazonArn, region) } spinnies.succ(amazonSpinnieName, { text: `🟢 Deployed ${name} to AWS Lambda ${amazonUrl || ''}` }) // (return url) return amazonUrl } catch (e) { spinnies.f(amazonSpinnieName, { text: `Error deploying ${name} to AWS Lambda: ${e.stack}`, }) logdev(e, e.stack) return null } } // TODO probieren // TODO tests anpassen // TODO testen // TODO tests schreiben, refactoren /** * @description Deploys and publishes a give code .zip to Google Cloud Functions * @param {string} name * @param {string} region * @param {string} project * @param {string} zipPath * @param {boolean} isPublic whether to publish * @returns {string?} If isPublic was true, URL of the Google Cloud Function */ async function deployPublishGoogle(name, region, project, zipPath, isPublic) { const googleSpinnieName = `google-main-${name}` try { spinnies.add(googleSpinnieName, { text: `Deploying ${name} to Google Cloud Functions` }) const googleOptions = { name: name, project: project, // process.env.GC_PROJECT, region: region, // TODO get from parsedhyperfromjson runtime: 'nodejs12', } const googleUrl = await deployGoogle(zipPath, googleOptions) if (isPublic === true) { // enables anyone with the URL to call the function await publishGoogle(name, project, region) } spinnies.succ(googleSpinnieName, { text: `🟢 Deployed ${name} to Google Cloud Functions ${googleUrl || ''}` }) console.log('Google takes another 1 - 2m for changes to take effect') // return url return googleUrl } catch (e) { spinnies.f(googleSpinnieName, { text: `${chalk.rgb(255, 255, 255).bgWhite(' Google ')} ${name}: ${e.stack}`, }) logdev(e, e.stack) return null } } /** * @param {string} dir * @param {Regex} fpath the path to the .js file whose exports should be deployed * @param {amazon|google} platform * @param {boolean?} _isPublic * @param {{amazon: {aws_access_key_id:string, aws_secret_access_key: string, aws_region: string}}} parsedHyperformJson */ async function main(dir, fpath, platform, parsedHyperformJson, _isPublic) { // Check node version (again) const version = packagejson.engines.node if (semver.satisfies(process.version, version) !== true) { console.log(`Hyperform needs node ${version} or newer, but version is ${process.version}.`); process.exit(1); } // verify parsedHyperformJson (again) let schema if (platform === 'amazon') schema = amazonSchema if (platform === 'google') schema = googleSchema const { error, value } = schema.validate(parsedHyperformJson) if (error) { throw new Error(`${error} ${value}`) } const absfpath = path.resolve(dir, fpath) // determine named exports const exps = getNamedExportKeys(absfpath) if (exps.length === 0) { log(`No named CommonJS exports found in ${absfpath}. ${EOL}Named exports have the form 'module.exports = { ... }' or 'exports.... = ...' `) return [] // no endpoint URLs created } const isToAmazon = platform === 'amazon' const isToGoogle = platform === 'google' let amazonZipPath let googleZipPath if (isToAmazon === true) { amazonZipPath = await bundleTranspileZipAmazon(absfpath) } if (isToGoogle === true) { googleZipPath = await bundleTranspileZipGoogle(absfpath, dir, exps) } /// /////////////////////////////////////////////////// /// Each export, deploy as function & publish. Obtain URL. /// /////////////////////////////////////////////////// const isPublic = _isPublic || false let endpoints = await Promise.all( // For each export exps.map(async (exp) => { /// ////////////////////////////////////////////////////////// /// Deploy to Amazon /// ////////////////////////////////////////////////////////// let amazonUrl if (isToAmazon === true) { amazonUrl = await deployPublishAmazon( exp, parsedHyperformJson.amazon.aws_region, amazonZipPath, isPublic, ) } /// ////////////////////////////////////////////////////////// /// Deploy to Google /// ////////////////////////////////////////////////////////// let googleUrl if (isToGoogle === true) { googleUrl = await deployPublishGoogle( exp, // TODO lol parsedHyperformJson.google.gc_region, parsedHyperformJson.google.gc_project, // TODO googleZipPath, isPublic, ) } return amazonUrl || googleUrl // for tests etc }), ) endpoints = endpoints.filter((el) => el) return { urls: endpoints } /// ////////////////////////////////////////////////////////// // Bundle and zip for Google (once) // /// ////////////////////////////////////////////////////////// // TODO // NOTE that google and amazon now work fundamentally different // Google - 1 deployment package // For each file // bundle // transpile // // Amazon // // zip // // deployAmazon // // publishAmazon // // Later instead of N times, just create 1 deployment package for all functions // const endpoints = await Promise.all( // // For each file // infos.map(async (info) => { // const toAmazon = parsedHyperformJson.amazon != null // const toGoogle = parsedHyperformJson.google != null // /// ////////////////////////////////////////////////////////// // // Bundle and zip for Amazon // // /// ////////////////////////////////////////////////////////// // let amazonZipPath // if (toAmazon === true) { // amazonZipPath = await bundleTranspileZipAmazon(info.p) // } // /// ////////////////////////////////////////////////////////// // // Bundle and zip for Google // // // NOW DONE ABOVE // /// ////////////////////////////////////////////////////////// // // let googleZipPath // // if (toGoogle === true) { // // googleZipPath = await bundleTranspileZipGoogle(info.p) // // } // // For each matching export // const endpts = await Promise.all( // info.exps.map(async (exp, idx) => { // /// ////////////////////////////////////////////////////////// // /// Deploy to Amazon // /// ////////////////////////////////////////////////////////// // let amazonUrl // if (toAmazon === true) { // amazonUrl = await deployPublishAmazon( // exp, // parsedHyperformJson.amazon.aws_region, // amazonZipPath, // isPublic, // ) // } // /// ////////////////////////////////////////////////////////// // /// Deploy to Google // /// ////////////////////////////////////////////////////////// // let googleUrl // if (toGoogle === true) { // googleUrl = await deployPublishGoogle( // exp, // 'us-central1', // 'hyperform-7fd42', // TODO // googleZipPath, // isPublic, // ) // } // return [amazonUrl, googleUrl].filter((el) => el) // for tests etc // }), // ) // return [].concat(...endpts) // }), // ) // return { urls: endpoints } } module.exports = { main, } ================================================ FILE: index.test.js ================================================ /* eslint-disable no-await-in-loop, global-require */ // One test to rule them all const os = require('os') const path = require('path') const fsp = require('fs').promises const TIMEOUT = 1 * 60 * 1000 describe('System tests (takes 1-2 minutes)', () => { describe('main', () => { describe('amazon', () => { test('completes', async () => { const { main } = require('./index') /// //////////////////////////////////////////// // Set up const tmpdir = path.join( os.tmpdir(), `${Math.ceil(Math.random() * 100000000000)}`, ) // // What we will pass to the functions // const random_string = uuidv4() // const event_body = { // random_string, // } // const event_querystring = `?random_string=${random_string}` // Create javascript files await fsp.mkdir(tmpdir) const code = ` function irrelevant() { return 100 } function jest_systemtest_amazon(event, context, callback) { context.succeed({}) } module.exports = { jest_systemtest_amazon } ` const tmpcodepath = path.join(tmpdir, 'index.js') await fsp.writeFile(tmpcodepath, code, { encoding: 'utf-8' }) const arg1 = tmpdir const arg2 = tmpcodepath /// //////////////////////////////////////////// // Run main for Amazon /// //////////////////////////////////////////// let amazonMainRes const amazonarg3 = 'amazon' const amazonarg4 = { amazon: { aws_access_key_id: process.env.AWS_ACCESS_KEY_ID, aws_secret_access_key: process.env.AWS_SECRET_ACCESS_KEY, aws_region: process.env.AWS_REGION, }, } let err try { amazonMainRes = await main(arg1, arg2, amazonarg3, amazonarg4) } catch (e) { console.log(e) err = e } // Expect main did not throw expect(err).not.toBeDefined() // // Expect main returned sensible data // expect(amazonMainRes).toBeDefined() // expect(amazonMainRes.urls).toBeDefined() }, TIMEOUT) }) describe('google', () => { test('completes', async () => { const { main } = require('./index') /// //////////////////////////////////////////// // Set up const tmpdir = path.join( os.tmpdir(), `${Math.ceil(Math.random() * 100000000000)}`, ) // // What we will pass to the functions // const random_string = uuidv4() // const event_body = { // random_string, // } // const event_querystring = `?random_string=${random_string}` // Create javascript files await fsp.mkdir(tmpdir) const code = ` function irrelevant() { return 100 } function jest_systemtest_google(req, resp) { resp.json({}) } module.exports = { jest_systemtest_google } ` const tmpcodepath = path.join(tmpdir, 'index.js') await fsp.writeFile(tmpcodepath, code, { encoding: 'utf-8' }) const arg1 = tmpdir const arg2 = tmpcodepath /// //////////////////////////////////////////// // Run main for Google /// //////////////////////////////////////////// let googleMainRes const googlearg3 = 'google' const googlearg4 = { google: { gc_project: process.env.GC_PROJECT, gc_region: process.env.GC_REGION, }, } let err try { googleMainRes = await main(arg1, arg2, googlearg3, googlearg4) } catch (e) { console.log(e) err = e } // Expect main did not throw expect(err).not.toBeDefined() // // Expect main returned sensible data // expect(googleMainRes).toBeDefined() // expect(googleMainRes.urls).toBeDefined() }, TIMEOUT) }) // test('completes, and echo endpoints return first arg (event) and second arg (http) on GET and POST', async () => { // const { main } = require('./index') // /// //////////////////////////////////////////// // // Set up // const tmpdir = path.join( // os.tmpdir(), // `${Math.ceil(Math.random() * 100000000000)}`, // ) // // // What we will pass to the functions // // const random_string = uuidv4() // // const event_body = { // // random_string, // // } // // const event_querystring = `?random_string=${random_string}` // // Create javascript files // await fsp.mkdir(tmpdir) // const code = ` // function irrelevant() { // return 100 // } // function jest_systemtest_echo(event, http) { // return { event: event, http: http } // } // module.exports = { // jest_systemtest_echo // } // ` // const tmpcodepath = path.join(tmpdir, 'index.js') // await fsp.writeFile(tmpcodepath, code, { encoding: 'utf-8' }) // const arg1 = tmpdir // const arg2 = tmpcodepath // /// //////////////////////////////////////////// // // Run main for Amazon // /// //////////////////////////////////////////// // let amazonMainRes // { // const amazonarg3 = { // amazon: { // aws_access_key_id: process.env.AWS_ACCESS_KEY_ID, // aws_secret_access_key: process.env.AWS_SECRET_ACCESS_KEY, // aws_region: process.env.AWS_REGION, // }, // } // // isPublic // const amazonarg4 = true // let err // try { // amazonMainRes = await main(arg1, arg2, amazonarg3, amazonarg4) // } catch (e) { // console.log(e) // err = e // } // // Expect main did not throw // expect(err).not.toBeDefined() // // Expect main returned sensible data // expect(amazonMainRes).toBeDefined() // expect(amazonMainRes.urls).toBeDefined() // } // /// //////////////////////////////////////////// // // Run main for Google // /// //////////////////////////////////////////// // let googleMainRes // { // const googlearg3 = { // google: { // gc_client_email: '', // gc_private_key: '', // gc_project: '', // }, // } // // to test publishing too // const googlearg4 = true // let err // try { // googleMainRes = await main(arg1, arg2, googlearg3, googlearg4) // } catch (e) { // console.log(e) // err = e // } // // Expect main did not throw // expect(err).not.toBeDefined() // // Expect main returned sensible data // expect(googleMainRes).toBeDefined() // expect(googleMainRes.urls).toBeDefined() // } // /// /////////////////////////////////////////// // // Ping each Amazon URL // // Expect correct result // /// /////////////////////////////////////////// // // Don't test Google ones, they take another 1-2min to be ready // const urls = [].concat(...amazonMainRes.urls) // // TODO ensure in deployGoogle we return only on truly completed // // TODO then, we can start testing them here again // for (let i = 0; i < urls.length; i += 1) { // const url = urls[i] // /// ///////////////// // // POST ///////////// // /// ///////////////// // { // const postres = await fetch(url, { // method: 'POST', // body: JSON.stringify(event_body), // }) // const statusCode = postres.status // const actualResult = await postres.json() // // HTTP Code 2XX // expect(/^2/.test(statusCode)).toBe(true) // // Echoed event // expect(actualResult.event).toEqual(event_body) // // Returned second argument; check if the wrapper formed it properly // expect(actualResult.http).toBeDefined() // expect(actualResult.http.headers).toBeDefined() // expect(actualResult.http.method).toBe('POST') // } // /// ///////////////// // // GET ///////////// // /// ///////////////// // { // const getres = await fetch(`${url}${event_querystring}`, { // method: 'GET', // }) // const statusCode = getres.status // const actualResult = await getres.json() // // HTTP Code 2XX // expect(/^2/.test(statusCode)).toBe(true) // // Echoed event // expect(actualResult.event).toEqual(event_body) // // Returned second argument; check if the wrapper formed it properly // expect(actualResult.http).toBeDefined() // expect(actualResult.http.headers).toBeDefined() // expect(actualResult.http.method).toBe('GET') // } // } // }, TIMEOUT) }) // describe('cli', () => { // // TODO // }) }) ================================================ FILE: initer/index.js ================================================ const fs = require('fs') const path = require('path') const os = require('os') const { EOL } = require('os') const { log, logdev } = require('../printers/index') /** * @description Extracts the [default] section of an AWS .aws/config or .aws/credentials file * @param {string} filecontents File contents of an .aws/credentials or .aws/config file * @returns {string} The string between [default] and the next [...] header, if exists. * Otherwise returns empty string */ function getDefaultSectionString(filecontents) { // Collect all lines below the [default] header ... let defaultSection = filecontents.split(/\[default\]/)[1] if (typeof defaultSection !== 'string' || !defaultSection.trim()) { // default section is non-existent return '' } // ... but above the next [...] header (if any) defaultSection = defaultSection.split(/\[[^\]]*\]/)[0] return defaultSection } // TODO refactor // TODO split up into better functions, for amazon, google inferrer // TODO error handling & meaningful stdout // TODO tests /** * @description Extracts aws_access_key_id and aws_secret_access_key fields from a given .aws/credentials file * @param {string} filecontents File contents of .aws/credentials * @returns { * default: { * aws_access_key_id?: string, * aws_secret_access_key?: string, * aws_region?: string * }} */ function parseAwsCredentialsOrConfigFile(filecontents) { /* filecontents looks something like this: [default] region=us-west-2 output=json [profile user1] region=us-east-1 output=text See: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html */ const fields = { default: {}, } try { const defaultSectionString = getDefaultSectionString(filecontents) if (!defaultSectionString || !defaultSectionString.trim()) { return fields } // we found something const defaultSectionLines = defaultSectionString.split('\n') // TODO split by os-specific newline // Try to extract aws_access_key_id if (defaultSectionLines.some((l) => /aws_access_key_id/.test(l))) { const awsAccessKeyIdLine = defaultSectionLines.filter((l) => /aws_access_key_id/.test(l))[0] let awsAccessKeyId = awsAccessKeyIdLine.split('=')[1] if (typeof awsAccessKeyId === 'string') { // don't crash on weird invalid lines such 'aws_access_key_id=' or 'aws_access_key_id' awsAccessKeyId = awsAccessKeyId.trim() fields.default.aws_access_key_id = awsAccessKeyId } } // Try to extract aws_secret_access_key if (defaultSectionLines.some((l) => /aws_secret_access_key/.test(l))) { const awsSecretAccessKeyLine = defaultSectionLines.filter((l) => /aws_secret_access_key/.test(l))[0] let awsSecretAccessKey = awsSecretAccessKeyLine.split('=')[1] if (typeof awsSecretAccessKey === 'string') { awsSecretAccessKey = awsSecretAccessKey.trim() fields.default.aws_secret_access_key = awsSecretAccessKey } } // Try to extract region if (defaultSectionLines.some((l) => /region/.test(l))) { const regionLine = defaultSectionLines.filter((l) => /region/.test(l))[0] let region = regionLine.split('=')[1] if (typeof region === 'string') { region = region.trim() fields.default.region = region } } return fields // } catch (e) { // console.log(e) // non-critical, just return what we have so far return fields } } /** * Just creates an empty hyperform.json * @param {string} absdir */ function initDumb(absdir, platform) { let json if (platform === 'amazon') { json = { amazon: { aws_access_key_id: '', aws_secret_access_key: '', aws_region: '', }, } } else if (platform === 'google') { json = { google: { gc_project: '', gc_region: '', }, } } else { throw new Error(`platform must be google or amazon but is ${platform}`) } // append 'hyperform.json' to .gitignore // (or create .gitignore if it does not exist yet) fs.appendFileSync( path.join(absdir, '.gitignore'), `${EOL}hyperform.json`, ) // write results to hyperform.json fs.writeFileSync( path.join(absdir, 'hyperform.json'), JSON.stringify(json, null, 2), ) log('✓ Created `hyperform.json` ') // log('✓ Added `hyperform.json` to `.gitignore`') // } // TODO shorten /** * @description Tries to infer AWS credentials and config, and creates a hyperform.json in "absdir" with what it could infer. If hyperform.json already exists in "absdir" it just prints a message. * @param {string} absdir The directory where 'hyperform.json' should be created * @returns {{ * amazon: { * aws_access_key_id: string?, * aws_secret_access_key: string?, * aws_region: string? * } * }} */ function init(absdir) { const hyperformJsonContents = { amazon: { aws_access_key_id: '', aws_secret_access_key: '', aws_region: '', }, google: { gc_project: '', gc_region: '', }, } const filedest = path.join(absdir, 'hyperform.json') if (fs.existsSync(filedest)) { log('hyperform.json exists already.') return } // try to infer AWS credentials // AWS CLI uses this precedence: // (1 - highest precedence) Environment variables AWS_ACCESS_KEY_ID, ... // (2) .aws/credentials and .aws/config // Hence, do the same here // First, start with (2) // Check ~/.aws/credentials and ~/.aws/config const possibleCredentialsPath = path.join(os.homedir(), '.aws', 'credentials') if (fs.existsSync(possibleCredentialsPath) === true) { const credentialsFileContents = fs.readFileSync(possibleCredentialsPath, { encoding: 'utf-8' }) // TODO offer selection to user when there are multiple profiles const parsedCredentials = parseAwsCredentialsOrConfigFile(credentialsFileContents) hyperformJsonContents.amazon.aws_access_key_id = parsedCredentials.default.aws_access_key_id hyperformJsonContents.amazon.aws_secret_access_key = parsedCredentials.default.aws_secret_access_key logdev(`Inferred AWS credentials from ${possibleCredentialsPath}`) } else { logdev(`Could not guess AWS credentials. No AWS credentials file found in ${possibleCredentialsPath}`) } /// ///////////////// /// ///////////////// // try to infer AWS region const possibleConfigPath = path.join(os.homedir(), '.aws', 'config') if (fs.existsSync(possibleConfigPath) === true) { const configFileContents = fs.readFileSync(possibleConfigPath, { encoding: 'utf-8' }) const parsedConfig = parseAwsCredentialsOrConfigFile(configFileContents) hyperformJsonContents.amazon.aws_region = parsedConfig.default.region logdev(`Inferred AWS region from ${possibleConfigPath}`) } else { logdev(`Could not guess AWS region. No AWS config file found in ${possibleConfigPath}`) // TODO region will not be a single region, but smartly multiple ones (or?) } // Then, do (1), possibly overriding values // Check environment variables if (typeof process.env.AWS_ACCESS_KEY_ID === 'string' && process.env.AWS_ACCESS_KEY_ID.trim().length > 0) { hyperformJsonContents.amazon.aws_access_key_id = process.env.AWS_ACCESS_KEY_ID.trim() logdev('Environment variable AWS_ACCESS_KEY_ID set, overriding value from credentials file') } if (typeof process.env.AWS_SECRET_ACCESS_KEY === 'string' && process.env.AWS_SECRET_ACCESS_KEY.trim().length > 0) { hyperformJsonContents.amazon.aws_secret_access_key = process.env.AWS_SECRET_ACCESS_KEY.trim() logdev('Environment variable AWS_SECRET_ACCESS_KEY set, overriding value from credentials file') } if (typeof process.env.AWS_REGION === 'string' && process.env.AWS_REGION.trim().length > 0) { hyperformJsonContents.amazon.aws_region = process.env.AWS_REGION.trim() logdev('Environment variable AWS_REGION set, overriding value from config file') } // append 'hyperform.json' to .gitignore // (or create .gitignore if it does not exist yet) fs.appendFileSync( path.join(absdir, '.gitignore'), `${EOL}hyperform.json`, ) // write results to hyperform.json fs.writeFileSync( path.join(absdir, 'hyperform.json'), JSON.stringify(hyperformJsonContents, null, 2), ) log('✓ Inferred AWS credentials (\'default\' Profile)') // TODO ask for defaults guide through in init log('✓ Created hyperform.json') // TODO ask for defaults guide through in init } module.exports = { init, initDumb, _only_for_testing_getDefaultSectionString: getDefaultSectionString, _only_for_testing_parseAwsCredentialsOrConfigFile: parseAwsCredentialsOrConfigFile, } ================================================ FILE: initer/index.test.js ================================================ describe('initer', () => { describe('getDefaultSectionString', () => { test('returns string on input: empty string', () => { const getDefaultSectionString = require('./index')._only_for_testing_getDefaultSectionString const filecontents = ' ' const res = getDefaultSectionString(filecontents) expect(typeof res).toBe('string') expect(res.trim()).toBe('') }) test('returns empty string on input: [default] header', () => { const getDefaultSectionString = require('./index')._only_for_testing_getDefaultSectionString const filecontents = ` [defaut] ` const res = getDefaultSectionString(filecontents) expect(typeof res).toBe('string') expect(res.trim()).toBe('') }) test('returns empty string on input: other header, other section', () => { const getDefaultSectionString = require('./index')._only_for_testing_getDefaultSectionString const filecontents = ` [some-other-header] first second ` const res = getDefaultSectionString(filecontents) expect(typeof res).toBe('string') expect(res.trim()).toBe('') }) test('returns section on input: [default] header and section', () => { const getDefaultSectionString = require('./index')._only_for_testing_getDefaultSectionString const filecontents = ` [default] first second ` const res = getDefaultSectionString(filecontents) expect(typeof res).toBe('string') const text = res.trim() // 'first\nsecond' expect(/first/.test(text)).toBe(true) expect(/second/.test(text)).toBe(true) }) test('returns section on input: other section 1, [default] header, section, other section 2', () => { const getDefaultSectionString = require('./index')._only_for_testing_getDefaultSectionString const filecontents = ` [some-other-profile-a] sky air [default] first second [some-other-profile-b] clouds ` const res = getDefaultSectionString(filecontents) expect(typeof res).toBe('string') const text = res.trim() // We want all between [default] and next header, but nothing else expect(text).toBe('first\nsecond') }) }) describe('parseAwsCredentialsOrConfigFile', () => { test('returns default credentials on just [default] section present', () => { const parseAwsCredentialsOrConfigFile = require('./index')._only_for_testing_parseAwsCredentialsOrConfigFile const filecontents = ` [default] aws_access_key_id = AKIA2WOM6JAHXXXXXXXX aws_secret_access_key = XXXXXXXXXX+tgppEZPzdN/XXXXlXXXXX/XXXXXXX ` const res = parseAwsCredentialsOrConfigFile(filecontents) expect(res).toBeDefined() expect(res.default).toBeDefined() expect(res.default.aws_access_key_id).toBe('AKIA2WOM6JAHXXXXXXXX') expect(res.default.aws_secret_access_key).toBe('XXXXXXXXXX+tgppEZPzdN/XXXXlXXXXX/XXXXXXX') }) test('returns default credentials on multiple sections present', () => { const parseAwsCredentialsOrConfigFile = require('./index')._only_for_testing_parseAwsCredentialsOrConfigFile // the more weirdly formed const filecontents = ` [some-other-section-a] some-other-section-field-b=1234567890 [default] aws_access_key_id= AKIA2WOM6JAHXXXXXXXX aws_secret_access_key =XXXXXXXXXX+tgppEZPzdN/XXXXlXXXXX/XXXXXXX [some-other-section-b] some-other-section-field-b=098765434567898765 ` const res = parseAwsCredentialsOrConfigFile(filecontents) expect(res).toBeDefined() expect(res.default).toBeDefined() expect(res.default.aws_access_key_id).toBe('AKIA2WOM6JAHXXXXXXXX') expect(res.default.aws_secret_access_key).toBe('XXXXXXXXXX+tgppEZPzdN/XXXXlXXXXX/XXXXXXX') }) }) describe('init', () => { // TODO create mock .aws and see if fields are extracted correctly test('runs, and output has expected structure', async () => { const os = require('os') const uuidv4 = require('uuid').v4 const fs = require('fs') const path = require('path') const { init } = require('./index') // init will write hyperform.json here const absdir = path.join(os.tmpdir(), uuidv4()) fs.mkdirSync(absdir) let err try { init(absdir) } catch (e) { console.log(e) err = e } // it didn't throw expect(err).not.toBeDefined() // it wrote hyperform.json const hyperformJsonPath = path.join(absdir, 'hyperform.json') expect(fs.existsSync(hyperformJsonPath)).toBe(true) // hyperform.json has the expected structure let hyperformJson = fs.readFileSync(hyperformJsonPath) hyperformJson = JSON.parse(hyperformJson) expect(hyperformJson.amazon).toBeDefined() expect(hyperformJson.amazon.aws_access_key_id).toBeDefined() expect(hyperformJson.amazon.aws_secret_access_key).toBeDefined() expect(hyperformJson.amazon.aws_region).toBeDefined() }) }) }) ================================================ FILE: kindler/index.js ================================================ /* eslint-disable arrow-body-style */ const path = require('path') /** * Appends code to an index.js ("code") in "dir" that imports * and immediately exports given "infos". Has no side effects. * Needed for Google that only looks into index.js * @param {string} code * @param {string} dir * @param {[ {p: string, exps: string[] }]} infos For instance [ { p: '/home/qng/dir/somefile.js', exps: [ 'endpoint_hello' ] } ] * */ function kindle(code, dir, infos) { const kindleAppendix = ` ;module.exports = { ${ // for each file infos.map(({ p, exps }) => { // for each endpoint export return exps.map((exp) => { const relPath = path.relative(dir, p) // it's exported from index.js, whose source code this will be (ie above) if (relPath === 'index.js') { return `${exp}: module.exports.${exp} || exports.${exp},` } else { // it's exported from other file return `${exp}: require('./${relPath}').${exp},` } }) .join('\n') }) .join('\n') } }; ` const kindledCode = ` ${code} ${kindleAppendix} ` return kindledCode } module.exports = { kindle, } ================================================ FILE: kindler/index.tes.js ================================================ // TODO ================================================ FILE: meta/index.js ================================================ /** * @description Detects whether jest is running this code * @returns {boolean} */ function isInTesting() { if (process.env.JEST_WORKER_ID != null) { return true } if (process.env.NODE_ENV === 'test') { return true } return false } module.exports = { isInTesting, } ================================================ FILE: package.json ================================================ { "name": "hyperform-cli", "version": "0.6.12", "description": "", "main": "index.js", "bin": { "hf": "./cli.js" }, "scripts": { "test": "jest --runInBand --setupFiles dotenv/config --coverage", "lint": "eslint", "lintfix": "eslint --fix" }, "engines": { "node": ">=10.10.0" }, "jest": { "testEnvironment": "node" }, "engineStrict": true, "author": "", "license": "Apache 2.0", "dependencies": { "@google-cloud/functions": "^1.1.2", "aws-sdk": "^2.820.0", "chalk": "^4.1.0", "clipboardy": "^2.3.0", "dotenv": "^8.2.0", "findit": "^2.0.0", "joi": "^17.3.0", "ncp": "^2.0.0", "node-fetch": "^2.6.1", "semver": "^7.3.5", "spinnies": "^0.5.1", "uuid": "^8.3.2", "webpack": "^5.4.0", "yazl": "^2.5.1", "zip-dir": "^2.0.0" }, "devDependencies": { "eslint": "^7.15.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-plugin-import": "^2.22.1", "husky": "^4.3.6", "jest": "25.0.0", "webpack-cli": "^4.2.0" } } ================================================ FILE: parser/index.js ================================================ // validates and parses hyperform.json const path = require('path') const { amazonSchema, googleSchema } = require('../schemas/index') let parsedHyperformJson /** * @description Parses v,alidates, and returns contents of "dir"/hyperform.json * @param {string} dir Directory where to look for hyperform.json * @param {string} platform Whether to expect 'amazon' or 'google' content * */ function getParsedHyperformJson(dir, platform) { if (parsedHyperformJson == null) { const json = require(path.join(dir, 'hyperform.json')) // validate its schema let schema if (platform === 'amazon') schema = amazonSchema if (platform === 'google') schema = googleSchema // throws if platform is not 'amazon' or 'google' const { error, value } = schema.validate(json) if (error) { throw new Error(`${error} ${value}`) } parsedHyperformJson = json } return parsedHyperformJson } module.exports = { getParsedHyperformJson, } ================================================ FILE: printers/index.js ================================================ const Spinnies = require('spinnies') const { isInTesting } = require('../meta/index') const spinner = { interval: 80, frames: [ '⠁', '⠂', '⠄', '⡀', '⢀', '⠠', '⠐', '⠈', ], } const spinnies = new Spinnies({ color: 'white', succeedColor: 'white', spinner: spinner }); const { log } = console let logdev // Don't show dev-level logging // (Comment out to show dev-level logging) logdev = () => { } // Don't show timings // (Comment out to see timings) console.time = () => { } console.timeEnd = () => { } // In testing, be silent but console.log successes and fails if (isInTesting() === true) { spinnies.add = () => { } spinnies.update = () => { } spinnies.remove = () => { } spinnies.succeed = (_, { text }) => console.log(text) spinnies.fail = (_, { text }) => console.log(text) spinnies.updateSpinnerState = () => {} } spinnies.f = spinnies.fail spinnies.succ = spinnies.succeed module.exports = { spinnies, log, logdev, } ================================================ FILE: publisher/amazon/index.js ================================================ const { createApi, createIntegration, createDefaultAutodeployStage, setRoute, allowApiGatewayToInvokeLambda, getApiDetails, } = require('./utils') // TODO handle regional / edge / read up on how edge works const HFAPINAME = 'hyperform-v1' /** * @description Creates a public HTTP endpoint that forwards request to a given Lambda. * @param {string} lambdaArn * @param {string} region * @returns {Promise} Full endpoint URL, eg. https://48ytz1e6f3.execute-api.us-east-2.amazonaws.com/endpoint-hello */ async function publishAmazon(lambdaArn, region) { // console.log('received lambdaar', lambdaArn) const lambdaName = lambdaArn.split(':').slice(-1)[0] // Lambda 'endpoint-hello' should be at 'https://.../endpoint-hello' const routePath = `/${lambdaName}` /// /////////////////////////////////////////////// /// //ensure HF API exists in that region ///////// // TODO to edge /// ////////////////////////////////////////////// let hfApiId let hfApiUrl /// /////////////////////////////////////////////// /// Check if HF umbrella API exists in that region /// ////////////////////////////////////////////// const apiDetails = await getApiDetails(HFAPINAME, region) // exists // use it if (apiDetails != null && apiDetails.apiId != null) { hfApiId = apiDetails.apiId hfApiUrl = apiDetails.apiUrl // does not exist // create HF API } else { const createRes = await createApi(HFAPINAME, region) hfApiId = createRes.apiId hfApiUrl = createRes.apiUrl } /// /////////////////////////////////////////////// /// Add permission to API to lambda accessed by API gateway /// ////////////////////////////////////////////// // todo iwann spezifisch der api access der lambda erlauben via SourceArn await allowApiGatewayToInvokeLambda(lambdaName, region) /// /////////////////////////////////////////////// /// Create integration that represents the Lambda /// ////////////////////////////////////////////// const integrationId = await createIntegration(hfApiId, region, lambdaArn) /// /////////////////////////////////////////////// /// Create $default Auto-Deploy stage /// ////////////////////////////////////////////// try { await createDefaultAutodeployStage(hfApiId, region) } catch (e) { // already exists (shouldn't throw because of anything other) // nice } /// /////////////////////////////////////////////// /// Create / update route with that integration /// ////////////////////////////////////////////// await setRoute( hfApiId, region, routePath, integrationId, ) const endpointUrl = hfApiUrl + routePath return endpointUrl } module.exports = { publishAmazon, } ================================================ FILE: publisher/amazon/utils.js ================================================ const AWS = require('aws-sdk') const { log, logdev } = require('../../printers/index') const conf = { accessKeyId: process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, region: process.env.AWS_REGION, // may, may not be defined // sessionToken: process.env.AWS_SESSION_TOKEN || undefined, } if (process.env.AWS_SESSION_TOKEN != null && process.env.AWS_SESSION_TOKEN !== 'undefined') { conf.sessionToken = process.env.AWS_SESSION_TOKEN } AWS.config.update(conf) /** * @description Creates a new REGIONAL API in "region" named "apiName" * @param {string} apiName Name of API * @param {string} apiRegion * @returns {Promise<{apiId: string, apiUrl: string}>} Id and URL of the endpoint */ async function createApi(apiName, apiRegion) { const apigatewayv2 = new AWS.ApiGatewayV2({ apiVersion: '2018-11-29', region: apiRegion, }) // @see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/ApiGatewayV2.html#createApi-property const createApiParams = { Name: apiName, ProtocolType: 'HTTP', // TODO regional is default, but how the to set to EDGE later on? // EndpointType: 'REGIONAL', // invalid field // Target: targetlambdaArn, // TODO CorsConfiguration: { AllowMethods: [ 'POST', 'GET', ], AllowOrigins: [ '*', ], }, } const createApiRes = await apigatewayv2.createApi(createApiParams).promise() const res = { apiId: createApiRes.ApiId, apiUrl: createApiRes.ApiEndpoint, } return res } /** * * @param {string} apiId * @param {string} apiRegion * @returns {Promise} */ async function deleteApi(apiId, apiRegion) { const apigatewayv2 = new AWS.ApiGatewayV2({ apiVersion: '2018-11-29', region: apiRegion, }) const deleteApiParams = { ApiId: apiId, } await apigatewayv2.deleteApi(deleteApiParams).promise() } /** * @description Returns the IntegrationId of the integration that matches 'name', or null. * If multiple exist, it returns the IntegrationId of the first one in the list. * @param {*} apiId * @param {*} apiRegion * @param {*} name */ async function getIntegrationId(apiId, apiRegion, name) { // On amazon, integration names are not unique, // but HF treats the as unique // and always reuses them const apigatewayv2 = new AWS.ApiGatewayV2({ apiVersion: '2018-11-29', region: apiRegion, }) const getParams = { ApiId: apiId, MaxResults: '9999', } // Get all integrations let res = await apigatewayv2.getIntegrations(getParams).promise() res = res.Items // Only integrations that match name .filter((el) => el.IntegrationUri.split(':')[-1] === name) // Just take the first one res = res[0] if (res && res.IntegrationId) { return res.IntegrationId } else { return null } } /** * @description Gets route Ids of GET and POST methods that match routePath * @param {*} apiId * @param {*} apiRegion * @param {*} routePath * @returns {[ { AuthorizationType: string, RouteId: string, RouteKey: string }]} */ async function getGETPOSTRoutesAt(apiId, apiRegion, routePath) { const apigatewayv2 = new AWS.ApiGatewayV2({ apiVersion: '2018-11-29', region: apiRegion, }) // TODO Amazon might return a paginated response here (?) // In that case with many routes, the route we look for may not be on first page const params = { ApiId: apiId, MaxResults: '9999', // string according to docs and it works... uuh? } const res = await apigatewayv2.getRoutes(params).promise() const matchingRoutes = res.Items .filter((item) => item.RouteKey && item.RouteKey.includes(routePath) === true) // only GET and POST ones .filter((item) => /GET|POST/.test(item.RouteKey) === true) return matchingRoutes } /** * Creates or update GET and POST routes with an integration. * If only one of GET or POST routes exist (user likely deleted one of them), * it updates that one. * Otherwise, it creates new both GET, POST routes. * Use createDefaultAutodeployStage too when creating, so that changes are made public * @param {*} apiId * @param {*} apiRegion * @param {*} routePath '/endpoint-1' for example * @param {*} integrationId */ async function setRoute(apiId, apiRegion, routePath, integrationId) { const apigatewayv2 = new AWS.ApiGatewayV2({ apiVersion: '2018-11-29', region: apiRegion, }) // Get route ids of GET & POST at that routePath const routes = await getGETPOSTRoutesAt(apiId, apiRegion, routePath) if (routes.length > 0) { /// //////////////////////////////////////////////// // Update routes (GET, POST) with integrationId /// //////////////////////////////////////////////// await Promise.all( routes.map(async (r) => { // skip if integration id is set correctly already if (r.Target && r.Target === `integrations/${integrationId}`) { return } const updateRouteParams = { ApiId: apiId, AuthorizationType: 'NONE', RouteId: r.RouteId, RouteKey: r.RouteKey, Target: `integrations/${integrationId}`, } try { const updateRes = await apigatewayv2.updateRoute(updateRouteParams).promise() } catch (e) { // This happens eg when there is only one of GET OR POST route (routes.length > 0) // Usually when the user deliberately deleted one of the // Ignore, as it's likely intented } }), ) } else { /// //////////////////////////////////////////////// // Create routes (GET, POST) with integrationId /// //////////////////////////////////////////////// // Create GET route const createGETRouteParams = { ApiId: apiId, AuthorizationType: 'NONE', RouteKey: `GET ${routePath}`, Target: `integrations/${integrationId}`, } const createGETRes = await apigatewayv2.createRoute(createGETRouteParams).promise() // Create POST route const createPOSTRouteParams = { ...createGETRouteParams } createPOSTRouteParams.RouteKey = `POST ${routePath}` const createPOSTRes = await apigatewayv2.createRoute(createPOSTRouteParams).promise() } } /** * Creates a (possibly duplicate) integration that can be attached. * Before creating, check getIntegrationId to avoid duplicates * @param {*} apiId * @param {*} apiRegion * @param {string} targetLambdaArn For instance a Lambda ARN * @returns {string} IntegrationId */ async function createIntegration(apiId, apiRegion, targetLambdaArn) { const apigatewayv2 = new AWS.ApiGatewayV2({ apiVersion: '2018-11-29', region: apiRegion, }) const params = { ApiId: apiId, IntegrationType: 'AWS_PROXY', IntegrationUri: targetLambdaArn, PayloadFormatVersion: '2.0', } const res = await apigatewayv2.createIntegration(params).promise() const integrationId = res.IntegrationId return integrationId } /** * @description Returns ApiId and ApiEndpoint of a regional API gateway API * with the name "apiName", in "apiRegion". * If multiple APIs exist with that name, it warns, and uses the first one in the received list. * If none exist, it returns null. * @param {string} apiName * @param {string} apiRegion * @returns {Promise<{apiId: string, apiUrl: string}>} Details of the API, or null */ async function getApiDetails(apiName, apiRegion) { // Check if API with that name exists // Follows Hyperform conv: same name implies identical, for lambdas, and api endpoints etc const apigatewayv2 = new AWS.ApiGatewayV2({ apiVersion: '2018-11-29', region: apiRegion, }) const getApisParams = { MaxResults: '9999', } const res = await apigatewayv2.getApis(getApisParams).promise() const matchingApis = res.Items.filter((item) => item.Name === apiName) if (matchingApis.length === 0) { // none exist return null } if (matchingApis.length >= 2) { log(`Multiple (${matchingApis.length}) APIs found with same name ${apiName}. Using first one`) } // just take first one // Hyperform convention is there's only one with any given name const apiDetails = { apiId: matchingApis[0].ApiId, apiUrl: matchingApis[0].ApiEndpoint, } return apiDetails } /** * * @param {*} apiId * @param {*} apiRegion * @throws If $default stage already exists */ async function createDefaultAutodeployStage(apiId, apiRegion) { const apigatewayv2 = new AWS.ApiGatewayV2({ apiVersion: '2018-11-29', region: apiRegion, }) const params = { ApiId: apiId, StageName: '$default', AutoDeploy: true, } const res = await apigatewayv2.createStage(params).promise() } /** * @description Add permssion to allow API gateway to invoke given Lambda * @param {string} lambdaName Name of Lambda * @param {string} region Region of Lambda * @returns {Promise} */ async function allowApiGatewayToInvokeLambda(lambdaName, region) { const lambda = new AWS.Lambda({ region: region, apiVersion: '2015-03-31', }) const addPermissionParams = { Action: 'lambda:InvokeFunction', FunctionName: lambdaName, Principal: 'apigateway.amazonaws.com', // TODO SourceArn https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#addPermission-property StatementId: `hf-stmnt-${lambdaName}`, } try { await lambda.addPermission(addPermissionParams).promise() } catch (e) { if (e.code === 'ResourceConflictException') { // API Gateway can already access that lambda (happens on all subsequent deploys), cool } else { logdev(`addpermission: some other error: ${e}`) throw e } } } module.exports = { createApi, _only_for_testing_deleteApi: deleteApi, allowApiGatewayToInvokeLambda, getApiDetails, createIntegration, setRoute, createDefaultAutodeployStage, } ================================================ FILE: publisher/amazon/utils.test.js ================================================ /* eslint-disable global-require */ const APIREGION = 'us-east-2' // other functions in publisher/amazon are pretty well covered by other tests describe('utils', () => { describe('createApi', () => { test('completes and returns apiId and apiUrl that is an URL', async () => { const uuidv4 = require('uuid').v4 const { createApi } = require('./utils') const deleteApi = require('./utils')._only_for_testing_deleteApi const apiName = `jest-reserved-api-${uuidv4()}` let err let res try { res = await createApi(apiName, APIREGION) } catch (e) { console.log(e) err = e } // createApi did not throw expect(err).not.toBeDefined() expect(res).toBeDefined() expect(typeof res.apiUrl).toBe('string') expect(typeof res.apiId).toBe('string') // apiUrl is an URL const tryUrl = () => new URL(res.apiUrl) expect(tryUrl).not.toThrow() /// ////////////////////// // Clean up: Delete API await deleteApi(res.apiId, APIREGION) }, 10000) }) describe('setRoute', () => { test('completes on non-existing routes, existing routes', async () => { /// Create API const uuidv4 = require('uuid').v4 const fetch = require('node-fetch') const { createApi, createIntegration, setRoute, createDefaultAutodeployStage, } = require('./utils') const deleteApi = require('./utils')._only_for_testing_deleteApi const apiName = `jest-reserved-api-${uuidv4()}` const routePath = '/endpoint_test' const { apiId, apiUrl } = await createApi(apiName, APIREGION) // Create Integration to some lambda // we won't call it, so doesn't matter really which lambda const targetLambdaArn = 'arn:aws:lambda:us-east-2:735406098573:function:endpoint_hello' const integrationId = await createIntegration(apiId, APIREGION, targetLambdaArn) // Always make changes public (importantly, things we will do with setRoute) await createDefaultAutodeployStage(apiId, APIREGION) let res let err try { res = await setRoute( apiId, APIREGION, routePath, integrationId, ) } catch (e) { err = e } // setRoute did not throw expect(err).not.toBeDefined() const fullurl = `${apiUrl}${routePath}` console.log(fullurl) // URL returns 200 // On GET { const getres = await fetch(fullurl, { method: 'GET', }) const statusCode = `${getres.status}` console.log(statusCode) // GET route returns 2XX expect(/^2/.test(statusCode)).toBe(true) } // On POST { const getres = await fetch(fullurl, { method: 'POST', }) const statusCode = `${getres.status}` console.log(statusCode) // POST route returns 2XX expect(/^2/.test(statusCode)).toBe(true) } await deleteApi(apiId, APIREGION) }, 10000) }) }) ================================================ FILE: response-collector/.gitignore ================================================ hyperform.json ================================================ FILE: response-collector/index.js ================================================ const aws = require('aws-sdk') const uuidv4 = require('uuid').v4 /** * @description This serverless function gathers responses sent * by users answering $ hyperform survey */ async function collectSurveyResponse(event) { const s3 = new aws.S3() const filename = `${new Date().toDateString()}:${uuidv4()}.json` if (event == null) return const putParams = { Bucket: 'hyperform-survey-responses', Key: `cli-responses/${filename}`, Body: JSON.stringify(event, null, 2), } await s3.putObject(putParams).promise() } function getSurveyQuestion() { return { text: 'Some survey text', postUrl: 'https://mj9jbzlpxi.execute-api.us-east-2.amazonaws.com', } } module.exports = { endpoint_getSurveyQuestion: getSurveyQuestion, endpoint_collectSurveyResponse: collectSurveyResponse, } ================================================ FILE: schemas/index.js ================================================ const joi = require('joi') const amazonSchema = joi.object({ amazon: joi.object({ aws_access_key_id: joi.string().required(), aws_secret_access_key: joi.string().required(), aws_region: joi.string().required(), // allow if user enters it aws_session_token: joi.string().allow(''), }).required(), }) const googleSchema = joi.object({ google: joi.object({ gc_project: joi.string().required(), gc_region: joi.string().required(), }).required(), }) module.exports = { amazonSchema, googleSchema, } ================================================ FILE: schemas/index.test.js ================================================ /* eslint-disable global-require */ describe('schemas', () => { describe('amazon schema', () => { test('normal case', () => { const { amazonSchema } = require('./index') const input = { amazon: { aws_access_key_id: 'xx', aws_secret_access_key: 'xx', aws_region: 'xx', }, } const { error } = amazonSchema.validate(input) expect(error).not.toBeDefined() }) test('allows aws_session_token field', () => { const { amazonSchema } = require('./index') const input = { amazon: { aws_access_key_id: 'xx', aws_secret_access_key: 'xx', aws_region: 'xx', aws_session_token: 'xx', }, } const { error } = amazonSchema.validate(input) expect(error).not.toBeDefined() }) test('does not allow missing field in amazon', () => { const { amazonSchema } = require('./index') const input = { amazon: { aws_access_key_id: 'xx', aws_secret_access_key: 'xx', // Missing: aws_region: '', }, } const { error } = amazonSchema.validate(input) expect(error).toBeDefined() }) test('does not allow both providers', () => { const { amazonSchema } = require('./index') const input = { amazon: { aws_access_key_id: 'abc', aws_secret_access_key: 'abc', aws_region: 'abc', }, google: { gc_project: 'abc', gc_region: 'abc', }, } const { error } = amazonSchema.validate(input) expect(error).toBeDefined() }) test('does not allow no provider', () => { const { amazonSchema } = require('./index') const input = { // empty } const { error } = amazonSchema.validate(input) expect(error).toBeDefined() }) }) describe('google schema', () => { test('normal case', () => { const { googleSchema } = require('./index') const input = { google: { gc_project: 'abc', gc_region: 'abc', }, } const { error } = googleSchema.validate(input) expect(error).not.toBeDefined() }) }) }) ================================================ FILE: surveyor/index.js ================================================ const fetch = require('node-fetch') // 0 to 1, with which to show survey const probability = 0.04 const getSurveyUrl = 'https://era1vrrco0.execute-api.us-east-2.amazonaws.com' const postSurveyAnswerUrl = 'https://mj9jbzlpxi.execute-api.us-east-2.amazonaws.com' function maybeShowSurvey() { if ((new Date().getSeconds() / 60) < probability) { fetch(getSurveyUrl) .then((res) => res.json()) .then((res) => console.log(` ${res.text} You can type $ hf answer ... to answer :) `)) } } async function answerSurvey(text) { const currentSurvey = await fetch(getSurveyUrl) .then((res) => res.json()) await fetch(postSurveyAnswerUrl, { method: 'POST', body: JSON.stringify({ currentSurvey: currentSurvey, answer: text, date: new Date(), }), }) } module.exports = { maybeShowSurvey, answerSurvey, } ================================================ FILE: template/.eslintrc.json ================================================ { "env": { "commonjs": true, "es2021": true, "node": true, "jest": true }, "extends": [ "airbnb-base" ], "parserOptions": { "ecmaVersion": 12 }, "rules": { "no-console": "off", "object-shorthand": "off", "no-restricted-syntax": "warn", "prefer-destructuring": "warn" } } ================================================ FILE: template/index.js ================================================ /** * @description The module appendix template. Never import this, * only copy-paste from here to transpiler/index.js * @param {*} moduleexp * @param {*} [exp] */ module.exports = () => { // START PASTE /** * Start Hyperform wrapper * It provides some simple usability features * Amazon: * Google: * - Send pre-flight headers * - console.error on error */ global.alreadyWrappedNames = []; function wrapExs(me, platform) { const newmoduleexports = { ...me }; const expkeys = Object.keys(me); for (let i = 0; i < expkeys.length; i += 1) { const expkey = expkeys[i]; const userfunc = newmoduleexports[expkey]; // it should be idempotent // TODO fix code so this doesn't happen if (global.alreadyWrappedNames.includes(expkey)) { continue; } global.alreadyWrappedNames.push(expkey); let wrappedfunc; if (platform === 'amazon') { wrappedfunc = async function handler(event, context, callback) { /// //////////////////////////////// // Invoke user function /////// /// //////////////////////////////// const res = await userfunc(event, context, callback); context.succeed(res); // throwing will call context.fail automatically }; } if (platform === 'google') { wrappedfunc = async function handler(req, resp, ...rest) { // allow to be called from anywhere (also localhost) // resp.header('Content-Type', 'application/json'); resp.header('Access-Control-Allow-Origin', '*'); resp.header('Access-Control-Allow-Headers', '*'); resp.header('Access-Control-Allow-Methods', '*'); resp.header('Access-Control-Max-Age', 30); // respond to CORS preflight requests if (req.method === 'OPTIONS') { resp.status(204).send(''); } else { // Invoke user function // (user must .json or .send himself) try { await userfunc(req, resp, ...rest); } catch (e) { console.error(e); resp.status(500).send(''); // TODO generate URL to logs (similar to GH) } } }; } newmoduleexports[expkey] = wrappedfunc; } return newmoduleexports; } const curr = { ...exports, ...module.exports }; const isInAmazon = !!(process.env.LAMBDA_TASK_ROOT || process.env.AWS_EXECUTION_ENV); const isInGoogle = (/google/.test(process.env._) === true); if (isInAmazon === true) { return wrapExs(curr, 'amazon'); } if (isInGoogle === true) { return wrapExs(curr, 'google'); } return curr; // Export unchanged (local, fallback) // END PASTE }; ================================================ FILE: transpiler/index.js ================================================ /** * @description Transpiles Javascript code so that its exported functions can run on Amazon, Google * @param {string} bundleCode Bundled Javascript code. * Output of the 'bundle' function in bundle/index.js * @returns {string} The code, transpiled for providers */ function transpile(bundleCode) { const appendix = ` ;module.exports = (() => { // START PASTE /** * Start Hyperform wrapper * It provides some simple usability features * Amazon: * Google: * - Send pre-flight headers * - console.error on error */ global.alreadyWrappedNames = []; function wrapExs(me, platform) { const newmoduleexports = { ...me }; const expkeys = Object.keys(me); for (let i = 0; i < expkeys.length; i += 1) { const expkey = expkeys[i]; const userfunc = newmoduleexports[expkey]; // it should be idempotent // TODO fix code so this doesn't happen if (global.alreadyWrappedNames.includes(expkey)) { continue; } global.alreadyWrappedNames.push(expkey); let wrappedfunc; if (platform === 'amazon') { wrappedfunc = async function handler(event, context, callback) { /// //////////////////////////////// // Invoke user function /////// /// //////////////////////////////// const res = await userfunc(event, context, callback); context.succeed(res); // throwing will call context.fail automatically }; } if (platform === 'google') { wrappedfunc = async function handler(req, resp, ...rest) { // allow to be called from anywhere (also localhost) // resp.header('Content-Type', 'application/json'); resp.header('Access-Control-Allow-Origin', '*'); resp.header('Access-Control-Allow-Headers', '*'); resp.header('Access-Control-Allow-Methods', '*'); resp.header('Access-Control-Max-Age', 30); // respond to CORS preflight requests if (req.method === 'OPTIONS') { resp.status(204).send(''); } else { // Invoke user function // (user must .json or .send himself) try { await userfunc(req, resp, ...rest); } catch (e) { console.error(e); resp.status(500).send(''); // TODO generate URL to logs (similar to GH) } } }; } newmoduleexports[expkey] = wrappedfunc; } return newmoduleexports; } const curr = { ...exports, ...module.exports }; const isInAmazon = !!(process.env.LAMBDA_TASK_ROOT || process.env.AWS_EXECUTION_ENV); const isInGoogle = (/google/.test(process.env._) === true); if (isInAmazon === true) { return wrapExs(curr, 'amazon'); } if (isInGoogle === true) { return wrapExs(curr, 'google'); } return curr; // Export unchanged (local, fallback) })(); ` const transpiledCode = bundleCode + appendix return transpiledCode } module.exports = { transpile, } ================================================ FILE: uploader/amazon/index.js ================================================ // const AWS = require('aws-sdk') // /** // * // * @param {*} localpath // * @param {*} bucket // * @param {*} key // */ // async function uploadAmazon(localpath, bucket, key) { // const s3 = new AWS.S3() // const fsp = require('fs').promises // const contents = await fsp.readFile(localpath) // const uploadParams = { // Bucket: bucket, // Key: key, // Body: contents, // } // await s3.upload(uploadParams).promise() // const s3path = `s3://${bucket}/${key}` // return s3path // } // module.exports = { // uploadAmazon, // } ================================================ FILE: uploader/amazon/index.test.js ================================================ /* eslint-disable global-require */ const S3BUCKET = 'jak-bridge-typical' describe('uploader', () => { describe('amazon', () => { describe('uploadAmazon', () => { test('uploads simple text file, subsequent get returns that file', async () => { // const AWS = require('aws-sdk') // const s3 = new AWS.S3() // const uuidv4 = require('uuid').v4 // const { uploadAmazon } = require('./index') // const key = `${uuidv4()}` // const filecontents = key // const filecontentsbuffer = Buffer.from(key) // let res // let err // try { // res = await uploadAmazon(filecontentsbuffer, S3BUCKET, key) // } catch (e) { // console.log(e) // err = e // } // // it didn't throw // expect(err).not.toBeDefined() // // it returned s3:// ... // expect(typeof res).toBe('string') // expect(res).toBe(`s3://${S3BUCKET}/${key}`) // // getting file immediately after must have same contents // const getParams = { // Bucket: S3BUCKET, // Key: key, // } // const getRes = await s3.getObject(getParams).promise() // const gottenFilecontents = getRes.Body.toString('utf-8') // expect(gottenFilecontents).toBe(filecontents) }) }) }) }) ================================================ FILE: uploader/google/index.js ================================================ // const { Storage } = require('@google-cloud/storage') // const gcloudstorage = new Storage() // async function uploadGoogle(localpath, bucket, key) { // const res = await gcloudstorage.bucket(bucket).upload(localpath, { // gzip: true, // destination: key, // metadata: { // // Docs: (If the contents will change, use cacheControl: 'no-cache') // // @see https://github.com/googleapis/nodejs-storage/blob/master/samples/uploadFile.js // cacheControl: 'no-cache', // }, // }) // const gsPath = `gs://${bucket}/${key}` // return gsPath // } // module.exports = { // uploadGoogle, // } ================================================ FILE: zipper/google/index.js ================================================ const _zipdir = require('zip-dir') const fsp = require('fs').promises const path = require('path') const os = require('os') /** * * @param {string} dir * @param {string[]} except names of directories or files that will not be included * (usually ["node_modules", ".git", ".github"]) Uses substring check. * @returns {string} outpath of the zip * */ async function zipDir(dir, except) { // create tmp dir const outdir = await fsp.mkdtemp(path.join(os.tmpdir(), 'zipDir-zipped-')) const outpath = path.join(outdir, 'deploypackage.zip') // The second argument of https://www.npmjs.com/package/zip-dir // Function that is called on every file / dir to determine if it'll be included in the zip const filterFunc = (p, stat) => { for (let i = 0; i < except.length; i += 1) { if (p.includes(except[i])) { console.log(`Excluding ${p}`) return false } } return true } const res = await new Promise((resolve, rej) => { _zipdir(dir, { saveTo: outpath, filter: filterFunc, }, (err, r) => { if (err) { rej(err) } else { resolve(outpath) // resolve with the outpath } }) }) return res } module.exports = { zipDir, } ================================================ FILE: zipper/index.js ================================================ const fsp = require('fs').promises const fs = require('fs') const os = require('os') const path = require('path') const { Readable } = require('stream') const yazl = require('yazl') /** * @description Creates a .zip that contains given filecontents, within given filenames. All at the zip root * @param {{}} filesobj For instance { 'file.txt': 'abc' } * @returns {Promise} Path to the created .zip */ async function zip(filesobj) { const uid = `${Math.ceil(Math.random() * 10000)}` const zipfile = new yazl.ZipFile() console.time(`zip-${uid}`) // create tmp dir const outdir = await fsp.mkdtemp(path.join(os.tmpdir(), 'zipped-')) const outpath = path.join(outdir, 'deploypackage.zip') zipfile.outputStream.pipe(fs.createWriteStream(outpath)) // filesobj is like { 'file.txt': 'abc', 'file2.txt': '123' } // for each such destination file,... const fnames = Object.keys(filesobj) for (let i = 0; i < fnames.length; i += 1) { const fname = fnames[i] const fcontent = filesobj[fname] // set up stream const s = new Readable(); s._read = () => {}; s.push(fcontent); s.push(null); // In zip, set last-modified header to 01-01-2020 // this way, rezipping identical files is deterministic (gives the same codesha256) // that way we can skip uploading zips that haven't changed const options = { mtime: new Date(1577836), } zipfile.addReadStream(s, fname, options); // place code in index.js inside zip // console.log(`created ${fname} in zip`) } zipfile.end() console.timeEnd(`zip-${uid}`) return outpath } module.exports = { zip, } ================================================ FILE: zipper/index.test.js ================================================ describe('zipper', () => { test('completes with multiple files and returns valid path', async () => { const path = require('path') const fs = require('fs') const { zip } = require('./index') const inp = { 'fileWithinZip.txt': 'abc', } let err let res try { res = await zip(inp) } catch (e) { console.log(e) err = e } expect(err).not.toBeDefined() // expect it's a path // https://stackoverflow.com/a/38974272 expect(res === path.basename(res)).toBe(false) // expect it ends with '.zip' expect(path.extname(res)).toBe('.zip') // expect we can get details about it let statErr try { fs.statSync(res) } catch (e) { statErr = e } expect(statErr).not.toBeDefined() }) })