[
{
"path": ".dockerignore",
"content": "npm-debug.log\nyarn.lock\n_build\n!_build/dev/lib/castore\ndeps\n!deps/castore\nnode_modules\ndist\n.gitignore"
},
{
"path": ".formatter.exs",
"content": "# Used by \"mix format\"\n[\n inputs: [\"{mix,.formatter}.exs\", \"{config,lib,test}/**/*.{ex,exs}\"]\n]\n"
},
{
"path": ".github/workflows/tests.yml",
"content": "name: Run tests\nconcurrency: ci_tests\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ] \n\njobs:\n build:\n name: Build and test\n runs-on: ubuntu-latest\n timeout-minutes: 5\n\n steps:\n - uses: actions/checkout@v3\n with:\n fetch-depth: 0\n # cache the ASDF directory, using the values from .tool-versions\n - name: ASDF cache\n uses: actions/cache@v3\n with:\n path: ~/.asdf\n key: ${{ runner.os }}-asdf-v2-${{ hashFiles('.tool-versions') }}\n id: asdf-cache\n # only run `asdf install` if we didn't hit the cache\n - uses: asdf-vm/actions/install@v1\n if: steps.asdf-cache.outputs.cache-hit != 'true'\n # if we did hit the cache, set up the environment\n - name: Setup ASDF environment\n run: |\n echo \"ASDF_DIR=$HOME/.asdf\" >> $GITHUB_ENV\n echo \"ASDF_DATA_DIR=$HOME/.asdf\" >> $GITHUB_ENV\n if: steps.asdf-cache.outputs.cache-hit == 'true'\n - name: Reshim ASDF\n run: |\n echo \"$ASDF_DIR/bin\" >> $GITHUB_PATH\n echo \"$ASDF_DIR/shims\" >> $GITHUB_PATH\n $ASDF_DIR/bin/asdf reshim\n - name: Restore dependencies cache\n uses: actions/cache@v3\n with:\n path: deps\n key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}\n restore-keys: ${{ runner.os }}-mix-\n - name: Cache compiled build\n id: cache-build\n uses: actions/cache@v3\n env:\n cache-name: cache-compiled-build\n with:\n path: _build\n key: ${{ runner.os }}-mix-${{ env.cache-name }}-${{ hashFiles('**/mix.lock') }}\n restore-keys: |\n ${{ runner.os }}-mix-${{ env.cache-name }}-\n ${{ runner.os }}-mix-\n - name: Install dependencies\n run: |\n mix local.rebar --force\n mix local.hex --force\n mix deps.get\n - name: Compile\n run: mix compile\n - name: Run tests\n run: mix test\n"
},
{
"path": ".gitignore",
"content": "# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\nlerna-debug.log*\n\n# Diagnostic reports (https://nodejs.org/api/report.html)\nreport.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json\n\n# Runtime data\npids\n*.pid\n*.seed\n*.pid.lock\n\n# Directory for instrumented libs generated by jscoverage/JSCover\nlib-cov\n\n# Coverage directory used by tools like istanbul\ncoverage\n*.lcov\n\n# nyc test coverage\n.nyc_output\n\n# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)\n.grunt\n\n# Bower dependency directory (https://bower.io/)\nbower_components\n\n# node-waf configuration\n.lock-wscript\n\n# Compiled binary addons (https://nodejs.org/api/addons.html)\nbuild/Release\n\n# Dependency directories\nnode_modules/\njspm_packages/\n\n# TypeScript v1 declaration files\ntypings/\n\n# TypeScript cache\n*.tsbuildinfo\n\n# Optional npm cache directory\n.npm\n\n# Optional eslint cache\n.eslintcache\n\n# Microbundle cache\n.rpt2_cache/\n.rts2_cache_cjs/\n.rts2_cache_es/\n.rts2_cache_umd/\n\n# Optional REPL history\n.node_repl_history\n\n# Output of 'npm pack'\n*.tgz\n\n# Yarn Integrity file\n.yarn-integrity\n\n# dotenv environment variables file\n.env\n.env.test\n\n# parcel-bundler cache (https://parceljs.org/)\n.cache\n\n# Next.js build output\n.next\n\n# Nuxt.js build / generate output\n.nuxt\ndist\n\n# Gatsby files\n.cache/\n# Comment in the public line in if your project uses Gatsby and *not* Next.js\n# https://nextjs.org/blog/next-9-1#public-directory-support\n# public\n\n# vuepress build output\n.vuepress/dist\n\n# Serverless directories\n.serverless/\n\n# FuseBox cache\n.fusebox/\n\n# DynamoDB Local files\n.dynamodb/\n\n# TernJS port file\n.tern-port\n/test-results/\n/playwright-report/\n/playwright/.cache/\n\n# The directory Mix will write compiled artifacts to.\n/_build/\n\n# If you run \"mix test --cover\", coverage assets end up here.\n/cover/\n\n# The directory Mix downloads your dependencies sources to.\n/deps/\n\n# Where 3rd-party dependencies like ExDoc output generated docs.\n/doc/\n\n# Ignore .fetch files in case you like to edit your project deps locally.\n/.fetch\n\n# If the VM crashes, it generates a dump, let's ignore it too.\nerl_crash.dump\n\n# Also ignore archive artifacts (built via \"mix archive.build\").\n*.ez\n\n.DS_Store\n.elixir_ls/\n.owl_env.tmp\nauth_cache/\nresults/"
},
{
"path": ".tool-versions",
"content": "elixir 1.14.3-otp-24\nerlang 24.2.1\nnodejs 19.6.0\n"
},
{
"path": "Dockerfile",
"content": "###\n### First Stage - Building the Elixir app as escript\n###\nFROM hexpm/elixir:1.14.3-erlang-23.2.6-alpine-3.16.0 AS build\n\n# install build dependencies\nRUN apk add --no-cache build-base git\n\n# prepare build dir\nWORKDIR /app\n\n# extend hex timeout\nENV HEX_HTTP_TIMEOUT=20\n\n# install hex + rebar\nRUN mix local.hex --force && \\\n mix local.rebar --force\n\n# Copy over the mix.exs and mix.lock files to load the dependencies. If those\n# files don't change, then we don't keep re-fetching and rebuilding the deps.\nCOPY mix.exs mix.lock ./\n\nRUN mix deps.get && \\\n mix deps.compile && \\\n mkdir priv && \\\n # because of https://github.com/elixir-mint/castore/issues/35\n cp _build/dev/lib/castore/priv/cacerts.pem priv/cacerts.pem\n\nCOPY lib lib\n\nRUN mix compile && \\\n mix escript.build\n\n###\n### Second Stage - Setup the Runtime Environment\n###\n\nFROM node:16-bullseye-slim AS app\n\nENV LANG=C.UTF-8\n\n# Install Firefox dependencies + tools\nRUN sh -c 'echo \"deb http://ftp.us.debian.org/debian bullseye main non-free\" >> /etc/apt/sources.list.d/fonts.list' && \\\n DEBIAN_FRONTEND=noninteractive apt-get update && \\\n DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends erlang && \\\n # clean apt cache\n rm -rf /var/lib/apt/lists/*\n\nWORKDIR /app\n\nRUN npm config --global set update-notifier false\nENV PLAYWRIGHT_BROWSERS_PATH=/playwright\nENV REQ_MINT_CACERTFILE=/app/bin/cacerts.pem\n\nCOPY package*.json ./\nRUN PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --include=dev --omit=optional --audit=false --progress=false --loglevel=error\nRUN npm_config_ignore_scripts=1 npx playwright install-deps firefox && \\\n npx playwright install firefox\n\nCOPY ts_src ts_src\nCOPY tsconfig.json ./\nRUN npm run build\n\nCOPY --from=build /app/owl ./bin/owl\nCOPY --from=build /app/priv/cacerts.pem ./bin/cacerts.pem\n\nRUN mkdir -p /app/results\n\nENV HOME=/app\nENV MIX_ENV=dev\n\nENTRYPOINT [\"./bin/owl\"]\n"
},
{
"path": "LICENSE",
"content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2023 Omni Owl GmbH\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License."
},
{
"path": "README.md",
"content": "# OpenOwl 🦉\n\n\n ![\"Release\"](\"https://img.shields.io/github/v/release/AccessOwl/open_owl?color=white\")
\n\n\n ![\"Build\"](\"https://img.shields.io/github/actions/workflow/status/AccessOwl/open_owl/tests.yml?branch=main\")
\n\n\nOpenOwl lets you download user lists including user permissions (and other additional data) from various SaaS applications without the need for a public API. This tool is commonly used to check for orphaned user accounts or as preparation for an access review.\n\nThis project is made with IT Ops, InfoSec and Compliance teams in mind - no developer experience needed. The [`recipes.yml`](recipes.yml) lists all supported applications. You are welcome to contribute to this project by setting up additional vendor integrations.\n\n## How to run it\n\nSince OpenOwl uses various technologies running it with Docker is recommended. There are additional instructions if you want to run it directly.\n\n### Option 1: Shell Wrapper with [`Docker`](https://docs.docker.com/get-docker/) (Recommended)\n\nThe Docker image is built automatically with the first launch.\n\n#### Show all available applications a.k.a recipes\n\n```bash\n./owl.sh recipes list\n```\n\n#### Step 1. Run `login` action \n\nFollowing is an example on how to use it with [`Mezmo`](https://www.mezmo.com/)\n\nParameters like `OWL_USERNAME` and `OWL_PASSWORD` are passed via the environment variable.\n\n```bash\nOWL_USERNAME=someone@acme.com OWL_PASSWORD=abc123 ./owl.sh mezmo login\n```\n\n#### Step 2. Run `download_users` action\n\n\n```bash\n./owl.sh mezmo download_users\n```\n\nDepending on the application there might be additonal parameters required which will be listed after running the command. If that is the case, add the parameters and re-run the command.\n\n\nExample for additional parameters:\nIn [recipes.yml](recipes.yml) you can see that there is a placeholder defined for Mezmo. Placeholders start with a `:` and look like this: `:account_id`. Depending on the recipe they are either passed as parameter or populated from other data (e.g. see Adobe). To pass the `:account_id` parameter, you prefix it with `OWL_` and upcase it. So `:account_id` becomes `OWL_ACCOUNT_ID`.\n\n```bash\nOWL_ACCOUNT_ID=YOUR_ACCOUNT_ID ./owl.sh mezmo download_users\n```\n\n### Option 2: Docker Compose\n\nExamples:\n```bash\ndocker compose run owl recipes list\ndocker compose run -e 'OWL_USERNAME=YOUR_USERNAME' -e 'OWL_PASSWORD=YOUR_PASSWORD' owl mezmo login\ndocker compose run -e 'OWL_ACCOUNT_ID=YOUR_ACCOUNT_ID' owl mezmo download_users\n```\n\n### Option 3: Directly \n\nYou can leverage the available [`.tool-versions`](.tool-versions) file to install requirements with [`asdf`](https://asdf-vm.com).\n\n```bash\nasdf install\n```\n\nAlternatively you can install the required Node, Elixir and Erlang version manually based on the version of the [`.tool-versions`](.tool-versions) file.\n\nRun it with:\n```bash\nmix run lib/cli.exs \n```\n\n## How does it work?\n\nOpenOwl signs in like a regular user by entering username and password (RPA via Playwright). It then uses the SaaS' internal APIs to request the list of users on your behalf.\n\nThis approach only works for SaaS applications with internal APIs. With the rise of single page apps (SPA) (think about React.js, Vue etc.), most applications can be supported. Besides SaaS applications this tool can also be used for internal apps.\n\n## Quick Demo\n\n\n ![\"Watch](\"http://img.youtube.com/vi/0Kz2EwL7xQs/0.jpg\")
\n\n\n## Known limitations\n1. User Account: The user account needs administrator rights (or a similar permissions that grants access to user lists).\n2. Login: Currently only direct logins via username and password are supported. Logins via SSO (Google, Okta, Onelogin,...) will be added in the future.\n3. Captchas: Login flows that include captchas are currently not supported.\n\n## How to contribute\n\n[Here](recipes.yml) is the list of available integrations. Open a PR to add further [recipes](recipes.yml), adjust existing ones or extend missing capabilities (like further pagination strategies) to support even more applications.\n\nWhen all the required capabilities exist, a further integration can take just 30m.\n\n### How to add a new vendor recipe\n\n1. Check that your intended SaaS application has an internal API.\n 1. Open the Developer Tools/Inspector of your favourite browser.\n 1. Navigate to the page that shows all users.\n 1. In your inspector filter by XHR requests and reload the page. When you find some that belong to the same domain like your tool, the application is probably supported.\n1. Use the Network tab in the inspector, find the request that includes your list of users with their permissions.\n1. Copy the request as `curl`-request and paste it into a tool like [Postman](https://www.postman.com/downloads/). You can import the request by clicking *Import*, selecting *Raw text* and pasting the copied `curl`-request.\n1. Execute the request in Postman and remove as many headers and parameters as possible to keep the request clean. Check that the request still works. Add the request to the [`recipes.yml`](recipes.yml).\n1. When you have many users in your SaaS account, you will not see all users at once but 10, 50 or 100. Adjust the pagination parameters and try to get all data by traversing through it. Check which [pagination strategy](lib/pagination_strategies/) applies and add it to your added recipe.\n1. Open the direct login page of the new vendor and find the right [selector](https://www.cuketest.com/playwright/docs/selectors) for the username and password field. Adjust the [`recipes.yml`](recipes.yml) accordingly.\n1. Test that the login flow works properly and the configured action.\n1. Open a PR.\n\n## Troubleshooting\n\n### Build Docker image\n\nIn case the Shell Wrapper cannot build the Docker image as expected, try the following command to build the image manually:\n\n```bash\ndocker build -t open_owl-owl:latest .\n```\n\n## The history of OpenOwl\n\nKnowing who has access to your SaaS tools is often guesswork at best and manually adding and deleting user accounts is tedious and error prone. The fact that SCIM/SAML are often locked up behind an enterprise-subscription is adding to the frustration. That's why Mathias and Philip decided to build [AccessOwl](https://www.accessowl.io/), a tool to automate SaaS account provisioning for employees for any tool.\n\nOpenOwl is essentially an open-source version of the underlying technology of AccessOwl. It was created out of a discussion that having access to your own teams user data should be a basic right for anybody. No matter whether it's used for audits, to discover orphaned user accounts or run access reviews. That's why we decided to open source a core part of AccessOwl to let anybody read out crucial information out of their SaaS stack.\n"
},
{
"path": "docker-compose.yml",
"content": "services:\n owl:\n build: ./\n volumes:\n - ${PWD}/auth_cache:/app/auth_cache\n - ${PWD}/results:/app/results\n - ${PWD}/recipes.yml:/app/recipes.yml\n"
},
{
"path": "lib/api_client.ex",
"content": "defmodule OpenOwl.ApiClient do\n import OpenOwl.Helpers.ApiUtils\n\n @base_header [\n {\"accept-encoding\", \"gzip\"},\n {\"accept\", \"*/*\"}\n ]\n @user_agent \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36\"\n\n def do_request(\n cookies,\n session_storage,\n http_method,\n body,\n url,\n headers,\n response_path,\n pagination,\n %{} = placeholder_params,\n placeholders_from_session_storage\n ) do\n cookie_string = filter_relevant_cookies(cookies, url) |> build_cookie_params_string()\n\n placeholder_params =\n get_params_from_session_storage(session_storage, placeholders_from_session_storage)\n |> Map.merge(placeholder_params)\n\n # required for local mix run\n Application.ensure_all_started(:telemetry)\n Application.ensure_all_started(:req)\n\n connect_options =\n if cacertfile = System.get_env(\"REQ_MINT_CACERTFILE\"),\n do: [transport_opts: [cacertfile: cacertfile]],\n else: []\n\n req =\n Req.new(\n method: http_method,\n body: body,\n url: url,\n headers: build_headers(headers, placeholder_params, cookie_string),\n path_params: placeholder_params,\n connect_options: connect_options,\n user_agent: @user_agent\n )\n\n case Req.request(req) do\n {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->\n if pagination != nil do\n pagination.__struct__.handle_paginated_response(\n req,\n body,\n pagination,\n response_path,\n &extract_data_from_response_body/2\n )\n else\n {:ok, extract_data_from_response_body(body, response_path)}\n end\n\n {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->\n {:http_error, {status, body}}\n\n {:error, exception} ->\n {:error, exception}\n end\n end\n\n defp extract_data_from_response_body(body, response_path) do\n get_field_via_response_path(body, response_path)\n end\n\n defp build_headers(headers, placeholders, cookie_string) do\n req_headers = [{\"cookie\", cookie_string} | @base_header]\n\n [headers | req_headers]\n |> List.flatten()\n |> Enum.map(fn {key, value} -> {key, apply_placeholder_params(value, placeholders)} end)\n end\nend\n"
},
{
"path": "lib/cli.ex",
"content": "defmodule OpenOwl.CLI do\n alias OpenOwl.Recipes\n alias OpenOwl.Recipes.Recipe\n alias OpenOwl.Recipes.Action\n alias OpenOwl.ResponseTransformer\n alias OpenOwl.LoginFlowWrapper\n import OpenOwl.Helpers.ApiUtils, only: [apply_placeholder_params: 2]\n import OpenOwl.Helpers.CliUtils\n\n @owl_cli_name \"./owl.sh\"\n @env_prefix \"OWL_\"\n @auth_cache_folder \"auth_cache\"\n\n # escript\n def main(args) do\n args\n |> parse_args()\n |> process_params()\n end\n\n # direct call via mix run\n def main() do\n System.argv()\n |> parse_args()\n |> process_params()\n end\n\n defp parse_args(args) do\n {flags, commands, _} =\n OptionParser.parse(args,\n strict: [debug: :boolean, pwdebug: :boolean, output: :string, help: :boolean]\n )\n\n {commands, flags}\n end\n\n defp process_params({[\"recipes\", \"list\"], _}) do\n IO.puts(\"Listing recipes with actions and parameters...\\n\")\n\n recipes = load_recipes()\n trailing_padding = get_trailing_padding(recipes)\n\n recipes\n |> Enum.each(fn {title, %Recipe{actions: actions} = recipe} ->\n title =\n title\n |> Atom.to_string()\n |> String.capitalize()\n |> then(&\"#{&1}: \")\n |> String.pad_trailing(trailing_padding)\n\n login_parameters =\n recipe\n |> Recipes.get_required_parameters_for_recipe(\"login\")\n |> parameter_list_as_env()\n |> maybe_in_brackets()\n\n IO.puts(title <> \"login #{login_parameters}\")\n\n Enum.each(actions, fn %Action{} = action ->\n parameters =\n recipe\n |> Recipes.get_required_parameters_for_recipe(action.name)\n |> parameter_list_as_env()\n |> maybe_in_brackets()\n\n IO.puts(String.pad_trailing(\"\", trailing_padding) <> \"#{action.name} #{parameters}\")\n end)\n end)\n end\n\n defp process_params({[vendor, \"login\"], flags}) do\n recipes = load_recipes()\n # TODO: until we pull that out from a config or sth\n app_env_params = get_app_env_params()\n\n with %Recipe{} = recipe <- Recipes.get_recipe(recipes, vendor),\n {:ok, %{username: username, password: password}} <-\n Recipes.validate_recipe_parameters(recipe, \"login\", app_env_params) do\n IO.puts(\"Starting #{vendor}: login...\")\n\n if Keyword.get(flags, :debug, false) do\n IO.puts(\"DEBUG=true\\n\")\n\n LoginFlowWrapper.get_local_cmd_command(\n vendor,\n apply_placeholder_params(recipe.login_url, app_env_params),\n apply_placeholder_params(recipe.destination_url_pattern, app_env_params),\n recipe.username_selector,\n recipe.password_selector,\n username,\n password\n )\n |> IO.puts()\n else\n case LoginFlowWrapper.call_local_cmd(\n vendor,\n apply_placeholder_params(recipe.login_url, app_env_params),\n apply_placeholder_params(recipe.destination_url_pattern, app_env_params),\n recipe.username_selector,\n recipe.password_selector,\n username,\n password,\n Keyword.get(flags, :pwdebug, false)\n ) do\n {:ok, _} ->\n IO.puts(\"... authentication info saved\")\n IO.puts(\"DONE\")\n\n {:error, %{exit_code: exit_code, result: result}} ->\n write_error(\"#{exit_code}: #{inspect(result)}\")\n end\n end\n else\n {:error, missing_parameters} ->\n write_error(\"missing parameters: #{parameter_list_as_env(missing_parameters)}\")\n\n nil ->\n write_error(\"vendor #{vendor} not found\")\n end\n end\n\n defp process_params({[vendor, action_name], flags}) when vendor != \"recipes\" do\n recipes = load_recipes()\n app_env_params = get_app_env_params()\n\n IO.puts(\"Starting #{vendor}: #{action_name}...\")\n cookies_path = Path.join([File.cwd!(), @auth_cache_folder, \"#{vendor}_cookies.json\"])\n\n session_storage_path =\n Path.join([File.cwd!(), @auth_cache_folder, \"#{vendor}_session_storage.json\"])\n\n session_storage =\n case File.read(session_storage_path) do\n {:ok, data} -> Jason.decode!(data)\n {:error, _} -> %{}\n end\n\n with {:ok, cookie_binary_json} <- File.read(cookies_path),\n {:ok, cookies} <- Jason.decode(cookie_binary_json),\n %Recipe{actions: actions} = recipe <- Recipes.get_recipe(recipes, vendor),\n %Action{} = action <- Recipes.get_action_for_name(actions, action_name),\n {:ok, _} <- Recipes.validate_recipe_parameters(recipe, action_name, app_env_params) do\n IO.puts(\n \"... #{action.http_method}-request #{apply_placeholder_params(action.url, app_env_params)}...\"\n )\n\n case OpenOwl.ApiClient.do_request(\n cookies,\n session_storage,\n action.http_method,\n action.body,\n action.url,\n action.headers,\n action.response_path,\n action.pagination,\n app_env_params,\n action.populate_placeholders_from_session_storage\n ) do\n {:ok, response} ->\n output_flag = Keyword.get(flags, :output)\n\n filename =\n if output_flag == nil, do: timebased_filename(\"#{vendor}.csv\"), else: output_flag\n\n path = Path.join([File.cwd!(), \"results\", filename])\n content = ResponseTransformer.records_to_csv(response)\n File.write!(path, content)\n IO.puts(\"... wrote results to #{filename}\")\n IO.puts(\"DONE\")\n\n {:http_error, {status, body}} ->\n write_error(\"HTTP (#{status}): #{inspect(body)}\")\n\n {:error, reason} ->\n write_error(\"#{inspect(reason)}\")\n end\n else\n nil ->\n write_error(\"Vendor #{vendor} or action #{action_name} not found\")\n\n {:error, missing_parameters} when is_list(missing_parameters) ->\n write_error(\"missing parameters: #{parameter_list_as_env(missing_parameters)}\")\n\n {:error, %Jason.DecodeError{} = reason} ->\n write_error(\"#{inspect(reason)}\")\n\n {:error, reason} ->\n write_error(\n \"Authentication cookies file #{cookies_path} could not be loaded: #{inspect(reason)}\"\n )\n end\n end\n\n defp process_params(_), do: print_help()\n\n defp print_help() do\n version = OpenOwl.version()\n\n IO.puts(\"OpenOwl v#{version}\\n\")\n IO.puts(\"Usage:\")\n\n IO.puts(\n \" #{@owl_cli_name} recipes list - Show list of recipes with their actions\"\n )\n\n IO.puts(\n \"[env_vars] #{@owl_cli_name} login - Authenticate and get required authentication\"\n )\n\n IO.puts(\n \"[env_vars] #{@owl_cli_name} [flags] - Triggers defined action of recipes.yml\\n\"\n )\n\n IO.puts(\"Env vars:\")\n IO.puts(\" Some commands need set environment variables. You can prepend commands with them.\")\n IO.puts(\" For instance OWL_USERNAME=user #{@owl_cli_name} login\\n\")\n\n IO.puts(\"Flags:\")\n IO.puts(\" --output - Set a custom output filename. Otherwise a timebased filename is used.\")\n IO.puts(\" --help - Print this help message\")\n end\n\n defp load_recipes() do\n case Recipes.load_recipes() do\n {:ok, recipes} ->\n recipes\n\n {:error, reason} ->\n raise \"File could not be loaded: #{inspect(reason)}\"\n end\n end\n\n defp get_trailing_padding(%{} = recipes) do\n Enum.map(recipes, fn {title, _} -> \"#{title}: \" end)\n |> get_trailing_padding()\n end\n\n defp get_trailing_padding(title_list) when is_list(title_list) do\n title_list\n |> Enum.max(fn left, right -> String.length(left) >= String.length(right) end)\n |> String.length()\n end\n\n defp parameter_list_as_env(list) do\n list\n |> Enum.map(&to_string/1)\n |> Enum.map(&\"#{@env_prefix}#{String.upcase(&1)}\")\n |> Enum.join(\", \")\n end\n\n defp maybe_in_brackets(\"\"), do: \"\"\n\n defp maybe_in_brackets(string) do\n \"(#{string})\"\n end\n\n defp write_error(msg) do\n IO.puts(\"ERROR: #{msg}\")\n System.halt(1)\n end\nend\n\nOpenOwl.CLI.main()\n"
},
{
"path": "lib/helpers/api_utils.ex",
"content": "defmodule OpenOwl.Helpers.ApiUtils do\n @path_separator \".\"\n\n alias OpenOwl.Recipes\n\n @doc \"\"\"\n Resolves a response path and get the described string field out of a map. Some APIs have a\n one-element list as root. We handle it by removing this layer.\n If the first argument is a list and the second is nil, the list is returned directly.\n\n iex> get_field_via_response_path(%{}, nil)\n nil\n\n iex> get_field_via_response_path(%{}, \"a.b\")\n nil\n\n iex> get_field_via_response_path(%{\"a\" => \"hit\"}, nil)\n nil\n\n iex> get_field_via_response_path(%{\"a\" => \"hit\", \"b\" => \"nob\"}, \"a\")\n \"hit\"\n\n iex> get_field_via_response_path([%{\"a\" => \"hit\", \"b\" => \"nob\"}], \"a\")\n \"hit\"\n\n iex> get_field_via_response_path(%{\"a\" => \"hit\", \"b\" => \"nob\"}, \"a.a1\")\n nil\n\n iex> get_field_via_response_path(%{\"a\" => %{\"a1\" => \"hit\", \"a2\" => \"noa1\"}, \"b\" => \"nob\"}, \"a.a1\")\n \"hit\"\n\n iex> get_field_via_response_path(%{\"a\" => %{\"a1\" => \"hit\", \"a2\" => \"hit2\"}, \"b\" => \"nob\"}, \"a.a2\")\n \"hit2\"\n\n iex> get_field_via_response_path(%{\"a\" => %{\"a1\" => \"hit\", \"a2\" => %{\"a2i\" => \"hit2\", \"a2ii\" => \"noa2ii\"}}, \"b\" => \"nob\"}, \"a.a2.a2i\")\n \"hit2\"\n\n iex> get_field_via_response_path([%{\"a\" => \"a1\"}, %{\"a\" => \"a2\"}], nil)\n [%{\"a\" => \"a1\"}, %{\"a\" => \"a2\"}]\n\n iex> get_field_via_response_path([%{\"a\" => \"a1\"}, %{\"a\" => \"a2\"}], \"a\")\n ** (FunctionClauseError) no function clause matching in OpenOwl.Helpers.ApiUtils.get_field_via_response_path/2\n \"\"\"\n def get_field_via_response_path(list, nil) when is_list(list), do: list\n\n def get_field_via_response_path([one], response_path),\n do: get_field_via_response_path(one, response_path)\n\n def get_field_via_response_path(_map, nil), do: nil\n\n def get_field_via_response_path(%{} = map, response_path) do\n path = String.split(response_path, @path_separator)\n get_in(map, path)\n rescue\n _ -> nil\n end\n\n @doc \"\"\"\n Sets a specific field in a map via the passed path. The (nested) structure must exist, otherwise it\n will raise. Some APIs pass a one-element list as root. Such a structure would be kept.\n\n iex> flat_map = %{\"a\" => \"a1\", \"b\" => \"b1\"}\n iex> set_field_via_path(flat_map, nil, \"v\")\n flat_map\n iex> set_field_via_path(flat_map, \"\", \"v\")\n flat_map\n iex> set_field_via_path(flat_map, \"c\", \"v\")\n %{\"a\" => \"a1\", \"b\" => \"b1\", \"c\" => \"v\"}\n iex> set_field_via_path(flat_map, \"b\", \"v\")\n %{\"a\" => \"a1\", \"b\" => \"v\"}\n iex> set_field_via_path([flat_map], \"b\", \"v\")\n [%{\"a\" => \"a1\", \"b\" => \"v\"}]\n\n iex> deep_map = %{\"a\" => \"a1\", \"b\" => \"b1\", \"d\" => %{\"d1\" => \"d1v\"}}\n iex> set_field_via_path(deep_map, \"d.d1\", \"v\")\n %{\"a\" => \"a1\", \"b\" => \"b1\", \"d\" => %{\"d1\" => \"v\"}}\n iex> set_field_via_path([deep_map], \"d.d1\", \"v\")\n [%{\"a\" => \"a1\", \"b\" => \"b1\", \"d\" => %{\"d1\" => \"v\"}}]\n iex> set_field_via_path(deep_map, \"d.d2\", \"v\")\n %{\"a\" => \"a1\", \"b\" => \"b1\", \"d\" => %{\"d1\" => \"d1v\", \"d2\" => \"v\"}}\n iex> set_field_via_path(deep_map, \"d.d1.deep\", \"v\")\n ** (FunctionClauseError) no function clause matching in Access.get_and_update/3\n \"\"\"\n def set_field_via_path(%{} = map, nil, _value), do: map\n def set_field_via_path(%{} = map, \"\", _value), do: map\n\n def set_field_via_path([one], field_path, value),\n do: set_field_via_path(one, field_path, value, true)\n\n def set_field_via_path(%{} = map, field_path, value, list? \\\\ false) do\n path = String.split(field_path, @path_separator)\n\n result = put_in(map, path, value)\n if list?, do: [result], else: result\n end\n\n @doc \"\"\"\n Filters for cookies of the passed url (domain + subdomain).\n\n iex> cookie_domain = %{\"name\" => \"session\",\"value\" => \"123\",\"domain\" => \"example.com\",\"path\" => \"/\",\"expires\" => -1,\"httpOnly\" => true,\"secure\" => true,\"sameSite\" => \"None\"}\n iex> cookie_subdomain = %{\"name\" => \"session\",\"value\" => \"123\",\"domain\" => \".example.com\",\"path\" => \"/\",\"expires\" => -1,\"httpOnly\" => true,\"secure\" => true,\"sameSite\" => \"None\"}\n iex> cookies = [cookie_domain, cookie_subdomain]\n iex> filter_relevant_cookies(cookies, \"http://example.com/a?b=2\")\n cookies\n iex> filter_relevant_cookies(cookies, \"http://s.bla.example.com/a?b=2\")\n cookies\n \"\"\"\n def filter_relevant_cookies(cookies, url) do\n %URI{host: host} = URI.parse(url)\n\n host =\n host |> String.split(@path_separator) |> Enum.slice(-2..-1) |> Enum.join(@path_separator)\n\n Enum.filter(cookies, fn cookie ->\n String.contains?(cookie[\"domain\"], host)\n end)\n end\n\n @doc ~S\"\"\"\n Gets the passed params (support paths separated by \".\") from session storage. Searches in values\n of all the session storage and merges with maps one level deeper. Also decodes JSON on first level\n if necessary.\n\n Current implementation is not flexible enough yet to support many use cases. Depending on future\n requirements we will adjust it (including the interface towards recipes).\n\n iex> get_params_from_session_storage(%{}, [])\n %{}\n\n iex> payload = %{\"adobeid_ims_access_token/ONESIE1/false/AdobeID,ab.manage\" => \"{\\\"REAUTH_SCOPE\\\":\\\"reauthenticated\\\",\\\"client_id\\\":\\\"ONESIE1\\\",\\\"scope\\\":\\\"openid,AdobeID,additional_info.projectedProductContext,read_organizations,read_members,read_countries_regions,additional_info.roles,adobeio_api,read_auth_src_domains,authSources.rwd,bis.read.pi,app_policies.read,app_policies.write,client.read,publisher.read,client.scopes.read,creative_cloud,service_principals.write,aps.read.app_merchandising,aps.eval_licensesforapps,ab.manage,aps.device_activation_mgmt\\\",\\\"expire\\\":\\\"2023-01-01T13:37:00.342Z\\\",\\\"user_id\\\":\\\"thisiauser_id\\\",\\\"tokenValue\\\":\\\"longtoken\\\",\\\"sid\\\":\\\"thisissid\\\",\\\"state\\\":{},\\\"fromFragment\\\":false,\\\"impersonatorId\\\":\\\"\\\",\\\"isImpersonatedSession\\\":false,\\\"other\\\":\\\"{}\\\",\\\"pbaSatisfiedPolicies\\\":[\\\"MedSecNoEV\\\",\\\"LowSec\\\"]}\", \"adobeid_ims_profile/ONESIE1/false/AdobeID,ab.manage,additional_info\" => \"{\\\"account_type\\\":\\\"type2e\\\",\\\"utcOffset\\\":\\\"null\\\",\\\"preferred_languages\\\":[\\\"en-us\\\"],\\\"displayName\\\":\\\"Integration Account\\\",\\\"roles\\\":[{\\\"principal\\\":\\\"LONGID@AdobeOrg:123456\\\",\\\"organization\\\":\\\"LONGID@AdobeOrg\\\",\\\"named_role\\\":\\\"org_admin\\\",\\\"target\\\":\\\"LONGID@AdobeOrg\\\",\\\"target_type\\\":\\\"TRG_ORG\\\",\\\"target_data\\\":{}}],\\\"last_name\\\":\\\"Account\\\",\\\"userId\\\":\\\"someuserid.e\\\",\\\"authId\\\":\\\"someauthid@AdobeID\\\",\\\"tags\\\":[\\\"agegroup_18plus\\\"],\\\"projectedProductContext\\\":[],\\\"emailVerified\\\":\\\"true\\\",\\\"toua\\\":[{\\\"touName\\\":\\\"creative_cloud\\\",\\\"current\\\":true}],\\\"phoneNumber\\\":null,\\\"countryCode\\\":\\\"US\\\",\\\"name\\\":\\\"Integration Account\\\",\\\"mrktPerm\\\":\\\"\\\",\\\"mrktPermEmail\\\":null,\\\"first_name\\\":\\\"Integration\\\",\\\"email\\\":\\\"bruce@wayne.org\\\"}\", \"sherlockId\" => \"sherlockUUID\"}\n iex> get_params_from_session_storage(payload, [])\n %{}\n iex> get_params_from_session_storage(payload, ~w(client_id tokenValue notexistent))\n %{client_id: \"ONESIE1\", tokenValue: \"longtoken\", notexistent: nil}\n iex> get_params_from_session_storage(payload, ~w(client_id roles.organization))\n %{client_id: \"ONESIE1\", organization: \"LONGID@AdobeOrg\"}\n \"\"\"\n def get_params_from_session_storage(session_storage, params)\n when is_map(session_storage) and is_list(params) do\n result =\n session_storage\n |> Map.values()\n |> Enum.map(fn item ->\n case Jason.decode(item) do\n {:ok, result} -> result\n {:error, _} -> item\n end\n end)\n |> Enum.reduce(%{}, fn\n item, acc when is_map(item) ->\n Map.merge(acc, item)\n\n # we ignore flat items here for the moment. We might need it later which means refactoring\n # of this approach\n _item, acc ->\n acc\n end)\n |> Enum.map(fn\n {key, []} ->\n {key, []}\n\n {key, value} when is_list(value) ->\n {key, hd(value)}\n\n {key, value} ->\n {key, value}\n end)\n |> Map.new()\n\n Enum.reduce(params, %{}, fn param, acc ->\n Map.put(\n acc,\n get_last_response_path_part(param) |> String.to_atom(),\n get_field_via_response_path(result, param)\n )\n end)\n end\n\n @doc \"\"\"\n Returns the last part of a response path that is splitted by #{@path_separator}.\n\n iex> get_last_response_path_part(\"a.b.c\")\n \"c\"\n\n iex> get_last_response_path_part(\"abc\")\n \"abc\"\n\n iex> get_last_response_path_part(\"\")\n \"\"\n \"\"\"\n def get_last_response_path_part(path) do\n path |> String.split(@path_separator) |> Enum.reverse() |> hd()\n end\n\n @doc \"\"\"\n Builds a string properly formatted to send it as cookies header.\n\n iex> cookie_a = %{\"name\" => \"session\",\"value\" => \"123\",\"domain\" => \"example.com\",\"path\" => \"/\",\"expires\" => -1,\"httpOnly\" => true,\"secure\" => true,\"sameSite\" => \"None\"}\n iex> cookie_b = %{\"name\" => \"bla\",\"value\" => \"123\",\"domain\" => \"example.com\",\"path\" => \"/\",\"expires\" => -1,\"httpOnly\" => true,\"secure\" => true,\"sameSite\" => \"None\"}\n iex> cookies = [cookie_a, cookie_b]\n iex> build_cookie_params_string([cookie_a])\n \"session=123\"\n iex> build_cookie_params_string(cookies)\n \"session=123; bla=123\"\n \"\"\"\n def build_cookie_params_string(cookies) do\n Enum.map_join(cookies, \"; \", &(&1[\"name\"] <> \"=\" <> &1[\"value\"]))\n end\n\n @doc \"\"\"\n Replaces placeholder in subject using passed params.\n\n iex> apply_placeholder_params(\"cool\", %{org: \"o1\", team_id: 123})\n \"cool\"\n\n iex> apply_placeholder_params(\"cool:org\", %{org: \"o1\", team_id: 123})\n \"coolo1\"\n\n iex> apply_placeholder_params(\"cool_:team_id_:org_\", %{org: \"o1\", team_id: 123})\n \"cool_123_o1_\"\n\n iex> apply_placeholder_params(\"cool_:team_bla_:org_\", %{_org: \"o1\", team_bla_id: 123})\n \"cool_:team_bla_:org_\"\n \"\"\"\n def apply_placeholder_params(subject, params) do\n Regex.replace(Recipes.placeholder_regex(), subject, fn whole_match, key ->\n to_string(params[String.to_atom(key)] || whole_match)\n end)\n end\nend\n"
},
{
"path": "lib/helpers/cli_utils.ex",
"content": "defmodule OpenOwl.Helpers.CliUtils do\n @prefix \"OWL_\"\n\n @doc \"\"\"\n Returns all app environment params downcased and without app prefix.\n\n iex> get_app_env_params(%{\"SHELL\" => \"/bin/zsh\", \"DOWL_BLA\" => \"no\", \"OWL_USERNAME\" => \"Oo\", \"OWL_PASSWORD\" => \"secret\", \"OWL_TEAM_ID\" => \"123\"})\n %{username: \"Oo\", password: \"secret\", team_id: \"123\"}\n\n iex> get_app_env_params(%{\"SHELL\" => \"/bin/zsh\", \"DOWL_BLA\" => \"no\"})\n %{}\n\n iex> get_app_env_params(%{})\n %{}\n \"\"\"\n def get_app_env_params(params \\\\ System.get_env()) do\n params\n |> Enum.filter(fn\n {@prefix <> _, _value} -> true\n _ -> false\n end)\n |> Enum.map(fn {key, value} -> {normalize_var(key), value} end)\n |> Map.new()\n end\n\n defp normalize_var(var) do\n var |> String.replace(@prefix, \"\") |> String.downcase() |> String.to_atom()\n end\n\n @doc \"\"\"\n Generates a timebased filename.\n\n iex> datetime = ~N[2017-11-06 00:23:51.123456]\n iex> timebased_filename(\"miro.csv\", datetime)\n \"20171106T002351_miro.csv\"\n\n iex> timebased_filename(nil)\n ** (ArgumentError) filename empty\n\n iex> timebased_filename(\"\")\n ** (ArgumentError) filename empty\n \"\"\"\n def timebased_filename(filename, datetime \\\\ NaiveDateTime.utc_now())\n def timebased_filename(nil, _), do: raise(ArgumentError, \"filename empty\")\n def timebased_filename(\"\", _), do: raise(ArgumentError, \"filename empty\")\n\n def timebased_filename(filename, datetime) do\n now =\n datetime\n |> NaiveDateTime.truncate(:second)\n |> NaiveDateTime.to_iso8601(:basic)\n\n \"#{now}_#{filename}\"\n end\nend\n"
},
{
"path": "lib/helpers/struct_utils.ex",
"content": "defmodule OpenOwl.Helpers.StructUtils do\n @moduledoc \"\"\"\n Converts a map with strings, a map with atoms or an existing struct with all\n of the necessary keys into a target struct.\n \"\"\"\n\n def to_struct(kind, attrs) do\n struct = struct(kind)\n\n map_list = Map.to_list(struct)\n\n Enum.reduce(map_list, struct, fn\n {:__struct__, _}, acc -> acc\n {key, _}, acc -> fetch_value(attrs, key, acc)\n end)\n end\n\n defp fetch_value(attrs, key, acc) when is_atom(key) do\n case Map.fetch(attrs, key) do\n {:ok, value} -> %{acc | key => value}\n :error -> fetch_value(attrs, Atom.to_string(key), acc)\n end\n end\n\n defp fetch_value(attrs, key, acc) do\n case Map.fetch(attrs, key) do\n # credo:disable-for-next-line\n {:ok, value} -> %{acc | String.to_atom(key) => value}\n :error -> acc\n end\n end\nend\n"
},
{
"path": "lib/login_flow_wrapper.ex",
"content": "defmodule OpenOwl.LoginFlowWrapper do\n def call_local_cmd(\n slug,\n url,\n destination_url_pattern,\n username_selector,\n password_selector,\n username,\n password,\n pw_debug?\n ) do\n case System.cmd(\"npm\", [\"run\", \"start\"],\n env: [\n {\"SLUG\", slug},\n {\"URL\", url},\n {\"DESTINATION_URL_PATTERN\", destination_url_pattern},\n {\"USERNAME_SELECTOR\", username_selector},\n {\"PASSWORD_SELECTOR\", password_selector},\n {\"USER\", username},\n {\"PASSWORD\", password},\n {\"PWDEBUG\", if(pw_debug?, do: \"1\", else: \"0\")}\n ]\n ) do\n {result, 0} ->\n {:ok, result}\n\n {result, exit_code} ->\n {:error, %{exit_code: exit_code, result: result}}\n end\n end\n\n @doc \"\"\"\n Returns the command to execute for the login flow.\n\n ## Examples\n\n iex> get_local_cmd_command(\"slug\", \"url\", \"dest\", \"usel\", \"psel\", \"user\", \"password\")\n ~s(PWDEBUG=1 SLUG=slug URL=\"url\" DESTINATION_URL_PATTERN=\"dest\" USERNAME_SELECTOR=\"usel\" PASSWORD_SELECTOR=\"psel\" USER=\"user\" PASSWORD=\"password\" npm run start)\n \"\"\"\n def get_local_cmd_command(\n slug,\n url,\n destination_url_pattern,\n username_selector,\n password_selector,\n username,\n password\n ) do\n env = [\n \"PWDEBUG=1\",\n \"SLUG=#{slug}\",\n ~s(URL=\"#{url}\"),\n ~s(DESTINATION_URL_PATTERN=\"#{destination_url_pattern}\"),\n ~s(USERNAME_SELECTOR=\"#{username_selector}\"),\n ~s(PASSWORD_SELECTOR=\"#{password_selector}\"),\n ~s(USER=\"#{username}\"),\n ~s(PASSWORD=\"#{password}\")\n ]\n\n \"#{Enum.join(env, \" \")} npm run start\"\n end\nend\n"
},
{
"path": "lib/pagination_strategies/pagination_next_cursor_in_body_to_send_as_json.ex",
"content": "defmodule OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson do\n @moduledoc \"\"\"\n Handles APIs that return a cursor in the response body. Puts the cursur into a field of the JSON\n request body using `json_body_cursor_path`.\n It needs a boolean flag (`has_next_page_response_path`) to indicate whether the last page was\n reached.\n\n ## Example response:\n ```json\n {\n \"results\": [{\"id\": 1}, {\"id\": 2}],\n \"pagination\": {\"next_cursor\": \"abc\", \"has_next_page\": true}\n }\n ```\n\n You can use the `next_cursor_response_path` parameter to define the field that has the cursor. You\n can traverse into nested structures with the \".\" syntax (e.g. with `root.deep.deeper`). Here it\n would be `pagination.next_cursor`.\n \"\"\"\n\n use OpenOwl.PaginationStrategy\n\n alias OpenOwl.Helpers.ApiUtils\n\n @enforce_keys [\n :strategy,\n :next_cursor_response_path,\n :has_next_page_response_path,\n :json_body_cursor_path\n ]\n defstruct strategy: nil,\n next_cursor_response_path: nil,\n has_next_page_response_path: nil,\n json_body_cursor_path: nil\n\n @type t :: %__MODULE__{\n strategy: :next_cursor_in_body_to_send_as_json,\n next_cursor_response_path: String.t(),\n has_next_page_response_path: String.t(),\n json_body_cursor_path: String.t()\n }\n\n @impl true\n def handle_paginated_response(\n %Req.Request{} = req,\n body,\n %__MODULE__{} = pagination,\n response_path,\n data_extraction_fn,\n data_acc \\\\ []\n ) do\n data = data_extraction_fn.(body, response_path)\n data_acc = data_acc ++ data\n\n next_cursor = ApiUtils.get_field_via_response_path(body, pagination.next_cursor_response_path)\n\n new_req_body =\n req.body\n |> Jason.decode!()\n |> ApiUtils.set_field_via_path(pagination.json_body_cursor_path, next_cursor)\n\n has_next_page? =\n ApiUtils.get_field_via_response_path(body, pagination.has_next_page_response_path)\n\n if has_next_page? do\n case Req.request(req, json: new_req_body) do\n {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->\n handle_paginated_response(\n req,\n body,\n pagination,\n response_path,\n data_extraction_fn,\n data_acc\n )\n\n {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->\n {:http_error, {status, body}}\n\n {:error, exception} ->\n {:error, exception}\n end\n else\n {:ok, data_acc}\n end\n end\nend\n"
},
{
"path": "lib/pagination_strategies/pagination_next_page_in_body.ex",
"content": "defmodule OpenOwl.PaginationStrategy.NextPageInBody do\n @moduledoc \"\"\"\n Handles APIs that return the next page cursor in the response body. Furthermore the cursor needs\n to be passed as query parameter.\n\n ## Example response:\n ```json\n {\n \"results\": [{\"id\": 1}, {\"id\": 2}],\n \"pagination\": {\"next_page\": 42\"}\n }\n ```\n\n Use `page_query_param` to define the name of the query parameter.\n You can use the `next_page_response_path` parameter to define the next page field. You can traverse\n into nested structures with the \".\" syntax (e.g. with `root.deep.deeper`). Here it would be\n `pagination.next_page`.\n \"\"\"\n\n use OpenOwl.PaginationStrategy\n\n alias OpenOwl.Helpers.ApiUtils\n\n @enforce_keys [:strategy, :next_page_response_path, :page_query_param]\n defstruct strategy: nil, next_page_response_path: nil, page_query_param: nil\n\n @type t :: %__MODULE__{\n strategy: :next_page_in_body,\n next_page_response_path: String.t(),\n page_query_param: String.t()\n }\n\n @impl true\n def handle_paginated_response(\n %Req.Request{} = req,\n body,\n %__MODULE__{} = pagination,\n response_path,\n data_extraction_fn,\n data_acc \\\\ []\n ) do\n data = data_extraction_fn.(body, response_path)\n data_acc = data_acc ++ data\n\n next_page = ApiUtils.get_field_via_response_path(body, pagination.next_page_response_path)\n query_param = Keyword.new([{String.to_atom(pagination.page_query_param), next_page}])\n\n if next_page != nil and next_page != \"\" do\n case Req.request(req, params: query_param) do\n {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->\n handle_paginated_response(\n req,\n body,\n pagination,\n response_path,\n data_extraction_fn,\n data_acc\n )\n\n {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->\n {:http_error, {status, body}}\n\n {:error, exception} ->\n {:error, exception}\n end\n else\n {:ok, data_acc}\n end\n end\nend\n"
},
{
"path": "lib/pagination_strategies/pagination_offset_params_in_url.ex",
"content": "defmodule OpenOwl.PaginationStrategy.OffsetParamsInUrl do\n @moduledoc \"\"\"\n Handles APIs that have offset params in URL query parameters. By default, it assumes `offset`\n is used as parameter. The parameter must be part of the URL in the first request. Subsequent\n requests will increment this parameter by the value of the `limit` query parameter. The name of\n this parameter can be defined with `limit_query_param` and is `limit` by default.\n\n The name of the passed `offset` param can be defined with `offset_query_param`.\n\n Request loop stops when the page has no data anymore (empty result).\n \"\"\"\n use OpenOwl.PaginationStrategy\n\n @enforce_keys [:strategy]\n defstruct strategy: nil, offset_query_param: \"offset\", limit_query_param: \"limit\"\n\n @type t :: %__MODULE__{\n strategy: :page_params_in_url,\n offset_query_param: String.t(),\n limit_query_param: String.t()\n }\n\n @impl true\n def handle_paginated_response(\n %Req.Request{} = req,\n body,\n %__MODULE__{} = pagination,\n response_path,\n data_extraction_fn,\n data_acc \\\\ []\n ) do\n data = data_extraction_fn.(body, response_path)\n data_acc = data_acc ++ data\n\n %URI{query: query} = req.url\n limit_query_param = pagination.limit_query_param\n %{^limit_query_param => limit} = query_params = URI.decode_query(query)\n\n query_params_string =\n query_params\n |> Map.update!(\n pagination.offset_query_param,\n &(String.to_integer(&1) + String.to_integer(limit))\n )\n |> URI.encode_query()\n\n req =\n update_in(req, [Access.key!(:url), Access.key!(:query)], fn _ -> query_params_string end)\n\n if data != [] do\n case Req.request(req) do\n {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->\n handle_paginated_response(\n req,\n body,\n pagination,\n response_path,\n data_extraction_fn,\n data_acc\n )\n\n {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->\n {:http_error, {status, body}}\n\n {:error, exception} ->\n {:error, exception}\n end\n else\n {:ok, data_acc}\n end\n end\nend\n"
},
{
"path": "lib/pagination_strategies/pagination_page_params_in_url.ex",
"content": "defmodule OpenOwl.PaginationStrategy.PageParamsInUrl do\n @moduledoc \"\"\"\n Handles APIs that have page params in URL query parameters. By default, it assumes `page`\n is used as parameter. The parameter must be part of the URL in the first request. Subsequent\n requests will increment this parameter.\n\n The name of the passed `page` param can be defined with `page_query_param`.\n\n Request loop stops when the page has no data anymore (empty result).\n \"\"\"\n use OpenOwl.PaginationStrategy\n\n @enforce_keys [:strategy]\n defstruct strategy: nil, page_query_param: \"page\"\n\n @type t :: %__MODULE__{strategy: :page_params_in_url, page_query_param: String.t()}\n\n @impl true\n def handle_paginated_response(\n %Req.Request{} = req,\n body,\n %__MODULE__{} = pagination,\n response_path,\n data_extraction_fn,\n data_acc \\\\ []\n ) do\n data = data_extraction_fn.(body, response_path)\n data_acc = data_acc ++ data\n\n %URI{query: query} = req.url\n\n query_params_string =\n query\n |> URI.decode_query()\n |> Map.update!(pagination.page_query_param, &(String.to_integer(&1) + 1))\n |> URI.encode_query()\n\n req =\n update_in(req, [Access.key!(:url), Access.key!(:query)], fn _ -> query_params_string end)\n\n if data != [] do\n case Req.request(req) do\n {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->\n handle_paginated_response(\n req,\n body,\n pagination,\n response_path,\n data_extraction_fn,\n data_acc\n )\n\n {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->\n {:http_error, {status, body}}\n\n {:error, exception} ->\n {:error, exception}\n end\n else\n {:ok, data_acc}\n end\n end\nend\n"
},
{
"path": "lib/pagination_strategies/pagination_strategy.ex",
"content": "defmodule OpenOwl.PaginationStrategy do\n @type t :: module()\n @type status() :: pos_integer()\n @type body() :: map()\n @type pagination() :: %{atom() => atom() | String.t()}\n @type response_path() :: String.t()\n\n @doc \"\"\"\n Casts a map to a typed struct.\n \"\"\"\n @callback cast(map()) :: %{\n :__struct__ => atom(),\n :strategy => atom(),\n optional(atom()) => any()\n }\n\n @doc \"\"\"\n Follows the pagination strategy to accumulate all the data and returns it at once.\n \"\"\"\n @callback handle_paginated_response(\n %Req.Request{},\n body(),\n pagination(),\n response_path(),\n fun(),\n [map()]\n ) :: {:ok, map()} | {:http_error, {status(), body()}} | {:error, any()}\n\n defmacro __using__(_options) do\n quote do\n @behaviour OpenOwl.PaginationStrategy\n\n @impl true\n def cast(attrs) do\n OpenOwl.Helpers.StructUtils.to_struct(__MODULE__, attrs)\n end\n end\n end\nend\n"
},
{
"path": "lib/pagination_strategies/pagination_url_in_body.ex",
"content": "defmodule OpenOwl.PaginationStrategy.UrlInBody do\n @moduledoc \"\"\"\n Handles APIs that return a URL in the response body where the next page can be fetched.\n\n ## Example response:\n ```json\n {\n \"results\": [{\"id\": 1}, {\"id\": 2}],\n \"nextLink\": \"https://api.example.com/?nextLink=456\"\n }\n ```\n\n You can use the `next_url_response_path` parameter to define the field that has the URL. You can\n traverse into nested structures with the \".\" syntax (e.g. with `root.deep.deeper`). Here it would\n be just `nextLink`.\n \"\"\"\n\n use OpenOwl.PaginationStrategy\n\n alias OpenOwl.Helpers.ApiUtils\n\n @enforce_keys [:strategy, :next_url_response_path]\n defstruct strategy: nil, next_url_response_path: nil\n\n @type t :: %__MODULE__{\n strategy: :url_in_body,\n next_url_response_path: String.t()\n }\n\n @impl true\n def handle_paginated_response(\n %Req.Request{} = req,\n body,\n %__MODULE__{} = pagination,\n response_path,\n data_extraction_fn,\n data_acc \\\\ []\n ) do\n data = data_extraction_fn.(body, response_path)\n data_acc = data_acc ++ data\n\n next_url = ApiUtils.get_field_via_response_path(body, pagination.next_url_response_path)\n\n if next_url != nil and next_url != \"\" do\n case Req.request(req, url: next_url) do\n {:ok, %Req.Response{status: status, body: body}} when status in [200, 201] ->\n handle_paginated_response(\n req,\n body,\n pagination,\n response_path,\n data_extraction_fn,\n data_acc\n )\n\n {:ok, %Req.Response{status: status, body: body}} when status in [400, 401, 403, 404] ->\n {:http_error, {status, body}}\n\n {:error, exception} ->\n {:error, exception}\n end\n else\n {:ok, data_acc}\n end\n end\nend\n"
},
{
"path": "lib/recipes/action.ex",
"content": "defmodule OpenOwl.Recipes.Action do\n alias OpenOwl.Helpers.StructUtils\n\n @enforce_keys [:name, :http_method, :url]\n defstruct name: nil,\n http_method: nil,\n body: nil,\n url: nil,\n headers: [],\n response_path: nil,\n pagination: nil,\n populate_placeholders_from_session_storage: []\n\n @type t :: %__MODULE__{\n name: String.t(),\n http_method: :get | :post | :put | :patch | :delete,\n body: String.t(),\n url: String.t(),\n headers: [{String.t(), String.t()}],\n response_path: String.t() | nil,\n pagination: OpenOwl.PaginationStrategy.t() | nil,\n populate_placeholders_from_session_storage: [String.t()] | nil\n }\n\n def cast(attrs) do\n http_method =\n case attrs do\n %{http_method: http_method} -> http_method\n %{\"http_method\" => http_method} -> http_method\n end\n\n headers =\n case attrs do\n %{headers: headers} -> headers\n %{\"headers\" => headers} -> headers\n _ -> []\n end\n |> Enum.map(fn header -> Map.to_list(header) |> hd() end)\n\n {pagination_strategy, pagination_attrs} =\n case attrs do\n %{pagination: pagination} ->\n {pagination[:strategy], pagination}\n\n %{\"pagination\" => pagination} ->\n {pagination[\"strategy\"], pagination}\n\n _ ->\n {nil, nil}\n end\n\n pagination_mod = modulize(pagination_strategy)\n\n action =\n StructUtils.to_struct(__MODULE__, attrs)\n |> Map.put(:http_method, http_method_to_atom(http_method))\n |> Map.put(:headers, headers)\n\n if pagination_mod != nil,\n do: Map.put(action, :pagination, apply(pagination_mod, :cast, [pagination_attrs])),\n else: action\n end\n\n defp http_method_to_atom(http_method) do\n http_method |> String.downcase() |> String.to_atom()\n end\n\n defp modulize(nil), do: nil\n\n defp modulize(underscored_string) do\n underscored_string\n |> String.split(\"_\")\n |> Enum.map_join(&String.capitalize/1)\n |> then(&Module.concat(OpenOwl.PaginationStrategy, &1))\n end\nend\n"
},
{
"path": "lib/recipes/recipe.ex",
"content": "defmodule OpenOwl.Recipes.Recipe do\n alias OpenOwl.Helpers.StructUtils\n alias OpenOwl.Recipes.Action\n\n @enforce_keys [\n :login_url,\n :destination_url_pattern,\n :username_selector,\n :password_selector\n ]\n defstruct login_url: nil,\n destination_url_pattern: nil,\n username_selector: nil,\n password_selector: nil,\n actions: []\n\n @type t :: %__MODULE__{\n login_url: String.t(),\n destination_url_pattern: String.t(),\n username_selector: String.t(),\n password_selector: String.t(),\n actions: [] | [Action.t()]\n }\n\n def cast(attrs, actions) do\n recipe = StructUtils.to_struct(__MODULE__, attrs)\n\n %{recipe | actions: Enum.map(actions, &Action.cast/1)}\n end\nend\n"
},
{
"path": "lib/recipes.ex",
"content": "defmodule OpenOwl.Recipes do\n @recipe_file_name \"recipes.yml\"\n @external_resource Path.join(File.cwd!(), @recipe_file_name)\n @placeholder_regex ~r/:([a-zA-Z]{1}([_]*[a-zA-Z]+)*)/\n\n alias OpenOwl.Recipes.Recipe\n alias OpenOwl.Recipes.Action\n\n def load_recipes(rel_path \\\\ @recipe_file_name) do\n path = Path.join(File.cwd!(), rel_path)\n\n case YamlElixir.read_from_file(path) do\n {:ok, data} -> {:ok, parse_yaml_data(data)}\n {:error, reason} -> {:error, reason}\n end\n end\n\n def get_recipe(recipes, slug) when is_binary(slug) do\n get_recipe(recipes, String.to_atom(slug))\n end\n\n def get_recipe(recipes, slug) do\n Map.get(recipes, slug)\n end\n\n def get_action_for_name(actions, name) do\n Enum.find(actions, &(&1.name == name))\n end\n\n @doc false\n def get_required_parameters_for_recipe(\n %Recipe{login_url: login_url, destination_url_pattern: destination_url_pattern},\n \"login\"\n ) do\n [:username, :password] ++ get_parameters([login_url, destination_url_pattern])\n end\n\n @doc false\n def get_required_parameters_for_recipe(%Recipe{actions: actions}, action_name) do\n with %Action{url: url, headers: headers} <- get_action_for_name(actions, action_name) do\n get_parameters([url | get_header_values(headers)])\n else\n nil -> []\n end\n end\n\n defp get_parameters(strings) do\n strings\n |> Enum.reduce([], fn string, acc ->\n matches = Regex.scan(@placeholder_regex, string, capture: :first)\n [matches | acc]\n end)\n |> List.flatten()\n |> Enum.uniq()\n |> Enum.map(&atomize_placeholder_string/1)\n end\n\n defp atomize_placeholder_string(placeholder) do\n \":\" <> name = placeholder\n String.to_atom(name)\n end\n\n defp get_header_values(headers) do\n Enum.map(headers, fn {_, value} -> value end)\n end\n\n def validate_recipe_parameters(%Recipe{} = recipe, action, %{} = parameter_map) do\n parameters = Map.keys(parameter_map)\n required_parameters = get_required_parameters_for_recipe(recipe, action)\n missing_parameters = required_parameters -- parameters\n\n if missing_parameters == [] do\n {:ok, parameter_map}\n else\n {:error, missing_parameters}\n end\n end\n\n @doc false\n def placeholder_regex() do\n @placeholder_regex\n end\n\n defp parse_yaml_data(%{} = data) do\n Enum.reduce(data, %{}, fn {key, value}, acc ->\n Map.put(acc, String.to_atom(key), OpenOwl.Recipes.Recipe.cast(value, value[\"actions\"]))\n end)\n end\nend\n"
},
{
"path": "lib/response_transformer.ex",
"content": "defmodule OpenOwl.ResponseTransformer do\n alias NimbleCSV.RFC4180, as: CSV\n\n @doc \"\"\"\n Converts a response that consists of a list of records (maps) of the same\n type to csv.\n \"\"\"\n def records_to_csv(records) do\n records\n |> records_to_list_with_header()\n |> list_with_header_to_csv()\n end\n\n def list_with_header_to_csv(list_with_header) when is_list(list_with_header) do\n list_with_header\n |> CSV.dump_to_iodata()\n |> IO.iodata_to_binary()\n end\n\n @doc false\n def records_to_list_with_header([]) do\n []\n end\n\n @doc false\n def records_to_list_with_header(map_list) when is_list(map_list) do\n records =\n map_list\n |> Enum.map(fn map ->\n flat_record(map)\n end)\n\n header = hd(records) |> Map.keys()\n content = Enum.map(records, &Map.values/1)\n\n [header | content]\n end\n\n @doc false\n def flat_record(map) do\n flat_record(map, nil)\n |> List.flatten()\n |> Map.new()\n end\n\n @doc false\n def flat_record(map, prefix) do\n map\n |> Enum.map(fn\n {key, value} when is_map(value) ->\n flat_record(value, [key | [prefix]])\n\n {key, value} when is_list(value) ->\n if Enum.any?(value, &is_map/1) do\n nil\n else\n {key, value}\n end\n\n {key, value} ->\n field_name =\n [key | [prefix]]\n |> List.flatten()\n |> Enum.reverse()\n |> Enum.reject(&is_nil/1)\n |> Enum.join(\".\")\n\n {field_name, value}\n end)\n |> Enum.reject(&is_nil/1)\n end\nend\n"
},
{
"path": "lib/saas_owl.ex",
"content": "defmodule OpenOwl do\n @version OpenOwl.MixProject.project() |> Keyword.fetch!(:version)\n\n def version do\n @version\n end\nend\n"
},
{
"path": "mix.exs",
"content": "defmodule OpenOwl.MixProject do\n use Mix.Project\n\n def project do\n [\n app: :open_owl,\n version: \"0.1.0\",\n elixir: \"~> 1.14\",\n elixirc_paths: elixirc_paths(Mix.env()),\n start_permanent: Mix.env() == :prod,\n escript: escript(),\n deps: deps()\n ]\n end\n\n # Run \"mix help compile.app\" to learn about applications.\n def application do\n [\n extra_applications: [:logger]\n ]\n end\n\n defp elixirc_paths(_), do: [\"lib\"]\n\n defp escript do\n [main_module: OpenOwl.CLI, name: \"owl\"]\n end\n\n # Run \"mix help deps\" to learn about dependencies.\n defp deps do\n [\n {:yaml_elixir, \"~> 2.9\"},\n {:jason, \"~> 1.4\"},\n {:nimble_csv, \"~> 1.2\"},\n {:req, \"~> 0.3.5\"},\n {:bypass, \"~> 2.1\", only: [:test]},\n {:mix_test_watch, \"~> 1.0\", only: [:dev, :test], runtime: false}\n ]\n end\nend\n"
},
{
"path": "owl.sh",
"content": "#!/usr/bin/env sh\n\nTMP_ENV_FILE=.owl_env.tmp\nIMAGE_NAME=open_owl-owl\n\nstore_relevant_env_variables() {\n printenv|grep OWL > $TMP_ENV_FILE\n}\n\nexists_docker_image() {\n docker image ls -q $IMAGE_NAME:latest 2> /dev/null\n}\n\nbuild_docker_image() {\n docker build -t $IMAGE_NAME:latest .\n}\n\nstore_relevant_env_variables\n\nif [ \"$(exists_docker_image)\" == \"\" ]; then\n echo \"Docker image does not exist yet, creating one...\"\n build_docker_image\nfi\n\ndocker run --rm -it \\\n -v `pwd`/auth_cache:/app/auth_cache \\\n -v `pwd`/results:/app/results \\\n -v `pwd`/recipes.yml:/app/recipes.yml \\\n --env-file $TMP_ENV_FILE \\\n $IMAGE_NAME $@\nresult=$?\n\nrm -f $TMP_ENV_FILE\n\nexit $result"
},
{
"path": "package.json",
"content": "{\n \"name\": \"open_owl\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"type\": \"module\",\n \"scripts\": {\n \"start\": \"npm run start:dev\",\n \"start:prod\": \"node dist/main.js\",\n \"start:dev\": \"ts-node-esm -T ts_src/main.ts\",\n \"build\": \"tsc\"\n },\n \"keywords\": [],\n \"dependencies\": {\n \"@types/js-yaml\": \"^4.0.5\",\n \"js-yaml\": \"^4.1.0\",\n \"playwright\": \"^1.31.2\"\n },\n \"devDependencies\": {\n \"ts-node\": \"^10.9.1\"\n }\n}"
},
{
"path": "recipes.yml",
"content": "adobe:\n login_url: https://adminconsole.adobe.com/team\n destination_url_pattern: https://adminconsole.adobe.com/**/overview\n username_selector: \"internal:label=\\\"Email address\\\"i\"\n password_selector: \"internal:label=\\\"Password\\\"i\"\n actions:\n - name: download_users\n http_method: GET\n url: https://bps-il.adobe.io/jil-api/v2/organizations/:organization/users?filter_exclude_domain=techacct.adobe.com&page=0&page_size=20&productConfigurations=true&sort=FNAME_LNAME&sort_order=ASC\n populate_placeholders_from_session_storage:\n - client_id\n - tokenValue\n - roles.organization\n headers: \n - x-api-key: :client_id\n - authorization: Bearer :tokenValue\n pagination:\n strategy: page_params_in_url\namplitude:\n login_url: https://analytics.amplitude.com/login/:org_name\n destination_url_pattern: https://analytics.amplitude.com/:org_name\n username_selector: \"input[placeholder=Email]\"\n password_selector: \"input[placeholder=Password]\"\n actions:\n - name: download_users\n http_method: POST\n body: '{\"query\":\"query Users {\\n users {\\n ...userFields\\n __typename\\n }\\n}\\n\\nfragment userFields on User {\\n id\\n alias\\n avatarVersion\\n blurb\\n createdAt\\n defaultAllProjectRole\\n defaultAppId\\n email\\n firstName\\n fullName\\n hasAvatar\\n hasOutstandingInvite\\n isConnectedToSlack\\n lastName\\n loginId\\n name\\n orgRole\\n orgTeam\\n title\\n pronouns\\n __typename\\n}\\n\"}'\n url: https://analytics.amplitude.com/t/graphql/org/:org_id?q=Users\n response_path: data.users\n headers:\n - content-type: application/json\n - origin: https://analytics.amplitude.com\ncalendly:\n login_url: https://calendly.com/app/login\n destination_url_pattern: https://calendly.com/event_types/user/me\n username_selector: \"input[placeholder='email address']\"\n password_selector: \"input[placeholder=password]\"\n actions:\n - name: download_users\n http_method: GET\n url: https://calendly.com/api/organization/memberships?filter_term=&sort_field=name&sort_order=asc\n response_path: results\n pagination:\n strategy: next_page_in_body\n next_page_response_path: pagination.next_page\n page_query_param: page\nfigma:\n login_url: https://www.figma.com/login\n destination_url_pattern: https://www.figma.com/files/**\n username_selector: \"input[placeholder=Email]\"\n password_selector: \"input[placeholder=Password]\"\n actions:\n - name: download_users\n http_method: GET\n url: https://www.figma.com/api/teams/:team_id/members\n response_path: meta\nloom:\n login_url: https://www.loom.com/settings/workspace\n destination_url_pattern: https://www.loom.com/settings/workspace\n username_selector: \"input[placeholder='Enter your email to continue…']\"\n password_selector: \"#password\"\n actions:\n - name: download_users\n http_method: POST\n body: '[{\"operationName\":\"FilterWorkspaceMembers\",\"variables\":{\"roles\":[\"admin\",\"viewer\",\"creator\",\"creator_lite\",\"guest\"],\"status\":\"active\",\"query\":\"\",\"first\":20,\"after\":\"\"},\"query\":\"query FilterWorkspaceMembers($query: String!, $roles: [OrganizationMemberRole!]!, $status: OrganizationMemberStatus!, $first: Int!, $after: String) {\\n result: searchPaginatedWorkspaceMembers {\\n ... on SearchPaginatedWorkspaceMembersResult {\\n accepted(\\n query: $query\\n roles: $roles\\n status: $status\\n first: $first\\n after: $after\\n ) {\\n edges {\\n node {\\n member_role\\n member_status\\n user {\\n id\\n email\\n display_name\\n first_name\\n last_name\\n createdAt\\n avatars {\\n thumb\\n __typename\\n }\\n __typename\\n }\\n pending_downgrade {\\n to_role\\n status\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n pageInfo {\\n hasPreviousPage\\n hasNextPage\\n startCursor\\n endCursor\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n}\\n\"}]'\n url: https://www.loom.com/graphql\n response_path: data.result.accepted.edges\n headers:\n - content-type: application/json\n pagination:\n strategy: next_cursor_in_body_to_send_as_json\n next_cursor_response_path: data.result.accepted.pageInfo.endCursor\n has_next_page_response_path: data.result.accepted.pageInfo.hasNextPage\n json_body_cursor_path: variables.after\nmezmo:\n login_url: https://app.mezmo.com/account/signin\n destination_url_pattern: https://app.mezmo.com/**/logs/*\n username_selector: \"data-testid=email\"\n password_selector: \"data-testid=password\"\n actions:\n - name: download_users\n http_method: GET\n url: https://app.mezmo.com/manage/get-team-members\n response_path: users\n headers: \n - x-account-context: :account_id\n - x-requested-with: XMLHttpRequest\nmiro:\n login_url: https://miro.com/login\n destination_url_pattern: https://miro.com/app/**\n username_selector: \"data-testid=mr-form-login-email-1\"\n password_selector: \"data-testid=mr-form-login-password-1\"\n actions:\n - name: download_users\n http_method: GET\n url: https://miro.com/api/v1/accounts/:team_id/users\n response_path: data\n pagination:\n strategy: url_in_body\n next_url_response_path: nextLink\n"
},
{
"path": "test/api_client_test.exs",
"content": "defmodule ApiClientTest do\n use ExUnit.Case, async: true\n\n alias OpenOwl.ApiClient\n alias OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson\n alias OpenOwl.PaginationStrategy.NextPageInBody\n alias OpenOwl.PaginationStrategy.OffsetParamsInUrl\n alias OpenOwl.PaginationStrategy.PageParamsInUrl\n alias OpenOwl.PaginationStrategy.UrlInBody\n\n # TODO: maybe add bypass to tests transparently\n # https://github.com/wojtekmach/req/issues/137\n setup do\n bypass = Bypass.open()\n [bypass: bypass, url: \"http://localhost:#{bypass.port}\"]\n end\n\n defp get_relevant_headers(headers) do\n Enum.reject(headers, &(elem(&1, 0) in [\"host\", \"content-length\"]))\n end\n\n describe \"do_request/6\" do\n @default_headers [\n {\"accept\", \"*/*\"},\n {\"accept-encoding\", \"gzip\"},\n {\"cookie\", \"\"},\n {\"user-agent\",\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36\"}\n ]\n @json_content_type {\"content-type\", \"application/json\"}\n @pagination %UrlInBody{strategy: :url_in_body, next_url_response_path: \"nextLink\"}\n\n for http_method <- [:get, :post, :put, :patch, :delete] do\n http_method_uppercase = Atom.to_string(http_method) |> String.upcase()\n\n test \"#{http_method_uppercase} returns response body for one page when success\", c do\n endpoint = \"/do/sth\"\n payload = [%{\"k\" => \"v\"}]\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n {:ok, body, conn} = Plug.Conn.read_body(conn)\n assert body == \"a=b\"\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(200, Jason.encode!(%{\"data\" => payload}))\n end)\n\n assert {:ok, payload} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n \"a=b\",\n c.url <> endpoint,\n [],\n \"data\",\n @pagination,\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for one page with deeper response path when success\",\n c do\n endpoint = \"/do/sth\"\n payload = [%{\"k\" => \"v\"}]\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(200, Jason.encode!(%{\"data\" => %{\"deeper\" => payload}}))\n end)\n\n assert {:ok, payload} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n nil,\n c.url <> endpoint,\n [],\n \"data.deeper\",\n @pagination,\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for one page without pagination strategy when success\",\n c do\n endpoint = \"/do/sth\"\n payload = [%{\"k\" => \"v\"}]\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(200, Jason.encode!(%{\"data\" => payload}))\n end)\n\n assert {:ok, payload} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n nil,\n c.url <> endpoint,\n [],\n \"data\",\n nil,\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for multiple pages with url_in_body strategy when success\",\n c do\n endpoint1 = \"/do/sth/p/1\"\n endpoint2 = \"/do/sth/p/2\"\n payload1 = [%{\"k\" => \"v1\"}, %{\"k\" => \"v2\"}]\n payload2 = [%{\"k\" => \"v3\"}, %{\"k\" => \"v4\"}]\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint1, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(\n 200,\n Jason.encode!(%{\"data\" => payload1, \"nextLink\" => %{\"deeper\" => c.url <> endpoint2}})\n )\n end)\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint2, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(\n 200,\n # TODO: test nextLink = nil\n Jason.encode!(%{\"data\" => payload2, \"nextLink\" => %{\"deeper\" => \"\"}})\n )\n end)\n\n assert {:ok, payload1 ++ payload2} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n nil,\n c.url <> endpoint1,\n [],\n \"data\",\n %UrlInBody{strategy: :url_in_body, next_url_response_path: \"nextLink.deeper\"},\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for multiple pages with next_page_in_body strategy when success\",\n c do\n endpoint = \"/do/sth\"\n payload1 = [%{\"k\" => \"v1\"}, %{\"k\" => \"v2\"}]\n payload2 = [%{\"k\" => \"v3\"}, %{\"k\" => \"v4\"}]\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n first_page? = not Map.has_key?(conn.query_params, \"page\")\n\n if not first_page? do\n assert conn.query_params == %{\"page\" => \"2\", \"param\" => \"stays\"}\n end\n\n response =\n if first_page?,\n do: %{\"data\" => payload1, \"pagination\" => %{\"next_page\" => 2}},\n # TODO: test next_page = \"\"\n else: %{\"data\" => payload2, \"pagination\" => %{\"next_page\" => nil}}\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(\n 200,\n Jason.encode!(response)\n )\n end)\n\n assert {:ok, payload1 ++ payload2} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n nil,\n c.url <> endpoint <> \"?param=stays\",\n [],\n \"data\",\n %NextPageInBody{\n strategy: :next_page_in_body,\n next_page_response_path: \"pagination.next_page\",\n page_query_param: \"page\"\n },\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for multiple pages with next_cursor_in_body_populate_as_placeholder strategy when success\",\n c do\n sent_body = %{\"a\" => \"a1\", \"vars\" => %{\"after\" => \"\", \"sth\" => \"keep\"}}\n endpoint = \"/do/sth\"\n payload1 = [%{\"k\" => \"v1\"}, %{\"k\" => \"v2\"}]\n payload2 = [%{\"k\" => \"v3\"}, %{\"k\" => \"v4\"}]\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n req_body =\n with {:ok, body, _conn} <- Plug.Conn.read_body(conn),\n {:ok, body} <- Jason.decode(body) do\n body\n else\n {:error, _} -> %{}\n end\n\n first_page? = req_body[\"vars\"][\"after\"] == \"\"\n\n if first_page? do\n assert req_body == sent_body\n assert get_relevant_headers(conn.req_headers) == @default_headers\n else\n assert req_body == put_in(sent_body, [\"vars\", \"after\"], \"abc\")\n\n assert get_relevant_headers(conn.req_headers) ==\n List.insert_at(@default_headers, 2, @json_content_type)\n end\n\n response =\n if first_page?,\n do: %{\n \"data\" => payload1,\n \"pagination\" => %{\"next_cursor\" => \"abc\", \"has_next\" => true}\n },\n else: %{\n \"data\" => payload2,\n \"pagination\" => %{\"next_cursor\" => \"efg\", \"has_next\" => false}\n }\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(\n 200,\n Jason.encode!(response)\n )\n end)\n\n assert {:ok, payload1 ++ payload2} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n Jason.encode!(sent_body),\n c.url <> endpoint,\n [],\n \"data\",\n %NextCursorInBodyToSendAsJson{\n strategy: :next_cursor_in_body_to_send_as_json,\n next_cursor_response_path: \"pagination.next_cursor\",\n has_next_page_response_path: \"pagination.has_next\",\n json_body_cursor_path: \"vars.after\"\n },\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for multiple pages with page_params_in_url strategy when success\",\n c do\n endpoint = \"/do/sth\"\n payload1 = [%{\"k\" => \"v1\"}, %{\"k\" => \"v2\"}]\n payload2 = [%{\"k\" => \"v3\"}, %{\"k\" => \"v4\"}]\n payload3 = []\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n page_param = conn.query_params[\"page\"]\n assert conn.query_params == %{\"page\" => page_param, \"param\" => \"stays\"}\n\n response =\n case page_param do\n \"0\" -> payload1\n \"1\" -> payload2\n \"2\" -> payload3\n end\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(\n 200,\n Jason.encode!(response)\n )\n end)\n\n assert {:ok, payload1 ++ payload2} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n nil,\n c.url <> endpoint <> \"?page=0¶m=stays\",\n [],\n nil,\n %PageParamsInUrl{strategy: :page_params_in_url},\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for multiple pages with page_params_in_url and custom page_query_param strategy when success\",\n c do\n endpoint = \"/do/sth\"\n payload1 = [%{\"k\" => \"v1\"}, %{\"k\" => \"v2\"}]\n payload2 = [%{\"k\" => \"v3\"}, %{\"k\" => \"v4\"}]\n payload3 = []\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n page_param = conn.query_params[\"p\"]\n assert conn.query_params == %{\"p\" => page_param, \"param\" => \"stays\"}\n\n response =\n case page_param do\n \"0\" -> payload1\n \"1\" -> payload2\n \"2\" -> payload3\n end\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(\n 200,\n Jason.encode!(response)\n )\n end)\n\n assert {:ok, payload1 ++ payload2} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n nil,\n c.url <> endpoint <> \"?p=0¶m=stays\",\n [],\n nil,\n %PageParamsInUrl{strategy: :page_params_in_url, page_query_param: \"p\"},\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for multiple pages with offset_params_in_url strategy when success\",\n c do\n endpoint = \"/do/sth\"\n payload1 = [%{\"k\" => \"v1\"}, %{\"k\" => \"v2\"}]\n payload2 = [%{\"k\" => \"v3\"}, %{\"k\" => \"v4\"}]\n payload3 = []\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n offset_param = conn.query_params[\"offset\"]\n limit_param = conn.query_params[\"limit\"]\n\n assert conn.query_params == %{\n \"offset\" => offset_param,\n \"limit\" => limit_param,\n \"param\" => \"stays\"\n }\n\n response =\n case offset_param do\n \"0\" -> payload1\n \"2\" -> payload2\n \"4\" -> payload3\n end\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(\n 200,\n Jason.encode!(response)\n )\n end)\n\n assert {:ok, payload1 ++ payload2} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n nil,\n c.url <> endpoint <> \"?offset=0&limit=2¶m=stays\",\n [],\n nil,\n %OffsetParamsInUrl{strategy: :offset_params_in_url},\n %{},\n []\n )\n end\n\n test \"#{http_method_uppercase} returns response body for multiple pages with offset_params_in_url and custom params strategy when success\",\n c do\n endpoint = \"/do/sth\"\n payload1 = [%{\"k\" => \"v1\"}, %{\"k\" => \"v2\"}]\n payload2 = [%{\"k\" => \"v3\"}, %{\"k\" => \"v4\"}]\n payload3 = []\n\n Bypass.expect(c.bypass, unquote(http_method_uppercase), endpoint, fn conn ->\n assert get_relevant_headers(conn.req_headers) == @default_headers\n\n offset_param = conn.query_params[\"start_offset\"]\n limit_param = conn.query_params[\"size\"]\n\n assert conn.query_params == %{\n \"start_offset\" => offset_param,\n \"size\" => limit_param,\n \"param\" => \"stays\"\n }\n\n response =\n case offset_param do\n \"0\" -> payload1\n \"2\" -> payload2\n \"4\" -> payload3\n end\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(\n 200,\n Jason.encode!(response)\n )\n end)\n\n assert {:ok, payload1 ++ payload2} ==\n ApiClient.do_request(\n [],\n %{},\n unquote(http_method),\n nil,\n c.url <> endpoint <> \"?start_offset=0&size=2¶m=stays\",\n [],\n nil,\n %OffsetParamsInUrl{\n strategy: :offset_params_in_url,\n offset_query_param: \"start_offset\",\n limit_query_param: \"size\"\n },\n %{},\n []\n )\n end\n end\n\n test \"replaces placeholder in passed URL and headers properly\", c do\n endpoint_prefix = \"/do/sth:not/\"\n payload = [%{\"k\" => \"v\"}]\n\n Bypass.expect(c.bypass, \"GET\", \"#{endpoint_prefix}123\", fn conn ->\n assert get_relevant_headers(conn.req_headers) ==\n @default_headers ++\n [{\"x-dynamic\", \"pre_123-post\"}, {\"x-not\", \":there\"}, {\"x-static\", \"Blub 1\"}]\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(200, Jason.encode!(%{\"data\" => payload}))\n end)\n\n assert {:ok, payload} ==\n ApiClient.do_request(\n [],\n %{},\n :get,\n nil,\n c.url <> endpoint_prefix <> \":team_id\",\n [\n {\"X-Dynamic\", \"pre_:team_id-post\"},\n {\"X-Not\", \":there\"},\n {\"X-Static\", \"Blub 1\"}\n ],\n \"data\",\n @pagination,\n %{team_id: \"123\"},\n []\n )\n end\n\n test \"uses session_storage to populate new placeholders and replaces it in passed URL and headers properly\",\n c do\n endpoint_prefix = \"/do/sth:not/\"\n payload = [%{\"k\" => \"v\"}]\n\n Bypass.expect(c.bypass, \"GET\", \"#{endpoint_prefix}123/v2\", fn conn ->\n assert get_relevant_headers(conn.req_headers) ==\n @default_headers ++\n [\n {\"x-passed\", \"pre_123-post\"},\n {\"x-populated\", \"pre_v3b\"},\n {\"x-static\", \"Blub 1\"}\n ]\n\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(200, Jason.encode!(%{\"data\" => payload}))\n end)\n\n assert {:ok, payload} ==\n ApiClient.do_request(\n [],\n %{\n \"whatever_1\" => Jason.encode!(%{\"kone\" => \"v1\", \"ktwo\" => \"v2\"}),\n \"whatever_2\" =>\n Jason.encode!(%{\"kthree\" => [%{\"kthree_a\" => \"v3a\", \"kthree_b\" => \"v3b\"}]}),\n \"none\" => \"nope\"\n },\n :get,\n nil,\n c.url <> endpoint_prefix <> \":team_id/:ktwo\",\n [\n {\"X-passed\", \"pre_:team_id-post\"},\n {\"X-populated\", \"pre_:kthree_b\"},\n {\"X-Static\", \"Blub 1\"}\n ],\n \"data\",\n @pagination,\n %{team_id: \"123\"},\n [\"ktwo\", \"kthree.kthree_b\"]\n )\n end\n\n for status <- [400, 401, 403, 404] do\n test \"returns http_error when status is #{status}\", c do\n endpoint = \"/do/sth\"\n payload = %{\"error\" => \"oops\"}\n\n Bypass.expect(c.bypass, \"GET\", endpoint, fn conn ->\n Plug.Conn.put_resp_header(conn, \"content-type\", \"application/json\")\n |> Plug.Conn.send_resp(unquote(status), Jason.encode!(payload))\n end)\n\n response =\n ApiClient.do_request(\n [],\n %{},\n :get,\n nil,\n c.url <> endpoint,\n [],\n \"data\",\n @pagination,\n %{},\n []\n )\n\n assert {:http_error, {unquote(status), ^payload}} = response\n end\n end\n end\nend\n"
},
{
"path": "test/fixtures/test_recipes.yml",
"content": "adobe:\n login_url: https://adminconsole.adobe.com/team\n destination_url_pattern: https://adminconsole.adobe.com/**/overview\n username_selector: \"internal:label=\\\"Email address\\\"i\"\n password_selector: \"internal:label=\\\"Password\\\"i\"\n actions:\n - name: download_users\n http_method: GET\n url: https://adobe.io/api/sth\n populate_placeholders_from_session_storage:\n - client_id\n - tokenValue\n - roles.organization\n headers: \n - x-api-key: :client_id\n - authorization: Bearer :tokenValue\n pagination:\n strategy: page_params_in_url\n page_query_param: \"p\"\nbadobe:\n login_url: https://adminconsole.adobe.com/team\n destination_url_pattern: https://adminconsole.adobe.com/**/overview\n username_selector: \"internal:label=\\\"Email address\\\"i\"\n password_selector: \"internal:label=\\\"Password\\\"i\"\n actions:\n - name: download_users\n http_method: GET\n url: https://adobe.io/api/sth\n pagination:\n strategy: offset_params_in_url\n offset_query_param: \"start_offset\"\n limit_query_param: \"size\"\nasana:\n login_url: https://asana.com/login\n destination_url_pattern: https://asana.com/app/**\n username_selector: \"#a\"\n password_selector: \"#b\"\n actions:\n - name: change_sth\n http_method: POST\n body: \"{\\\"query\\\":\\\"query Users\\\"}\"\n url: https://asana.com/api/sth\n headers: \n - X-Dynamic: :team_id\n - X-Static: Blub 1\n response_path: meta.data\ncalendly:\n login_url: https://calendly.com/app/login\n destination_url_pattern: https://calendly.com/event_types/user/me\n username_selector: \"input[placeholder='email address']\"\n password_selector: \"input[placeholder=password]\"\n actions:\n - name: download_users\n http_method: GET\n url: https://calendly.com/api/organization/memberships\n response_path: results\n pagination:\n strategy: next_page_in_body\n next_page_response_path: pagination.next_page\n page_query_param: page\nloom:\n login_url: https://loom.com/login\n destination_url_pattern: https://loom.com/app/**\n username_selector: \"#a\"\n password_selector: \"#b\"\n actions:\n - name: download_users\n http_method: GET\n url: https://loom.com/api/v1\n response_path: data\n pagination:\n strategy: next_cursor_in_body_to_send_as_json\n next_cursor_response_path: pagination.next_page\n has_next_page_response_path: pagination.has_next_page\n json_body_cursor_path: variables.after\n \nmiro:\n login_url: https://miro.com/login\n destination_url_pattern: https://miro.com/app/**\n username_selector: \"data-testid=mr1\"\n password_selector: \"data-testid=mr2\"\n actions:\n - name: download_users\n http_method: GET\n url: https://miro.com/api/v1/accounts/:team_id/users\n response_path: data\n pagination:\n strategy: url_in_body\n next_url_response_path: nextLink\n "
},
{
"path": "test/helpers/api_utils_test.exs",
"content": "defmodule ApiUtilsTest do\n use ExUnit.Case, async: true\n doctest OpenOwl.Helpers.ApiUtils, import: true\nend\n"
},
{
"path": "test/helpers/cli_utils_test.exs",
"content": "defmodule CliUtilsTest do\n use ExUnit.Case, async: true\n doctest OpenOwl.Helpers.CliUtils, import: true\nend\n"
},
{
"path": "test/helpers/struct_utils_test.exs",
"content": "defmodule StructUtilsTest do\n use ExUnit.Case, async: true\n\n alias OpenOwl.Helpers.StructUtils\n\n defmodule TestStruct do\n @enforce_keys [:title]\n defstruct [:id, :title]\n end\n\n defmodule AnotherTestStruct do\n defstruct [:id, :title]\n end\n\n describe \"to_struct/2 creates the specified struct\" do\n test \"with passed empty Map attrs\" do\n assert StructUtils.to_struct(TestStruct, %{}) == %TestStruct{id: nil, title: nil}\n\n assert StructUtils.to_struct(TestStruct, %{}) |> Map.to_list() == [\n __struct__: StructUtilsTest.TestStruct,\n id: nil,\n title: nil\n ]\n end\n\n test \"with passed string Map attrs\" do\n assert StructUtils.to_struct(TestStruct, %{\"id\" => \"id\"}) == %TestStruct{\n id: \"id\",\n title: nil\n }\n\n assert StructUtils.to_struct(TestStruct, %{\"title\" => \"title\"}) == %TestStruct{\n id: nil,\n title: \"title\"\n }\n end\n\n test \"with passed atom Map attrs\" do\n assert StructUtils.to_struct(TestStruct, %{id: \"id\"}) == %TestStruct{\n id: \"id\",\n title: nil\n }\n\n assert StructUtils.to_struct(TestStruct, %{title: \"title\"}) == %TestStruct{\n id: nil,\n title: \"title\"\n }\n end\n\n test \"with passed struct attrs\" do\n assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{}) |> Map.to_list() == [\n __struct__: StructUtilsTest.TestStruct,\n id: nil,\n title: nil\n ]\n\n assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{id: \"id\"}) == %TestStruct{\n id: \"id\",\n title: nil\n }\n\n assert StructUtils.to_struct(TestStruct, %AnotherTestStruct{title: \"title\"}) == %TestStruct{\n id: nil,\n title: \"title\"\n }\n end\n end\nend\n"
},
{
"path": "test/login_flow_wrapper_test.exs",
"content": "defmodule LoginFlowWrapperTest do\n use ExUnit.Case, async: true\n doctest OpenOwl.LoginFlowWrapper, import: true\nend\n"
},
{
"path": "test/recipes_test.exs",
"content": "defmodule RecipesTest do\n use ExUnit.Case, async: true\n\n alias OpenOwl.Recipes.Recipe\n alias OpenOwl.Recipes.Action\n alias OpenOwl.Recipes\n\n @test_recipe1 %Recipe{\n login_url: \"https://example1.com\",\n destination_url_pattern: \"dup\",\n username_selector: \"us\",\n password_selector: \"ps\"\n }\n @test_recipe2 %{@test_recipe1 | login_url: \"https://example2.com\"}\n @action1 %Action{\n name: :download,\n http_method: :get,\n url: \"https://example.com/:url_id\",\n headers: [{\"x-1\", \"cool_:header_id_one\"}, {\"x-2\", \":url_id/:header_id_two\"}]\n }\n @action2 %Action{name: :list, http_method: :get, url: \"https://example.com/:url_id\"}\n\n describe \"load_recipes/1\" do\n test \"returns parsed recipes from a yaml file\" do\n assert {:ok, recipes} = Recipes.load_recipes(\"test/fixtures/test_recipes.yml\")\n\n assert recipes == %{\n adobe: %OpenOwl.Recipes.Recipe{\n login_url: \"https://adminconsole.adobe.com/team\",\n destination_url_pattern: \"https://adminconsole.adobe.com/**/overview\",\n username_selector: \"internal:label=\\\"Email address\\\"i\",\n password_selector: \"internal:label=\\\"Password\\\"i\",\n actions: [\n %OpenOwl.Recipes.Action{\n name: \"download_users\",\n http_method: :get,\n url: \"https://adobe.io/api/sth\",\n populate_placeholders_from_session_storage:\n ~w(client_id tokenValue roles.organization),\n response_path: nil,\n headers: [\n {\"x-api-key\", \":client_id\"},\n {\"authorization\", \"Bearer :tokenValue\"}\n ],\n pagination: %OpenOwl.PaginationStrategy.PageParamsInUrl{\n strategy: \"page_params_in_url\",\n page_query_param: \"p\"\n }\n }\n ]\n },\n badobe: %OpenOwl.Recipes.Recipe{\n login_url: \"https://adminconsole.adobe.com/team\",\n destination_url_pattern: \"https://adminconsole.adobe.com/**/overview\",\n username_selector: \"internal:label=\\\"Email address\\\"i\",\n password_selector: \"internal:label=\\\"Password\\\"i\",\n actions: [\n %OpenOwl.Recipes.Action{\n name: \"download_users\",\n http_method: :get,\n url: \"https://adobe.io/api/sth\",\n response_path: nil,\n pagination: %OpenOwl.PaginationStrategy.OffsetParamsInUrl{\n strategy: \"offset_params_in_url\",\n offset_query_param: \"start_offset\",\n limit_query_param: \"size\"\n }\n }\n ]\n },\n asana: %OpenOwl.Recipes.Recipe{\n login_url: \"https://asana.com/login\",\n destination_url_pattern: \"https://asana.com/app/**\",\n username_selector: \"#a\",\n password_selector: \"#b\",\n actions: [\n %OpenOwl.Recipes.Action{\n name: \"change_sth\",\n http_method: :post,\n body: \"{\\\"query\\\":\\\"query Users\\\"}\",\n url: \"https://asana.com/api/sth\",\n response_path: \"meta.data\",\n headers: [{\"X-Dynamic\", \":team_id\"}, {\"X-Static\", \"Blub 1\"}]\n }\n ]\n },\n calendly: %OpenOwl.Recipes.Recipe{\n login_url: \"https://calendly.com/app/login\",\n destination_url_pattern: \"https://calendly.com/event_types/user/me\",\n username_selector: \"input[placeholder='email address']\",\n password_selector: \"input[placeholder=password]\",\n actions: [\n %OpenOwl.Recipes.Action{\n name: \"download_users\",\n http_method: :get,\n url: \"https://calendly.com/api/organization/memberships\",\n response_path: \"results\",\n pagination: %OpenOwl.PaginationStrategy.NextPageInBody{\n strategy: \"next_page_in_body\",\n next_page_response_path: \"pagination.next_page\",\n page_query_param: \"page\"\n },\n headers: []\n }\n ]\n },\n loom: %OpenOwl.Recipes.Recipe{\n login_url: \"https://loom.com/login\",\n destination_url_pattern: \"https://loom.com/app/**\",\n username_selector: \"#a\",\n password_selector: \"#b\",\n actions: [\n %OpenOwl.Recipes.Action{\n name: \"download_users\",\n http_method: :get,\n url: \"https://loom.com/api/v1\",\n response_path: \"data\",\n pagination: %OpenOwl.PaginationStrategy.NextCursorInBodyToSendAsJson{\n strategy: \"next_cursor_in_body_to_send_as_json\",\n next_cursor_response_path: \"pagination.next_page\",\n has_next_page_response_path: \"pagination.has_next_page\",\n json_body_cursor_path: \"variables.after\"\n },\n headers: []\n }\n ]\n },\n miro: %OpenOwl.Recipes.Recipe{\n login_url: \"https://miro.com/login\",\n destination_url_pattern: \"https://miro.com/app/**\",\n username_selector: \"data-testid=mr1\",\n password_selector: \"data-testid=mr2\",\n actions: [\n %OpenOwl.Recipes.Action{\n name: \"download_users\",\n http_method: :get,\n url: \"https://miro.com/api/v1/accounts/:team_id/users\",\n response_path: \"data\",\n pagination: %OpenOwl.PaginationStrategy.UrlInBody{\n strategy: \"url_in_body\",\n next_url_response_path: \"nextLink\"\n },\n headers: []\n }\n ]\n }\n }\n end\n\n test \"returns error if file was not found\" do\n assert {:error, %YamlElixir.FileNotFoundError{}} =\n Recipes.load_recipes(\"test/fixtures/not_existent.yml\")\n end\n end\n\n test \"get_recipe/2 returns recipe for a slug\" do\n recipes = %{test1: @test_recipe1, test2: @test_recipe2}\n\n assert Recipes.get_recipe(%{}, :test) == nil\n assert Recipes.get_recipe(recipes, :test) == nil\n assert Recipes.get_recipe(recipes, :test1) == @test_recipe1\n assert Recipes.get_recipe(recipes, :test2) == @test_recipe2\n assert Recipes.get_recipe(recipes, \"test\") == nil\n assert Recipes.get_recipe(recipes, \"test1\") == @test_recipe1\n assert Recipes.get_recipe(recipes, \"test2\") == @test_recipe2\n end\n\n test \"get_action_for_name/2 returns action for name or nil\" do\n actions = [@action1, @action2]\n assert Recipes.get_action_for_name(actions, :not_existent) == nil\n assert Recipes.get_action_for_name(actions, :download) == @action1\n assert Recipes.get_action_for_name(actions, :list) == @action2\n end\n\n describe \"get_required_parameters_for_recipe/2\" do\n test \"returns required parameters for special recipe action login\" do\n recipe_login_param = %{@test_recipe1 | login_url: \"https://example.com/:org_id/:bla\"}\n\n recipe_dest_param = %{\n @test_recipe1\n | destination_url_pattern: \"https://example.com/*/:bla\"\n }\n\n recipe_both_duplicated = %{\n @test_recipe1\n | login_url: \"https://example.com/:org_id/:bla\",\n destination_url_pattern: \"https://example.com/*/:bla\"\n }\n\n assert Recipes.get_required_parameters_for_recipe(@test_recipe1, \"login\") == [\n :username,\n :password\n ]\n\n assert Recipes.get_required_parameters_for_recipe(recipe_login_param, \"login\") == [\n :username,\n :password,\n :org_id,\n :bla\n ]\n\n assert Recipes.get_required_parameters_for_recipe(recipe_dest_param, \"login\") == [\n :username,\n :password,\n :bla\n ]\n\n assert Recipes.get_required_parameters_for_recipe(recipe_both_duplicated, \"login\") == [\n :username,\n :password,\n :bla,\n :org_id\n ]\n end\n\n test \"returns required parameters for dynamic recipe action\" do\n recipe_base = %{\n @test_recipe1\n | login_url: \"https://example.com/:org_id\",\n destination_url_pattern: \":dest_id\"\n }\n\n recipe_with_actions = %{recipe_base | actions: [@action1, @action2]}\n\n # ignores login parameters\n assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :not_existent) == []\n\n assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :download) == [\n :url_id,\n :header_id_two,\n :header_id_one\n ]\n\n assert Recipes.get_required_parameters_for_recipe(recipe_with_actions, :list) == [:url_id]\n end\n end\n\n describe \"validate_recipe_parameters/3\" do\n @recipe_with_params %{\n @test_recipe1\n | login_url: \"https://example.com/:org_id\",\n actions: [@action1]\n }\n\n test \"returns ok with passed parameter map if all required parameters are present\" do\n # without params\n login_parameters_pure = %{username: \"u\", password: \"p\"}\n\n assert Recipes.validate_recipe_parameters(@test_recipe1, \"login\", login_parameters_pure) ==\n {:ok, login_parameters_pure}\n\n login_parameters_not_existing = Map.put(login_parameters_pure, :not_existent, 1)\n\n assert Recipes.validate_recipe_parameters(\n @test_recipe1,\n \"login\",\n login_parameters_not_existing\n ) ==\n {:ok, login_parameters_not_existing}\n\n login_parameters_required = Map.put(login_parameters_pure, :org_id, \"1\")\n\n assert Recipes.validate_recipe_parameters(\n @recipe_with_params,\n \"login\",\n login_parameters_required\n ) ==\n {:ok, login_parameters_required}\n\n action_parameters = %{\n url_id: \"1\",\n header_id_two: 2,\n header_id_one: \"a3\"\n }\n\n assert Recipes.validate_recipe_parameters(\n @recipe_with_params,\n @action1.name,\n action_parameters\n ) ==\n {:ok, action_parameters}\n end\n\n test \"returns error with list of missing parameters if not all required parameters are present\" do\n assert Recipes.validate_recipe_parameters(@recipe_with_params, \"login\", %{wrong: 1}) ==\n {:error, [:username, :password, :org_id]}\n\n assert Recipes.validate_recipe_parameters(@recipe_with_params, \"login\", %{\n password: \"p\",\n wrong: 1\n }) ==\n {:error, [:username, :org_id]}\n\n assert Recipes.validate_recipe_parameters(@recipe_with_params, @action1.name, %{\n wrong: 1,\n header_id_two: \"2\"\n }) ==\n {:error, [:url_id, :header_id_one]}\n end\n end\n\n test \"placeholder_regex/0 returns regex for placeholders\" do\n assert Recipes.placeholder_regex() == ~r/:([a-zA-Z]{1}([_]*[a-zA-Z]+)*)/\n end\nend\n"
},
{
"path": "test/response_transformer_test.exs",
"content": "defmodule ResponseTransformerTest do\n use ExUnit.Case, async: true\n\n alias OpenOwl.ResponseTransformer\n\n @record1 %{\n \"email\" => \"batman@wayne.org\",\n \"emailConfirmed\" => true,\n \"id\" => 123,\n \"lastLoginDate\" => \"2022-12-13T14:07:59.371Z\",\n \"name\" => \"Bruce Wayne\",\n \"team_role\" => %{\n \"level\" => 999,\n \"pending\" => false,\n \"pending_email\" => nil,\n \"resource_type\" => \"team\",\n \"role\" => %{\n \"name\" => \"Heroes\",\n \"id\" => \"abc\"\n },\n \"user_id\" => \"456\"\n }\n }\n @record2 %{@record1 | \"email\" => \"robin@wayne.org\", \"name\" => \"Robin\"}\n @map_list [@record1, @record2]\n\n describe \"records_to_list_with_header/1\" do\n test \"returns table structure with header\" do\n assert ResponseTransformer.records_to_list_with_header(@map_list) == [\n [\n \"email\",\n \"emailConfirmed\",\n \"id\",\n \"lastLoginDate\",\n \"name\",\n \"team_role.level\",\n \"team_role.pending\",\n \"team_role.pending_email\",\n \"team_role.resource_type\",\n \"team_role.role.id\",\n \"team_role.role.name\",\n \"team_role.user_id\"\n ],\n [\n \"batman@wayne.org\",\n true,\n 123,\n \"2022-12-13T14:07:59.371Z\",\n \"Bruce Wayne\",\n 999,\n false,\n nil,\n \"team\",\n \"abc\",\n \"Heroes\",\n \"456\"\n ],\n [\n \"robin@wayne.org\",\n true,\n 123,\n \"2022-12-13T14:07:59.371Z\",\n \"Robin\",\n 999,\n false,\n nil,\n \"team\",\n \"abc\",\n \"Heroes\",\n \"456\"\n ]\n ]\n end\n\n test \"returns empty list if input was an empty list\" do\n assert ResponseTransformer.records_to_list_with_header([]) == []\n end\n\n test \"raises error if it is not a list\" do\n assert_raise FunctionClauseError, fn ->\n ResponseTransformer.records_to_list_with_header(@record1)\n end\n end\n end\n\n describe \"list_with_header_to_csv/1\" do\n test \"returns a CSV string\" do\n assert ResponseTransformer.list_with_header_to_csv([]) == \"\"\n\n assert ResponseTransformer.list_with_header_to_csv([~w(col_a col_b), ~w(hey you)]) ==\n \"col_a,col_b\\r\\nhey,you\\r\\n\"\n end\n\n test \"raises error if it is not a list\" do\n assert_raise FunctionClauseError, fn ->\n ResponseTransformer.list_with_header_to_csv(%{})\n end\n end\n end\n\n describe \"flat_record/1\" do\n test \"returns zero-level map untouched\" do\n record = %{\"hey\" => \"you\", \"what\" => 4, \"or\" => true}\n\n assert ResponseTransformer.flat_record(record) == record\n end\n\n test \"ignores fields that hold lists with maps\" do\n record = %{\n \"hey\" => \"you\",\n \"simple_array\" => [\"you\", 1],\n \"complex_array\" => [%{\"sub_map\" => 1}]\n }\n\n assert ResponseTransformer.flat_record(record) == %{\n \"hey\" => \"you\",\n \"simple_array\" => [\"you\", 1]\n }\n end\n\n test \"returns one-level map with concatted field names\" do\n record = %{\"hey\" => \"you\", \"what\" => 4, \"deep\" => %{\"name\" => \"cracker\", \"good\" => true}}\n\n assert ResponseTransformer.flat_record(record) == %{\n \"hey\" => \"you\",\n \"what\" => 4,\n \"deep.name\" => \"cracker\",\n \"deep.good\" => true\n }\n end\n\n test \"returns multi-level map with concatted field names\" do\n assert ResponseTransformer.flat_record(@record1) == %{\n \"email\" => \"batman@wayne.org\",\n \"emailConfirmed\" => true,\n \"id\" => 123,\n \"lastLoginDate\" => \"2022-12-13T14:07:59.371Z\",\n \"name\" => \"Bruce Wayne\",\n \"team_role.level\" => 999,\n \"team_role.pending\" => false,\n \"team_role.pending_email\" => nil,\n \"team_role.resource_type\" => \"team\",\n \"team_role.role.name\" => \"Heroes\",\n \"team_role.role.id\" => \"abc\",\n \"team_role.user_id\" => \"456\"\n }\n end\n end\nend\n"
},
{
"path": "test/test_helper.exs",
"content": "ExUnit.start(exclude: :skip)\n"
},
{
"path": "ts_src/login_flow.ts",
"content": "import { firefox, Page } from 'playwright';\nimport path from 'path';\nimport * as fs from 'fs';\n\nconst authCacheFolder = 'auth_cache';\nconst resultFolder = 'results';\nconst screenshotsFolder = 'login_screenshots';\n\nfunction maybeCreateScreenshotsFolder() {\n fs.mkdirSync(path.join(resultFolder, screenshotsFolder), { recursive: true });\n}\n\nfunction clearScreenshots() {\n fs.readdirSync(path.join(resultFolder, screenshotsFolder)).forEach(file => {\n fs.rmSync(path.join(resultFolder, screenshotsFolder, file));\n });\n}\n\nasync function makeScreenshot(page: Page, slug: String) {\n const screenshot = await page.screenshot();\n fs.writeFileSync(path.join(resultFolder, screenshotsFolder, Date.now() + '_' + slug + '.png'), screenshot);\n}\n\nexport async function loginAndSaveCookies(\n slug: string,\n url: string,\n destination_url_pattern: string,\n username_selector: string,\n password_selector: string,\n username: string,\n password: string): Promise {\n\n maybeCreateScreenshotsFolder();\n clearScreenshots();\n const browser = await firefox.launch({ headless: true });\n const context = await browser.newContext()\n const page = await context.newPage()\n\n await page.goto(url);\n await page.fill(username_selector, username);\n await makeScreenshot(page, slug);\n\n // if password field is not visible, press actively enter so that it appears\n try {\n await page.waitForSelector(password_selector, { timeout: 2 });\n } catch (e) {\n await page.press(username_selector, 'Enter');\n }\n await makeScreenshot(page, slug);\n\n await page.fill(password_selector, password);\n await page.press(password_selector, 'Enter');\n await makeScreenshot(page, slug);\n\n await page.waitForURL(destination_url_pattern, { timeout: 10000 });\n await makeScreenshot(page, slug);\n\n const cookies = await context.cookies();\n const cookieJson = JSON.stringify(cookies);\n fs.writeFileSync(path.join(authCacheFolder, slug + '_cookies.json'), cookieJson);\n const sessionStorage = await page.evaluate(() => JSON.stringify(sessionStorage));\n fs.writeFileSync(path.join(authCacheFolder, slug + '_session_storage.json'), sessionStorage, 'utf-8');\n\n clearScreenshots();\n await browser.close();\n}"
},
{
"path": "ts_src/main.ts",
"content": "import { loginAndSaveCookies } from './login_flow.js';\n\nconsole.log(\"started...\");\n\nloginAndSaveCookies(process.env.SLUG, process.env.URL, process.env.DESTINATION_URL_PATTERN,\n process.env.USERNAME_SELECTOR, process.env.PASSWORD_SELECTOR, process.env.USER,\n process.env.PASSWORD);\n\nconsole.log(\"DONE\");\n"
},
{
"path": "ts_src/recipe_manager.ts",
"content": "import type { VendorTemplate, VendorAction } from \"./types\";\nimport path from 'path';\nimport { fileURLToPath } from 'url';\nimport fs from 'fs';\nimport yaml from 'js-yaml';\n\nclass RecipeManager {\n recipes: { [key: string]: VendorTemplate };\n\n constructor() {\n // https://flaviocopes.com/fix-dirname-not-defined-es-module-scope/\n const __filename = fileURLToPath(import.meta.url);\n const __dirname = path.dirname(__filename);\n\n let recipesPath = path.join(__dirname, '../recipes.yml');\n this.recipes = yaml.load(fs.readFileSync(recipesPath).toString()) as { string: VendorTemplate }\n }\n\n getRecipes(): { [key: string]: VendorTemplate } {\n return this.recipes;\n }\n}\n\nexport default new RecipeManager();"
},
{
"path": "ts_src/types.ts",
"content": "export interface VendorTemplate {\n login_url: string;\n destination_url_pattern: string;\n username_selector: string;\n password_selector: string;\n actions?: VendorAction[];\n}\n\nexport interface VendorAction {\n name: string;\n http_method: HttpMethod;\n url: string;\n response_path: string;\n}\n\nexport enum HttpMethod {\n GET = 'GET',\n POST = 'POST',\n PUT = 'PUT',\n PATCH = 'POST',\n DELETE = 'DELETE'\n}"
},
{
"path": "tsconfig.json",
"content": "{\n \"compileOnSave\": true,\n \"compilerOptions\": {\n \"module\": \"ES2022\",\n \"target\": \"ES2022\",\n \"outDir\": \"dist\",\n \"moduleResolution\": \"node\",\n \"allowSyntheticDefaultImports\": true,\n \"noUnusedLocals\": false,\n \"skipLibCheck\": true,\n \"forceConsistentCasingInFileNames\": true,\n \"lib\": [\n \"ESNext\"\n ]\n },\n \"baseUrl\": \"./\",\n \"include\": [\n \"ts_src/**/*\"\n ]\n}"
}
]